From a2830d6aa2e8ad0da0e58f1642ed09723602e838 Mon Sep 17 00:00:00 2001
From: Bill Wendling <morbo@google.com>
Date: Thu, 15 Aug 2024 02:34:04 -0700
Subject: [PATCH 001/441] Revert "Remove empty line."

Accidental commit.

This reverts commit 894d3eebe2109f7dec488d3415d5c6236e55a0da.
---
 clang/lib/Sema/SemaExpr.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index c4aa02ff0c217d2..0f58eb2840211de 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -6593,6 +6593,7 @@ ExprResult Sema::BuildCallExpr(Scope *Scope, Expr *Fn, SourceLocation LParenLoc,
            "should only occur in error-recovery path.");
     return CallExpr::Create(Context, Fn, ArgExprs, Context.DependentTy,
                             VK_PRValue, RParenLoc, CurFPFeatureOverrides());
+  }
   return BuildResolvedCallExpr(Fn, NDecl, LParenLoc, ArgExprs, RParenLoc,
                                ExecConfig, IsExecConfig);
 }

From ed8cfb651327a00f6ccf2b26890ee921f16f64a2 Mon Sep 17 00:00:00 2001
From: Andrey Timonin <112198242+EtoAndruwa@users.noreply.github.com>
Date: Thu, 15 Aug 2024 12:40:55 +0300
Subject: [PATCH 002/441] [NFC][mlir][scf] Fix misspelling of replace (#101683)

---
 mlir/lib/Dialect/SCF/IR/SCF.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mlir/lib/Dialect/SCF/IR/SCF.cpp b/mlir/lib/Dialect/SCF/IR/SCF.cpp
index 4de8dacc0edbf33..e92d9503372cdf2 100644
--- a/mlir/lib/Dialect/SCF/IR/SCF.cpp
+++ b/mlir/lib/Dialect/SCF/IR/SCF.cpp
@@ -4322,7 +4322,7 @@ struct FoldConstantCase : OpRewritePattern<scf::IndexSwitchOp> {
 
     rewriter.inlineBlockBefore(&source, op);
     rewriter.eraseOp(terminator);
-    // Repalce the operation with a potentially empty list of results.
+    // Replace the operation with a potentially empty list of results.
     // Fold mechanism doesn't support the case where the result list is empty.
     rewriter.replaceOp(op, results);
 

From 5e95571a90b1ec193b735b7312c5c2559d7ee5ea Mon Sep 17 00:00:00 2001
From: Pavel Skripkin <paskripkin@gmail.com>
Date: Thu, 15 Aug 2024 12:42:27 +0300
Subject: [PATCH 003/441] [analyzer] Do not reason about locations passed as
 inline asm input (#103714)

If pointer is passed as input operand for inline assembly, it's possible
that asm block will change memory behind this pointer. So if pointer is
passed inside inline asm block, it's better to not guess and assume
memory has unknown state.

Without such change, we observed a lot of FP with hand-written `memcpy`
and friends.
---
 clang/lib/StaticAnalyzer/Core/ExprEngine.cpp |  8 +++++
 clang/test/Analysis/asm.cpp                  | 32 +++++++++++++++++++-
 2 files changed, 39 insertions(+), 1 deletion(-)

diff --git a/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp b/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
index 686310d38ebd58f..493914f88ba9a50 100644
--- a/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
+++ b/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
@@ -3807,6 +3807,14 @@ void ExprEngine::VisitGCCAsmStmt(const GCCAsmStmt *A, ExplodedNode *Pred,
       state = state->bindLoc(*LV, UnknownVal(), Pred->getLocationContext());
   }
 
+  // Do not reason about locations passed inside inline assembly.
+  for (const Expr *I : A->inputs()) {
+    SVal X = state->getSVal(I, Pred->getLocationContext());
+
+    if (std::optional<Loc> LV = X.getAs<Loc>())
+      state = state->bindLoc(*LV, UnknownVal(), Pred->getLocationContext());
+  }
+
   Bldr.generateNode(A, Pred, state);
 }
 
diff --git a/clang/test/Analysis/asm.cpp b/clang/test/Analysis/asm.cpp
index 1180063502168f8..3181aea870c8aad 100644
--- a/clang/test/Analysis/asm.cpp
+++ b/clang/test/Analysis/asm.cpp
@@ -1,4 +1,5 @@
-// RUN: %clang_analyze_cc1 -analyzer-checker debug.ExprInspection -fheinous-gnu-extensions -w %s -verify
+// RUN: %clang_analyze_cc1 -triple=x86_64-unknown-unknown \
+// RUN:      -analyzer-checker debug.ExprInspection,core -fheinous-gnu-extensions -w %s -verify
 
 int clang_analyzer_eval(int);
 
@@ -10,3 +11,32 @@ void testRValueOutput() {
   clang_analyzer_eval(global == 1); // expected-warning{{UNKNOWN}}
   clang_analyzer_eval(ref == 1);    // expected-warning{{UNKNOWN}}
 }
+
+void *MyMemcpy(void *d, const void *s, const int n) {
+  asm volatile (
+    "cld\n rep movsb\n"
+    :: "S" (s), "D" (d), "c" (n) : "memory"
+  );
+  return d;
+}
+
+void testInlineAsmMemcpy(void)
+{
+    int a, b = 10, c;
+    MyMemcpy(&a, &b, sizeof(b));
+    c = a; // no-warning
+}
+
+void testInlineAsmMemcpyArray(void)
+{
+    int a[10], b[10] = {}, c;
+    MyMemcpy(&a, &b, sizeof(b));
+    c = a[8]; // no-warning
+}
+
+void testInlineAsmMemcpyUninit(void)
+{
+    int a[10], b[10] = {}, c;
+    MyMemcpy(&a[1], &b[1], sizeof(b) - sizeof(b[1]));
+    c = a[0]; // expected-warning{{Assigned value is garbage or undefined}}
+}

From 05f663081513c6293f80469132d083e2603ed036 Mon Sep 17 00:00:00 2001
From: Hans Wennborg <hans@chromium.org>
Date: Thu, 15 Aug 2024 11:37:59 +0200
Subject: [PATCH 004/441] Revert "[Clang] [AST] Fix placeholder return type
 name mangling for MSVC 1920+ / VS2019+ (#102848)"

It cause builds to start failing with

  Invalid type expected
  UNREACHABLE executed at clang/lib/AST/MicrosoftMangle.cpp:2551!

see comments on the PR.

> Partial fix for https://github.com/llvm/llvm-project/issues/92204.
> This PR just fixes VS2019+ since that is the suite of compilers that I
> require link compatibility with at the moment.
> I still intend to fix VS2017 and to update llvm-undname in future PRs.
> Once those are also finished and merged I'll close out
> https://github.com/llvm/llvm-project/issues/92204.
> I am hoping to get the llvm-undname PR up in a couple of weeks to be
> able to demangle the VS2019+ name mangling.
>
> MSVC 1920+ mangles placeholder return types for non-templated functions
> with "@".
> For example `auto foo() { return 0; }` is mangled as `?foo@@YA@XZ`.
>
> MSVC 1920+ mangles placeholder return types for templated functions as
> the qualifiers of the AutoType followed by "_P" for `auto` and "_T" for
> `decltype(auto)`.
> For example `template<class T> auto foo() { return 0; }` is mangled as
> `??$foo@H@@YA?A_PXZ` when `foo` is instantiated as follows `foo<int>()`.
>
> Lambdas with placeholder return types are still mangled with clang's
> custom mangling since MSVC lambda mangling hasn't been deciphered yet.
> Similarly any pointers in the return type with an address space are
> mangled with clang's custom mangling since that is a clang extension.
>
> We cannot augment `mangleType` to support this mangling scheme as the
> mangling schemes for variables and functions differ.
> auto variables are encoded with the fully deduced type where auto return
> types are not.
> The following two functions with a static variable are mangled the same
> ```
> template<class T>
> int test()
> {
>     static int i = 0; // "?i@?1???$test@H@@YAHXZ@4HA"
>     return i;
> }
>
> template<class T>
> int test()
> {
>     static auto i = 0; // "?i@?1???$test@H@@YAHXZ@4HA"
>     return i;
> }
> ```
> Inside `mangleType` once we get to mangling the `AutoType` we have no
> context if we are from a variable encoding or some other encoding.
> Therefore it was easier to handle any special casing for `AutoType`
> return types with a separate function instead of using the `mangleType`
> infrastructure.

This reverts commit e0d173d44161bf9b68243845666d58999e74f759
and the wollow-up fa343be414f9364911b947f109f3df5539e23068.
---
 clang/docs/ReleaseNotes.rst                   |   2 -
 clang/lib/AST/MicrosoftMangle.cpp             | 162 +-------
 .../test/CodeGenCXX/mangle-ms-auto-return.cpp | 369 ------------------
 .../mangle-ms-auto-templates-memptrs.cpp      |  12 +-
 .../mangle-ms-auto-templates-nullptr.cpp      |   2 +-
 .../CodeGenCXX/mangle-ms-auto-templates.cpp   |   6 +-
 6 files changed, 19 insertions(+), 534 deletions(-)
 delete mode 100644 clang/test/CodeGenCXX/mangle-ms-auto-return.cpp

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index b1864901e7bddbc..f5696d6ce15dc7c 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -77,8 +77,6 @@ C++ Specific Potentially Breaking Changes
 ABI Changes in This Version
 ---------------------------
 
-- Fixed Microsoft name mangling of placeholder, auto and decltype(auto), return types for MSVC 1920+. This change resolves incompatibilities with code compiled by MSVC 1920+ but will introduce incompatibilities with code compiled by earlier versions of Clang unless such code is built with the compiler option -fms-compatibility-version=19.14 to imitate the MSVC 1914 mangling behavior.
-
 AST Dumping Potentially Breaking Changes
 ----------------------------------------
 
diff --git a/clang/lib/AST/MicrosoftMangle.cpp b/clang/lib/AST/MicrosoftMangle.cpp
index db8000e25dc7cc6..ed8d1cf1b98dd80 100644
--- a/clang/lib/AST/MicrosoftMangle.cpp
+++ b/clang/lib/AST/MicrosoftMangle.cpp
@@ -408,8 +408,6 @@ class MicrosoftCXXNameMangler {
   void mangleSourceName(StringRef Name);
   void mangleNestedName(GlobalDecl GD);
 
-  void mangleAutoReturnType(QualType T, QualifierMangleMode QMM);
-
 private:
   bool isStructorDecl(const NamedDecl *ND) const {
     return ND == Structor || getStructor(ND) == Structor;
@@ -479,11 +477,6 @@ class MicrosoftCXXNameMangler {
                           SourceRange Range);
   void mangleObjCKindOfType(const ObjCObjectType *T, Qualifiers Quals,
                             SourceRange Range);
-
-  void mangleAutoReturnType(const MemberPointerType *T, Qualifiers Quals);
-  void mangleAutoReturnType(const PointerType *T, Qualifiers Quals);
-  void mangleAutoReturnType(const LValueReferenceType *T, Qualifiers Quals);
-  void mangleAutoReturnType(const RValueReferenceType *T, Qualifiers Quals);
 };
 }
 
@@ -2501,57 +2494,6 @@ void MicrosoftCXXNameMangler::mangleAddressSpaceType(QualType T,
   mangleArtificialTagType(TagTypeKind::Struct, ASMangling, {"__clang"});
 }
 
-void MicrosoftCXXNameMangler::mangleAutoReturnType(QualType T,
-                                                   QualifierMangleMode QMM) {
-  assert(getASTContext().getLangOpts().isCompatibleWithMSVC(
-             LangOptions::MSVC2019) &&
-         "Cannot mangle MSVC 2017 auto return types!");
-
-  if (isa<AutoType>(T)) {
-    const auto *AT = T->getContainedAutoType();
-    Qualifiers Quals = T.getLocalQualifiers();
-
-    if (QMM == QMM_Result)
-      Out << '?';
-    if (QMM != QMM_Drop)
-      mangleQualifiers(Quals, false);
-    Out << (AT->isDecltypeAuto() ? "_T" : "_P");
-    return;
-  }
-
-  T = T.getDesugaredType(getASTContext());
-  Qualifiers Quals = T.getLocalQualifiers();
-
-  switch (QMM) {
-  case QMM_Drop:
-  case QMM_Result:
-    break;
-  case QMM_Mangle:
-    mangleQualifiers(Quals, false);
-    break;
-  default:
-    llvm_unreachable("QMM_Escape unexpected");
-  }
-
-  const Type *ty = T.getTypePtr();
-  switch (ty->getTypeClass()) {
-  case Type::MemberPointer:
-    mangleAutoReturnType(cast<MemberPointerType>(ty), Quals);
-    break;
-  case Type::Pointer:
-    mangleAutoReturnType(cast<PointerType>(ty), Quals);
-    break;
-  case Type::LValueReference:
-    mangleAutoReturnType(cast<LValueReferenceType>(ty), Quals);
-    break;
-  case Type::RValueReference:
-    mangleAutoReturnType(cast<RValueReferenceType>(ty), Quals);
-    break;
-  default:
-    llvm_unreachable("Invalid type expected");
-  }
-}
-
 void MicrosoftCXXNameMangler::mangleType(QualType T, SourceRange Range,
                                          QualifierMangleMode QMM) {
   // Don't use the canonical types.  MSVC includes things like 'const' on
@@ -2965,52 +2907,17 @@ void MicrosoftCXXNameMangler::mangleFunctionType(const FunctionType *T,
       // can differ by their calling convention and are typically deduced.  So
       // we make sure that this type gets mangled properly.
       mangleType(ResultType, Range, QMM_Result);
-    } else if (IsInLambda) {
-      if (const auto *AT = ResultType->getContainedAutoType()) {
-        assert(AT->getKeyword() == AutoTypeKeyword::Auto &&
-               "should only need to mangle auto!");
-        (void)AT;
-        Out << '?';
-        mangleQualifiers(ResultType.getLocalQualifiers(), /*IsMember=*/false);
-        Out << '?';
-        mangleSourceName("<auto>");
-        Out << '@';
-      } else {
-        Out << '@';
-      }
-    } else if (const auto *AT = ResultType->getContainedAutoType()) {
+    } else if (const auto *AT = dyn_cast_or_null<AutoType>(
+                   ResultType->getContainedAutoType())) {
+      Out << '?';
+      mangleQualifiers(ResultType.getLocalQualifiers(), /*IsMember=*/false);
+      Out << '?';
       assert(AT->getKeyword() != AutoTypeKeyword::GNUAutoType &&
              "shouldn't need to mangle __auto_type!");
-
-      // If we have any pointer types with the clang address space extension
-      // then defer to the custom clang mangling to keep backwards
-      // compatibility. See `mangleType(const PointerType *T, Qualifiers Quals,
-      // SourceRange Range)` for details.
-      auto UseClangMangling = [](QualType ResultType) {
-        QualType T = ResultType;
-        while (isa<PointerType>(T.getTypePtr())) {
-          T = T->getPointeeType();
-          if (T.getQualifiers().hasAddressSpace())
-            return true;
-        }
-        return false;
-      };
-
-      if (getASTContext().getLangOpts().isCompatibleWithMSVC(
-              LangOptions::MSVC2019) &&
-          !UseClangMangling(ResultType)) {
-        if (D && !D->getPrimaryTemplate()) {
-          Out << '@';
-        } else {
-          mangleAutoReturnType(ResultType, QMM_Result);
-        }
-      } else {
-        Out << '?';
-        mangleQualifiers(ResultType.getLocalQualifiers(), /*IsMember=*/false);
-        Out << '?';
-        mangleSourceName(AT->isDecltypeAuto() ? "<decltype-auto>" : "<auto>");
-        Out << '@';
-      }
+      mangleSourceName(AT->isDecltypeAuto() ? "<decltype-auto>" : "<auto>");
+      Out << '@';
+    } else if (IsInLambda) {
+      Out << '@';
     } else {
       if (ResultType->isVoidType())
         ResultType = ResultType.getUnqualifiedType();
@@ -4313,57 +4220,6 @@ void MicrosoftMangleContextImpl::mangleStringLiteral(const StringLiteral *SL,
   Mangler.getStream() << '@';
 }
 
-void MicrosoftCXXNameMangler::mangleAutoReturnType(const MemberPointerType *T,
-                                                   Qualifiers Quals) {
-  QualType PointeeType = T->getPointeeType();
-  manglePointerCVQualifiers(Quals);
-  manglePointerExtQualifiers(Quals, PointeeType);
-  if (const FunctionProtoType *FPT = PointeeType->getAs<FunctionProtoType>()) {
-    Out << '8';
-    mangleName(T->getClass()->castAs<RecordType>()->getDecl());
-    mangleFunctionType(FPT, nullptr, true);
-  } else {
-    mangleQualifiers(PointeeType.getQualifiers(), true);
-    mangleName(T->getClass()->castAs<RecordType>()->getDecl());
-    mangleAutoReturnType(PointeeType, QMM_Drop);
-  }
-}
-
-void MicrosoftCXXNameMangler::mangleAutoReturnType(const PointerType *T,
-                                                   Qualifiers Quals) {
-  QualType PointeeType = T->getPointeeType();
-  assert(!PointeeType.getQualifiers().hasAddressSpace() &&
-         "Unexpected address space mangling required");
-
-  manglePointerCVQualifiers(Quals);
-  manglePointerExtQualifiers(Quals, PointeeType);
-
-  if (const FunctionProtoType *FPT = PointeeType->getAs<FunctionProtoType>()) {
-    Out << '6';
-    mangleFunctionType(FPT);
-  } else {
-    mangleAutoReturnType(PointeeType, QMM_Mangle);
-  }
-}
-
-void MicrosoftCXXNameMangler::mangleAutoReturnType(const LValueReferenceType *T,
-                                                   Qualifiers Quals) {
-  QualType PointeeType = T->getPointeeType();
-  assert(!Quals.hasConst() && !Quals.hasVolatile() && "unexpected qualifier!");
-  Out << 'A';
-  manglePointerExtQualifiers(Quals, PointeeType);
-  mangleAutoReturnType(PointeeType, QMM_Mangle);
-}
-
-void MicrosoftCXXNameMangler::mangleAutoReturnType(const RValueReferenceType *T,
-                                                   Qualifiers Quals) {
-  QualType PointeeType = T->getPointeeType();
-  assert(!Quals.hasConst() && !Quals.hasVolatile() && "unexpected qualifier!");
-  Out << "$$Q";
-  manglePointerExtQualifiers(Quals, PointeeType);
-  mangleAutoReturnType(PointeeType, QMM_Mangle);
-}
-
 MicrosoftMangleContext *MicrosoftMangleContext::create(ASTContext &Context,
                                                        DiagnosticsEngine &Diags,
                                                        bool IsAux) {
diff --git a/clang/test/CodeGenCXX/mangle-ms-auto-return.cpp b/clang/test/CodeGenCXX/mangle-ms-auto-return.cpp
deleted file mode 100644
index 737c9c407f47037..000000000000000
--- a/clang/test/CodeGenCXX/mangle-ms-auto-return.cpp
+++ /dev/null
@@ -1,369 +0,0 @@
-// RUN: %clang_cc1 -std=c++17 -fms-compatibility-version=19.20 -emit-llvm %s -o - -fms-extensions -fdelayed-template-parsing -triple=x86_64-pc-windows-msvc | FileCheck %s
-
-struct StructA {};
-
-template<class T>
-auto AutoT() { return T(); }
-
-template<class T>
-const auto AutoConstT() { return T(); }
-
-template<class T>
-volatile auto AutoVolatileT() { return T(); }
-
-template<class T>
-const volatile auto AutoConstVolatileT() { return T(); }
-
-// The qualifiers of the return type should always be emitted even for void types.
-// Void types usually have their qualifers stripped in the mangled name for MSVC ABI.
-void test_template_auto_void() {
-  AutoT<void>();
-  // CHECK: call {{.*}} @"??$AutoT@X@@YA?A_PXZ"
-
-  AutoT<const void>();
-  // CHECK: call {{.*}} @"??$AutoT@$$CBX@@YA?A_PXZ"
-
-  AutoT<volatile void>();
-  // CHECK: call {{.*}} @"??$AutoT@$$CCX@@YA?A_PXZ"
-
-  AutoT<const volatile void>();
-  // CHECK: call {{.*}} @"??$AutoT@$$CDX@@YA?A_PXZ"
-
-  AutoConstT<void>();
-  // CHECK: call {{.*}} @"??$AutoConstT@X@@YA?B_PXZ"
-
-  AutoVolatileT<void>();
-  // CHECK: call {{.*}} @"??$AutoVolatileT@X@@YA?C_PXZ"
-
-  AutoConstVolatileT<void>();
-  // CHECK: call {{.*}} @"??$AutoConstVolatileT@X@@YA?D_PXZ"
-}
-
-void test_template_auto_int() {
-  AutoT<int>();
-  // CHECK: call {{.*}} @"??$AutoT@H@@YA?A_PXZ"
-
-  AutoT<const int>();
-  // CHECK: call {{.*}} @"??$AutoT@$$CBH@@YA?A_PXZ"
-
-  AutoT<volatile int>();
-  // CHECK: call {{.*}} @"??$AutoT@$$CCH@@YA?A_PXZ"
-
-  AutoT<const volatile int>();
-  // CHECK: call {{.*}} @"??$AutoT@$$CDH@@YA?A_PXZ"
-
-  AutoConstT<int>();
-  // CHECK: call {{.*}} @"??$AutoConstT@H@@YA?B_PXZ"
-
-  AutoVolatileT<int>();
-  // CHECK: call {{.*}} @"??$AutoVolatileT@H@@YA?C_PXZ"
-
-  AutoConstVolatileT<int>();
-  // CHECK: call {{.*}} @"??$AutoConstVolatileT@H@@YA?D_PXZ"
-}
-
-void test_template_auto_struct() {
-  AutoT<StructA>();
-  // CHECK: call {{.*}} @"??$AutoT@UStructA@@@@YA?A_PXZ"
-
-  AutoT<const StructA>();
-  // CHECK: call {{.*}} @"??$AutoT@$$CBUStructA@@@@YA?A_PXZ"
-
-  AutoConstT<StructA>();
-  // CHECK: call {{.*}} @"??$AutoConstT@UStructA@@@@YA?B_PXZ"
-
-  AutoVolatileT<StructA>();
-  // CHECK: call {{.*}} @"??$AutoVolatileT@UStructA@@@@YA?C_PXZ"
-
-  AutoConstVolatileT<StructA>();
-  // CHECK: call {{.*}} @"??$AutoConstVolatileT@UStructA@@@@YA?D_PXZ"
-}
-
-void test_template_auto_ptr() {
-  AutoT<int*>();
-  // CHECK: call {{.*}} @"??$AutoT@PEAH@@YA?A_PXZ"
-
-  AutoT<const int*>();
-  // CHECK: call {{.*}} @"??$AutoT@PEBH@@YA?A_PXZ"
-
-  AutoT<const int* const>();
-  // CHECK: call {{.*}} @"??$AutoT@QEBH@@YA?A_PXZ"
-
-  AutoConstT<int*>();
-  // CHECK: call {{.*}} @"??$AutoConstT@PEAH@@YA?B_PXZ"
-
-  AutoVolatileT<int*>();
-  // CHECK: call {{.*}} @"??$AutoVolatileT@PEAH@@YA?C_PXZ"
-
-  AutoConstVolatileT<int*>();
-  // CHECK: call {{.*}} @"??$AutoConstVolatileT@PEAH@@YA?D_PXZ"
-}
-
-template<class T>
-auto* PtrAutoT() { return T(); }
-
-template<class T>
-const auto* PtrAutoConstT() { return T(); }
-
-template<class T>
-volatile auto* PtrAutoVolatileT() { return T(); }
-
-template<class T>
-const volatile auto* PtrAutoConstVolatileT() { return T(); }
-
-void test_template_ptr_auto() {
-  PtrAutoT<int*>();
-  // CHECK: call {{.*}} @"??$PtrAutoT@PEAH@@YAPEA_PXZ"
-
-  PtrAutoT<const int*>();
-  // CHECK: call {{.*}} @"??$PtrAutoT@PEBH@@YAPEA_PXZ"
-
-  PtrAutoT<const int* const>();
-  // CHECK: call {{.*}} @"??$PtrAutoT@QEBH@@YAPEA_PXZ"
-
-  PtrAutoConstT<int*>();
-  // CHECK: call {{.*}} @"??$PtrAutoConstT@PEAH@@YAPEB_PXZ"
-
-  PtrAutoVolatileT<int*>();
-  // CHECK: call {{.*}} @"??$PtrAutoVolatileT@PEAH@@YAPEC_PXZ"
-
-  PtrAutoConstVolatileT<int*>();
-  // CHECK: call {{.*}} @"??$PtrAutoConstVolatileT@PEAH@@YAPED_PXZ"
-}
-
-int func_int();
-const int func_constint();
-void func_void();
-int* func_intptr();
-
-template<class T, T v>
-auto (*FuncPtrAutoT())() { return v; }
-
-void test_template_func_ptr_auto() {
-  FuncPtrAutoT<int (*)(), &func_int>();
-  // CHECK: call {{.*}} @"??$FuncPtrAutoT@P6AHXZ$1?func_int@@YAHXZ@@YAP6A?A_PXZXZ"
-
-  FuncPtrAutoT<const int (*)(), &func_constint>();
-  // CHECK: call {{.*}} @"??$FuncPtrAutoT@P6A?BHXZ$1?func_constint@@YA?BHXZ@@YAP6A?A_PXZXZ"
-
-  FuncPtrAutoT<void (*)(), &func_void>();
-  // CHECK: call {{.*}} @"??$FuncPtrAutoT@P6AXXZ$1?func_void@@YAXXZ@@YAP6A?A_PXZXZ"
-
-  FuncPtrAutoT<int * (*)(), &func_intptr>();
-  // CHECK: call {{.*}} @"??$FuncPtrAutoT@P6APEAHXZ$1?func_intptr@@YAPEAHXZ@@YAP6A?A_PXZXZ"
-}
-
-template<class T>
-auto& RefAutoT(T& x) { return x; }
-
-template<class T>
-const auto& ConstRefAutoT(T& x) { return x; }
-
-template<class T>
-auto&& RRefAutoT(T& x) { return static_cast<int&&>(x); }
-
-void test_template_ref_auto() {
-  int x;
-
-  RefAutoT(x);
-  // CHECK: call {{.*}} @"??$RefAutoT@H@@YAAEA_PAEAH@Z"
-
-  ConstRefAutoT(x);
-  // CHECK: call {{.*}} @"??$ConstRefAutoT@H@@YAAEB_PAEAH@Z"
-
-  RRefAutoT(x);
-  // CHECK: call {{.*}} @"??$RRefAutoT@H@@YA$$QEA_PAEAH@Z"
-}
-
-template<class T>
-decltype(auto) DecltypeAutoT() { return T(); }
-
-template<class T>
-decltype(auto) DecltypeAutoT2(T& x) { return static_cast<T&&>(x); }
-
-void test_template_decltypeauto() {
-  DecltypeAutoT<void>();
-  // CHECK: call {{.*}} @"??$DecltypeAutoT@X@@YA?A_TXZ"
-
-  DecltypeAutoT<const void>();
-  // CHECK: call {{.*}} @"??$DecltypeAutoT@$$CBX@@YA?A_TXZ"
-
-  DecltypeAutoT<volatile void>();
-  // CHECK: call {{.*}} @"??$DecltypeAutoT@$$CCX@@YA?A_TXZ"
-
-  DecltypeAutoT<const volatile void>();
-  // CHECK: call {{.*}} @"??$DecltypeAutoT@$$CDX@@YA?A_TXZ"
-
-  DecltypeAutoT<int>();
-  // CHECK: call {{.*}} @"??$DecltypeAutoT@H@@YA?A_TXZ"
-
-  DecltypeAutoT<const int>();
-  // CHECK: call {{.*}} @"??$DecltypeAutoT@$$CBH@@YA?A_TXZ"
-
-  DecltypeAutoT<volatile int>();
-  // CHECK: call {{.*}} @"??$DecltypeAutoT@$$CCH@@YA?A_TXZ"
-
-  DecltypeAutoT<const volatile int>();
-  // CHECK: call {{.*}} @"??$DecltypeAutoT@$$CDH@@YA?A_TXZ"
-
-  int x;
-
-  DecltypeAutoT2(x);
-  // CHECK: call {{.*}} @"??$DecltypeAutoT2@H@@YA?A_TAEAH@Z"
-}
-
-// Still want to use clang's custom mangling for lambdas to keep backwards compatibility until
-// MSVC lambda name mangling has been deciphered.
-void test_lambda() {
-  auto lambdaIntRetAuto = []() { return 0; };
-  lambdaIntRetAuto();
-  // CHECK: call {{.*}} @"??R<lambda_1>@?0??test_lambda@@YAXXZ@QEBA?A?<auto>@@XZ"
-
-  auto lambdaIntRet = []() -> int { return 0; };
-  lambdaIntRet();
-  // CHECK: call {{.*}} @"??R<lambda_2>@?0??test_lambda@@YAXXZ@QEBA@XZ"
-
-  auto lambdaGenericIntIntRetAuto = [](auto a) { return a; };
-  lambdaGenericIntIntRetAuto(0);
-  // CHECK: call {{.*}} @"??$?RH@<lambda_0>@?0??test_lambda@@YAXXZ@QEBA?A?<auto>@@H@Z"
-}
-
-auto TestTrailingInt() -> int {
-  return 0;
-}
-
-auto TestTrailingConstVolatileVoid() -> const volatile void {
-}
-
-auto TestTrailingStructA() -> StructA {
-  return StructA{};
-}
-
-void test_trailing_return() {
-  TestTrailingInt();
-  // CHECK: call {{.*}} @"?TestTrailingInt@@YAHXZ"
-
-  TestTrailingConstVolatileVoid();
-  // CHECK: call {{.*}} @"?TestTrailingConstVolatileVoid@@YAXXZ"
-
-  TestTrailingStructA();
-  // CHECK: call {{.*}} @"?TestTrailingStructA@@YA?AUStructA@@XZ"
-}
-
-auto TestNonTemplateAutoInt() {
-  return 0;
-}
-
-auto TestNonTemplateAutoVoid() {
-  return;
-}
-
-auto TestNonTemplateAutoStructA() {
-  return StructA{};
-}
-
-const auto TestNonTemplateConstAutoInt() {
-  return 0;
-}
-
-const auto TestNonTemplateConstAutoVoid() {
-  return;
-}
-
-const auto TestNonTemplateConstAutoStructA() {
-  return StructA{};
-}
-
-void test_nontemplate_auto() {
-  TestNonTemplateAutoInt();
-  // CHECK: call {{.*}} @"?TestNonTemplateAutoInt@@YA@XZ"
-
-  TestNonTemplateAutoVoid();
-  // CHECK: call {{.*}} @"?TestNonTemplateAutoVoid@@YA@XZ"
-
-  TestNonTemplateAutoStructA();
-  // CHECK: call {{.*}} @"?TestNonTemplateAutoStructA@@YA@XZ"
-
-  TestNonTemplateConstAutoInt();
-  // CHECK: call {{.*}} @"?TestNonTemplateConstAutoInt@@YA@XZ"
-
-  TestNonTemplateConstAutoVoid();
-  // CHECK: call {{.*}} @"?TestNonTemplateConstAutoVoid@@YA@XZ"
-
-  TestNonTemplateConstAutoStructA();
-  // CHECK: call {{.*}} @"?TestNonTemplateConstAutoStructA@@YA@XZ"
-}
-
-decltype(auto) TestNonTemplateDecltypeAutoInt() {
-    return 0;
-}
-
-decltype(auto) TestNonTemplateDecltypeAutoVoid() {
-    return;
-}
-
-decltype(auto) TestNonTemplateDecltypeAutoStructA() {
-    return StructA{};
-}
-
-void test_nontemplate_decltypeauto() {
-  TestNonTemplateDecltypeAutoInt();
-  // CHECK: call {{.*}} @"?TestNonTemplateDecltypeAutoInt@@YA@XZ"
-
-  TestNonTemplateDecltypeAutoVoid();
-  // CHECK: call {{.*}} @"?TestNonTemplateDecltypeAutoVoid@@YA@XZ"
-
-  TestNonTemplateDecltypeAutoStructA();
-  // CHECK: call {{.*}} @"?TestNonTemplateDecltypeAutoStructA@@YA@XZ"
-}
-
-struct StructB {
-  int x;
-};
-
-template<class T>
-auto StructB::* AutoMemberDataPtrT(T x) { return x; }
-
-template<class T>
-const auto StructB::* AutoConstMemberDataPtrT(T x) { return x; }
-
-void test_template_auto_member_data_ptr() {
-  AutoMemberDataPtrT(&StructB::x);
-  // CHECK: call {{.*}} @"??$AutoMemberDataPtrT@PEQStructB@@H@@YAPEQStructB@@_PPEQ0@H@Z"
-
-  AutoConstMemberDataPtrT(&StructB::x);
-  // CHECK: call {{.*}} @"??$AutoConstMemberDataPtrT@PEQStructB@@H@@YAPERStructB@@_PPEQ0@H@Z"
-}
-
-struct StructC {
-  void test() {}
-};
-
-struct StructD {
-  const int test() { return 0; }
-};
-
-template<class T>
-auto (StructC::*AutoMemberFuncPtrT(T x))() { return x; }
-
-template<class T>
-const auto (StructD::*AutoConstMemberFuncPtrT(T x))() { return x; }
-
-void test_template_auto_member_func_ptr() {
-  AutoMemberFuncPtrT(&StructC::test);
-  // CHECK: call {{.*}} @"??$AutoMemberFuncPtrT@P8StructC@@EAAXXZ@@YAP8StructC@@EAA?A_PXZP80@EAAXXZ@Z"
-
-  AutoConstMemberFuncPtrT(&StructD::test);
-  // CHECK: call {{.*}} @"??$AutoConstMemberFuncPtrT@P8StructD@@EAA?BHXZ@@YAP8StructD@@EAA?B_PXZP80@EAA?BHXZ@Z"
-}
-
-template<class T>
-auto * __attribute__((address_space(1))) * AutoPtrAddressSpaceT() {
-  T * __attribute__((address_space(1))) * p = nullptr;
-  return p;
-}
-
-void test_template_auto_address_space_ptr() {
-  AutoPtrAddressSpaceT<int>();
-  // CHECK: call {{.*}} @"??$AutoPtrAddressSpaceT@H@@YA?A?<auto>@@XZ"
-}
diff --git a/clang/test/CodeGenCXX/mangle-ms-auto-templates-memptrs.cpp b/clang/test/CodeGenCXX/mangle-ms-auto-templates-memptrs.cpp
index b7bc3953f0b4380..360ebdecc5562be 100644
--- a/clang/test/CodeGenCXX/mangle-ms-auto-templates-memptrs.cpp
+++ b/clang/test/CodeGenCXX/mangle-ms-auto-templates-memptrs.cpp
@@ -34,15 +34,15 @@ void template_mangling() {
   // BEFORE: call {{.*}} @"??0?$AutoParmTemplate@$I?f@V@@QEAAXXZA@A@@@QEAA@XZ"
 
   AutoFunc<&S::f>();
-  // AFTER: call {{.*}} @"??$AutoFunc@$MP8S@@EAAXXZ1?f@1@QEAAXXZ@@YA?A_PXZ"
+  // AFTER: call {{.*}} @"??$AutoFunc@$MP8S@@EAAXXZ1?f@1@QEAAXXZ@@YA?A?<auto>@@XZ"
   // BEFORE: call {{.*}} @"??$AutoFunc@$1?f@S@@QEAAXXZ@@YA?A?<auto>@@XZ"
 
   AutoFunc<&M::f>();
-  // AFTER: call {{.*}} @"??$AutoFunc@$MP8M@@EAAXXZH?f@1@QEAAXXZA@@@YA?A_PXZ"
+  // AFTER: call {{.*}} @"??$AutoFunc@$MP8M@@EAAXXZH?f@1@QEAAXXZA@@@YA?A?<auto>@@XZ"
   // BEFORE: call {{.*}} @"??$AutoFunc@$H?f@M@@QEAAXXZA@@@YA?A?<auto>@@XZ"
 
   AutoFunc<&V::f>();
-  // AFTER: call {{.*}} @"??$AutoFunc@$MP8V@@EAAXXZI?f@1@QEAAXXZA@A@@@YA?A_PXZ"
+  // AFTER: call {{.*}} @"??$AutoFunc@$MP8V@@EAAXXZI?f@1@QEAAXXZA@A@@@YA?A?<auto>@@XZ"
   // BEFORE: call {{.*}} @"??$AutoFunc@$I?f@V@@QEAAXXZA@A@@@YA?A?<auto>@@XZ"
 
   AutoParmTemplate<&S::a> auto_data_single_inheritance;
@@ -58,14 +58,14 @@ void template_mangling() {
   // BEFORE: call {{.*}} @"??0?$AutoParmTemplate@$FBA@A@@@QEAA@XZ"
 
   AutoFunc<&S::a>();
-  // AFTER: call {{.*}} @"??$AutoFunc@$MPEQS@@H07@@YA?A_PXZ"
+  // AFTER: call {{.*}} @"??$AutoFunc@$MPEQS@@H07@@YA?A?<auto>@@XZ"
   // BEFORE: call {{.*}} @"??$AutoFunc@$07@@YA?A?<auto>@@XZ"
 
   AutoFunc<&M::a>();
-  // AFTER: call {{.*}} @"??$AutoFunc@$MPEQM@@H0M@@@YA?A_PXZ"
+  // AFTER: call {{.*}} @"??$AutoFunc@$MPEQM@@H0M@@@YA?A?<auto>@@XZ"
   // BEFORE: call {{.*}} @"??$AutoFunc@$0M@@@YA?A?<auto>@@XZ"
 
   AutoFunc<&V::a>();
-  // AFTER: call {{.*}} @"??$AutoFunc@$MPEQV@@HFBA@A@@@YA?A_PXZ"
+  // AFTER: call {{.*}} @"??$AutoFunc@$MPEQV@@HFBA@A@@@YA?A?<auto>@@XZ"
   // BEFORE: call {{.*}} @"??$AutoFunc@$FBA@A@@@YA?A?<auto>@@XZ"
 }
diff --git a/clang/test/CodeGenCXX/mangle-ms-auto-templates-nullptr.cpp b/clang/test/CodeGenCXX/mangle-ms-auto-templates-nullptr.cpp
index 251d9219c01ce28..8f98c1e59f73d79 100644
--- a/clang/test/CodeGenCXX/mangle-ms-auto-templates-nullptr.cpp
+++ b/clang/test/CodeGenCXX/mangle-ms-auto-templates-nullptr.cpp
@@ -19,6 +19,6 @@ void template_mangling() {
   // BEFORE: call {{.*}} @"??0?$AutoParmTemplate@$0A@@@QEAA@XZ"
 
   AutoFunc<nullptr>();
-  // AFTER: call {{.*}} @"??$AutoFunc@$M$$T0A@@@YA?A_PXZ"
+  // AFTER: call {{.*}} @"??$AutoFunc@$M$$T0A@@@YA?A?<auto>@@XZ"
   // BEFORE: call {{.*}} @"??$AutoFunc@$0A@@@YA?A?<auto>@@XZ"
 }
diff --git a/clang/test/CodeGenCXX/mangle-ms-auto-templates.cpp b/clang/test/CodeGenCXX/mangle-ms-auto-templates.cpp
index effcc31ee311038..ff5395cea75eb77 100644
--- a/clang/test/CodeGenCXX/mangle-ms-auto-templates.cpp
+++ b/clang/test/CodeGenCXX/mangle-ms-auto-templates.cpp
@@ -26,7 +26,7 @@ int j;
 
 void template_mangling() {
   AutoFunc<1>();
-  // AFTER: call {{.*}} @"??$AutoFunc@$MH00@@YA?A_PXZ"
+  // AFTER: call {{.*}} @"??$AutoFunc@$MH00@@YA?A?<auto>@@XZ"
   // BEFORE: call {{.*}} @"??$AutoFunc@$00@@YA?A?<auto>@@XZ"
   AutoParmTemplate<0> auto_int;
   // AFTER: call {{.*}} @"??0?$AutoParmTemplate@$MH0A@@@QEAA@XZ"
@@ -52,7 +52,7 @@ void template_mangling() {
   // BEFORE: call {{.*}} @"??0?$AutoParmsTemplate@$00$0HPPPPPPPPPPPPPPP@@@QEAA@XZ"
 
   AutoFunc<&i>();
-  // AFTER: call {{.*}} @"??$AutoFunc@$MPEAH1?i@@3HA@@YA?A_PXZ"
+  // AFTER: call {{.*}} @"??$AutoFunc@$MPEAH1?i@@3HA@@YA?A?<auto>@@XZ"
   // BEFORE: call {{.*}} @"??$AutoFunc@$1?i@@3HA@@YA?A?<auto>@@XZ"
 
   AutoParmTemplate<&i> auto_int_ptr;
@@ -64,7 +64,7 @@ void template_mangling() {
   // BEFORE: call {{.*}} @"??0?$AutoParmsTemplate@$1?i@@3HA$1?j@@3HA@@QEAA@XZ"
 
   AutoFunc<&Func>();
-  // AFTER: call {{.*}} @"??$AutoFunc@$MP6AHXZ1?Func@@YAHXZ@@YA?A_PXZ"
+  // AFTER: call {{.*}} @"??$AutoFunc@$MP6AHXZ1?Func@@YAHXZ@@YA?A?<auto>@@XZ"
   // BEFORE: call {{.*}} @"??$AutoFunc@$1?Func@@YAHXZ@@YA?A?<auto>@@XZ"
 
   AutoParmTemplate<&Func> auto_func_ptr;

From 36231a5b5525b950daf1b7430859061b31a8e01e Mon Sep 17 00:00:00 2001
From: David Green <david.green@arm.com>
Date: Thu, 15 Aug 2024 11:20:20 +0100
Subject: [PATCH 005/441] [AArch64] Add verification for MemOp immediate ranges
 (#97561)

This adds an implementation of AArch64InstrInfo::verifyInstruction for
AArch64, and adds some basic verification of the range of immediate
ranges of memory operations using the information from getMemOpInfo.

Some extra memory operations have been added to getMemOpInfo, along with
the equivalent opcodes to getLoadStoreImmIdx to ensure we use the
correct index.

Please let us know if this starts reporting verification failures, Thanks.
---
 llvm/lib/Target/AArch64/AArch64InstrInfo.cpp  | 394 +++++++++++++-----
 llvm/lib/Target/AArch64/AArch64InstrInfo.h    |   4 +
 .../machine-outliner-unsafe-stack-call.mir    |  16 +-
 llvm/test/CodeGen/AArch64/verify-memop.mir    |  51 +++
 4 files changed, 359 insertions(+), 106 deletions(-)
 create mode 100644 llvm/test/CodeGen/AArch64/verify-memop.mir

diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index e425984a6778443..5f8ed9cb6180a2c 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -2344,132 +2344,258 @@ std::optional<unsigned> AArch64InstrInfo::getUnscaledLdSt(unsigned Opc) {
 unsigned AArch64InstrInfo::getLoadStoreImmIdx(unsigned Opc) {
   switch (Opc) {
   default:
+    llvm_unreachable("Unhandled Opcode in getLoadStoreImmIdx");
+  case AArch64::ADDG:
+  case AArch64::LDAPURBi:
+  case AArch64::LDAPURHi:
+  case AArch64::LDAPURi:
+  case AArch64::LDAPURSBWi:
+  case AArch64::LDAPURSBXi:
+  case AArch64::LDAPURSHWi:
+  case AArch64::LDAPURSHXi:
+  case AArch64::LDAPURSWi:
+  case AArch64::LDAPURXi:
+  case AArch64::LDR_PPXI:
+  case AArch64::LDR_PXI:
+  case AArch64::LDR_ZXI:
+  case AArch64::LDR_ZZXI:
+  case AArch64::LDR_ZZZXI:
+  case AArch64::LDR_ZZZZXI:
+  case AArch64::LDRBBui:
+  case AArch64::LDRBui:
+  case AArch64::LDRDui:
+  case AArch64::LDRHHui:
+  case AArch64::LDRHui:
+  case AArch64::LDRQui:
+  case AArch64::LDRSBWui:
+  case AArch64::LDRSBXui:
+  case AArch64::LDRSHWui:
+  case AArch64::LDRSHXui:
+  case AArch64::LDRSui:
+  case AArch64::LDRSWui:
+  case AArch64::LDRWui:
+  case AArch64::LDRXui:
+  case AArch64::LDURBBi:
+  case AArch64::LDURBi:
+  case AArch64::LDURDi:
+  case AArch64::LDURHHi:
+  case AArch64::LDURHi:
+  case AArch64::LDURQi:
+  case AArch64::LDURSBWi:
+  case AArch64::LDURSBXi:
+  case AArch64::LDURSHWi:
+  case AArch64::LDURSHXi:
+  case AArch64::LDURSi:
+  case AArch64::LDURSWi:
+  case AArch64::LDURWi:
+  case AArch64::LDURXi:
+  case AArch64::PRFMui:
+  case AArch64::PRFUMi:
+  case AArch64::ST2Gi:
+  case AArch64::STGi:
+  case AArch64::STLURBi:
+  case AArch64::STLURHi:
+  case AArch64::STLURWi:
+  case AArch64::STLURXi:
+  case AArch64::StoreSwiftAsyncContext:
+  case AArch64::STR_PPXI:
+  case AArch64::STR_PXI:
+  case AArch64::STR_ZXI:
+  case AArch64::STR_ZZXI:
+  case AArch64::STR_ZZZXI:
+  case AArch64::STR_ZZZZXI:
+  case AArch64::STRBBui:
+  case AArch64::STRBui:
+  case AArch64::STRDui:
+  case AArch64::STRHHui:
+  case AArch64::STRHui:
+  case AArch64::STRQui:
+  case AArch64::STRSui:
+  case AArch64::STRWui:
+  case AArch64::STRXui:
+  case AArch64::STURBBi:
+  case AArch64::STURBi:
+  case AArch64::STURDi:
+  case AArch64::STURHHi:
+  case AArch64::STURHi:
+  case AArch64::STURQi:
+  case AArch64::STURSi:
+  case AArch64::STURWi:
+  case AArch64::STURXi:
+  case AArch64::STZ2Gi:
+  case AArch64::STZGi:
+  case AArch64::TAGPstack:
     return 2;
-  case AArch64::LDPXi:
-  case AArch64::LDPDi:
-  case AArch64::STPXi:
-  case AArch64::STPDi:
-  case AArch64::LDNPXi:
-  case AArch64::LDNPDi:
-  case AArch64::STNPXi:
-  case AArch64::STNPDi:
-  case AArch64::LDPQi:
-  case AArch64::STPQi:
-  case AArch64::LDNPQi:
-  case AArch64::STNPQi:
-  case AArch64::LDPWi:
-  case AArch64::LDPSi:
-  case AArch64::STPWi:
-  case AArch64::STPSi:
-  case AArch64::LDNPWi:
-  case AArch64::LDNPSi:
-  case AArch64::STNPWi:
-  case AArch64::STNPSi:
-  case AArch64::LDG:
-  case AArch64::STGPi:
-
-  case AArch64::LD1B_IMM:
+  case AArch64::LD1B_D_IMM:
   case AArch64::LD1B_H_IMM:
+  case AArch64::LD1B_IMM:
   case AArch64::LD1B_S_IMM:
-  case AArch64::LD1B_D_IMM:
-  case AArch64::LD1SB_H_IMM:
-  case AArch64::LD1SB_S_IMM:
-  case AArch64::LD1SB_D_IMM:
+  case AArch64::LD1D_IMM:
+  case AArch64::LD1H_D_IMM:
   case AArch64::LD1H_IMM:
   case AArch64::LD1H_S_IMM:
-  case AArch64::LD1H_D_IMM:
-  case AArch64::LD1SH_S_IMM:
+  case AArch64::LD1RB_D_IMM:
+  case AArch64::LD1RB_H_IMM:
+  case AArch64::LD1RB_IMM:
+  case AArch64::LD1RB_S_IMM:
+  case AArch64::LD1RD_IMM:
+  case AArch64::LD1RH_D_IMM:
+  case AArch64::LD1RH_IMM:
+  case AArch64::LD1RH_S_IMM:
+  case AArch64::LD1RSB_D_IMM:
+  case AArch64::LD1RSB_H_IMM:
+  case AArch64::LD1RSB_S_IMM:
+  case AArch64::LD1RSH_D_IMM:
+  case AArch64::LD1RSH_S_IMM:
+  case AArch64::LD1RSW_IMM:
+  case AArch64::LD1RW_D_IMM:
+  case AArch64::LD1RW_IMM:
+  case AArch64::LD1SB_D_IMM:
+  case AArch64::LD1SB_H_IMM:
+  case AArch64::LD1SB_S_IMM:
   case AArch64::LD1SH_D_IMM:
-  case AArch64::LD1W_IMM:
-  case AArch64::LD1W_D_IMM:
+  case AArch64::LD1SH_S_IMM:
   case AArch64::LD1SW_D_IMM:
-  case AArch64::LD1D_IMM:
-
+  case AArch64::LD1W_D_IMM:
+  case AArch64::LD1W_IMM:
   case AArch64::LD2B_IMM:
+  case AArch64::LD2D_IMM:
   case AArch64::LD2H_IMM:
   case AArch64::LD2W_IMM:
-  case AArch64::LD2D_IMM:
   case AArch64::LD3B_IMM:
+  case AArch64::LD3D_IMM:
   case AArch64::LD3H_IMM:
   case AArch64::LD3W_IMM:
-  case AArch64::LD3D_IMM:
   case AArch64::LD4B_IMM:
+  case AArch64::LD4D_IMM:
   case AArch64::LD4H_IMM:
   case AArch64::LD4W_IMM:
-  case AArch64::LD4D_IMM:
-
-  case AArch64::ST1B_IMM:
+  case AArch64::LDG:
+  case AArch64::LDNF1B_D_IMM:
+  case AArch64::LDNF1B_H_IMM:
+  case AArch64::LDNF1B_IMM:
+  case AArch64::LDNF1B_S_IMM:
+  case AArch64::LDNF1D_IMM:
+  case AArch64::LDNF1H_D_IMM:
+  case AArch64::LDNF1H_IMM:
+  case AArch64::LDNF1H_S_IMM:
+  case AArch64::LDNF1SB_D_IMM:
+  case AArch64::LDNF1SB_H_IMM:
+  case AArch64::LDNF1SB_S_IMM:
+  case AArch64::LDNF1SH_D_IMM:
+  case AArch64::LDNF1SH_S_IMM:
+  case AArch64::LDNF1SW_D_IMM:
+  case AArch64::LDNF1W_D_IMM:
+  case AArch64::LDNF1W_IMM:
+  case AArch64::LDNPDi:
+  case AArch64::LDNPQi:
+  case AArch64::LDNPSi:
+  case AArch64::LDNPWi:
+  case AArch64::LDNPXi:
+  case AArch64::LDNT1B_ZRI:
+  case AArch64::LDNT1D_ZRI:
+  case AArch64::LDNT1H_ZRI:
+  case AArch64::LDNT1W_ZRI:
+  case AArch64::LDPDi:
+  case AArch64::LDPQi:
+  case AArch64::LDPSi:
+  case AArch64::LDPWi:
+  case AArch64::LDPXi:
+  case AArch64::LDRBBpost:
+  case AArch64::LDRBBpre:
+  case AArch64::LDRBpost:
+  case AArch64::LDRBpre:
+  case AArch64::LDRDpost:
+  case AArch64::LDRDpre:
+  case AArch64::LDRHHpost:
+  case AArch64::LDRHHpre:
+  case AArch64::LDRHpost:
+  case AArch64::LDRHpre:
+  case AArch64::LDRQpost:
+  case AArch64::LDRQpre:
+  case AArch64::LDRSpost:
+  case AArch64::LDRSpre:
+  case AArch64::LDRWpost:
+  case AArch64::LDRWpre:
+  case AArch64::LDRXpost:
+  case AArch64::LDRXpre:
+  case AArch64::ST1B_D_IMM:
   case AArch64::ST1B_H_IMM:
+  case AArch64::ST1B_IMM:
   case AArch64::ST1B_S_IMM:
-  case AArch64::ST1B_D_IMM:
+  case AArch64::ST1D_IMM:
+  case AArch64::ST1H_D_IMM:
   case AArch64::ST1H_IMM:
   case AArch64::ST1H_S_IMM:
-  case AArch64::ST1H_D_IMM:
-  case AArch64::ST1W_IMM:
   case AArch64::ST1W_D_IMM:
-  case AArch64::ST1D_IMM:
-
+  case AArch64::ST1W_IMM:
   case AArch64::ST2B_IMM:
+  case AArch64::ST2D_IMM:
   case AArch64::ST2H_IMM:
   case AArch64::ST2W_IMM:
-  case AArch64::ST2D_IMM:
   case AArch64::ST3B_IMM:
+  case AArch64::ST3D_IMM:
   case AArch64::ST3H_IMM:
   case AArch64::ST3W_IMM:
-  case AArch64::ST3D_IMM:
   case AArch64::ST4B_IMM:
+  case AArch64::ST4D_IMM:
   case AArch64::ST4H_IMM:
   case AArch64::ST4W_IMM:
-  case AArch64::ST4D_IMM:
-
-  case AArch64::LD1RB_IMM:
-  case AArch64::LD1RB_H_IMM:
-  case AArch64::LD1RB_S_IMM:
-  case AArch64::LD1RB_D_IMM:
-  case AArch64::LD1RSB_H_IMM:
-  case AArch64::LD1RSB_S_IMM:
-  case AArch64::LD1RSB_D_IMM:
-  case AArch64::LD1RH_IMM:
-  case AArch64::LD1RH_S_IMM:
-  case AArch64::LD1RH_D_IMM:
-  case AArch64::LD1RSH_S_IMM:
-  case AArch64::LD1RSH_D_IMM:
-  case AArch64::LD1RW_IMM:
-  case AArch64::LD1RW_D_IMM:
-  case AArch64::LD1RSW_IMM:
-  case AArch64::LD1RD_IMM:
-
-  case AArch64::LDNT1B_ZRI:
-  case AArch64::LDNT1H_ZRI:
-  case AArch64::LDNT1W_ZRI:
-  case AArch64::LDNT1D_ZRI:
+  case AArch64::STGPi:
+  case AArch64::STNPDi:
+  case AArch64::STNPQi:
+  case AArch64::STNPSi:
+  case AArch64::STNPWi:
+  case AArch64::STNPXi:
   case AArch64::STNT1B_ZRI:
+  case AArch64::STNT1D_ZRI:
   case AArch64::STNT1H_ZRI:
   case AArch64::STNT1W_ZRI:
-  case AArch64::STNT1D_ZRI:
-
-  case AArch64::LDNF1B_IMM:
-  case AArch64::LDNF1B_H_IMM:
-  case AArch64::LDNF1B_S_IMM:
-  case AArch64::LDNF1B_D_IMM:
-  case AArch64::LDNF1SB_H_IMM:
-  case AArch64::LDNF1SB_S_IMM:
-  case AArch64::LDNF1SB_D_IMM:
-  case AArch64::LDNF1H_IMM:
-  case AArch64::LDNF1H_S_IMM:
-  case AArch64::LDNF1H_D_IMM:
-  case AArch64::LDNF1SH_S_IMM:
-  case AArch64::LDNF1SH_D_IMM:
-  case AArch64::LDNF1W_IMM:
-  case AArch64::LDNF1W_D_IMM:
-  case AArch64::LDNF1SW_D_IMM:
-  case AArch64::LDNF1D_IMM:
+  case AArch64::STPDi:
+  case AArch64::STPQi:
+  case AArch64::STPSi:
+  case AArch64::STPWi:
+  case AArch64::STPXi:
+  case AArch64::STRBBpost:
+  case AArch64::STRBBpre:
+  case AArch64::STRBpost:
+  case AArch64::STRBpre:
+  case AArch64::STRDpost:
+  case AArch64::STRDpre:
+  case AArch64::STRHHpost:
+  case AArch64::STRHHpre:
+  case AArch64::STRHpost:
+  case AArch64::STRHpre:
+  case AArch64::STRQpost:
+  case AArch64::STRQpre:
+  case AArch64::STRSpost:
+  case AArch64::STRSpre:
+  case AArch64::STRWpost:
+  case AArch64::STRWpre:
+  case AArch64::STRXpost:
+  case AArch64::STRXpre:
     return 3;
-  case AArch64::ADDG:
-  case AArch64::STGi:
-  case AArch64::LDR_PXI:
-  case AArch64::STR_PXI:
-    return 2;
+  case AArch64::LDPDpost:
+  case AArch64::LDPDpre:
+  case AArch64::LDPQpost:
+  case AArch64::LDPQpre:
+  case AArch64::LDPSpost:
+  case AArch64::LDPSpre:
+  case AArch64::LDPWpost:
+  case AArch64::LDPWpre:
+  case AArch64::LDPXpost:
+  case AArch64::LDPXpre:
+  case AArch64::STPDpost:
+  case AArch64::STPDpre:
+  case AArch64::STPQpost:
+  case AArch64::STPQpre:
+  case AArch64::STPSpost:
+  case AArch64::STPSpre:
+  case AArch64::STPWpost:
+  case AArch64::STPWpre:
+  case AArch64::STPXpost:
+  case AArch64::STPXpre:
+    return 4;
   }
 }
 
@@ -3834,22 +3960,58 @@ bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, TypeSize &Scale,
     MinOffset = -256;
     MaxOffset = 255;
     break;
-  case AArch64::STRXpre:
-  case AArch64::STRDpre:
-  case AArch64::LDRXpost:
   case AArch64::LDRDpost:
+  case AArch64::LDRDpre:
+  case AArch64::LDRXpost:
+  case AArch64::LDRXpre:
+  case AArch64::STRDpost:
+  case AArch64::STRDpre:
+  case AArch64::STRXpost:
+  case AArch64::STRXpre:
     Scale = TypeSize::getFixed(1);
     Width = TypeSize::getFixed(8);
     MinOffset = -256;
     MaxOffset = 255;
     break;
   case AArch64::STRWpost:
+  case AArch64::STRWpre:
   case AArch64::LDRWpost:
+  case AArch64::LDRWpre:
+  case AArch64::STRSpost:
+  case AArch64::STRSpre:
+  case AArch64::LDRSpost:
+  case AArch64::LDRSpre:
     Scale = TypeSize::getFixed(1);
     Width = TypeSize::getFixed(4);
     MinOffset = -256;
     MaxOffset = 255;
     break;
+  case AArch64::LDRHpost:
+  case AArch64::LDRHpre:
+  case AArch64::STRHpost:
+  case AArch64::STRHpre:
+  case AArch64::LDRHHpost:
+  case AArch64::LDRHHpre:
+  case AArch64::STRHHpost:
+  case AArch64::STRHHpre:
+    Scale = TypeSize::getFixed(1);
+    Width = TypeSize::getFixed(2);
+    MinOffset = -256;
+    MaxOffset = 255;
+    break;
+  case AArch64::LDRBpost:
+  case AArch64::LDRBpre:
+  case AArch64::STRBpost:
+  case AArch64::STRBpre:
+  case AArch64::LDRBBpost:
+  case AArch64::LDRBBpre:
+  case AArch64::STRBBpost:
+  case AArch64::STRBBpre:
+    Scale = TypeSize::getFixed(1);
+    Width = TypeSize::getFixed(1);
+    MinOffset = -256;
+    MaxOffset = 255;
+    break;
   // Unscaled
   case AArch64::LDURQi:
   case AArch64::STURQi:
@@ -3918,8 +4080,10 @@ bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, TypeSize &Scale,
   case AArch64::LDNPQi:
   case AArch64::STPQi:
   case AArch64::STNPQi:
-  case AArch64::STPQpre:
   case AArch64::LDPQpost:
+  case AArch64::LDPQpre:
+  case AArch64::STPQpost:
+  case AArch64::STPQpre:
     Scale = TypeSize::getFixed(16);
     Width = TypeSize::getFixed(16 * 2);
     MinOffset = -64;
@@ -3933,10 +4097,14 @@ bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, TypeSize &Scale,
   case AArch64::STPDi:
   case AArch64::STNPXi:
   case AArch64::STNPDi:
-  case AArch64::STPXpre:
+  case AArch64::LDPDpost:
+  case AArch64::LDPDpre:
   case AArch64::LDPXpost:
+  case AArch64::LDPXpre:
+  case AArch64::STPDpost:
   case AArch64::STPDpre:
-  case AArch64::LDPDpost:
+  case AArch64::STPXpost:
+  case AArch64::STPXpre:
     Scale = TypeSize::getFixed(8);
     Width = TypeSize::getFixed(8 * 2);
     MinOffset = -64;
@@ -3950,6 +4118,14 @@ bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, TypeSize &Scale,
   case AArch64::STPSi:
   case AArch64::STNPWi:
   case AArch64::STNPSi:
+  case AArch64::LDPSpost:
+  case AArch64::LDPSpre:
+  case AArch64::LDPWpost:
+  case AArch64::LDPWpre:
+  case AArch64::STPSpost:
+  case AArch64::STPSpre:
+  case AArch64::STPWpost:
+  case AArch64::STPWpre:
     Scale = TypeSize::getFixed(4);
     Width = TypeSize::getFixed(4 * 2);
     MinOffset = -64;
@@ -10250,6 +10426,28 @@ AArch64InstrInfo::analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const {
       Init, IsUpdatePriorComp, Cond);
 }
 
+/// verifyInstruction - Perform target specific instruction verification.
+bool AArch64InstrInfo::verifyInstruction(const MachineInstr &MI,
+                                         StringRef &ErrInfo) const {
+
+  // Verify that immediate offsets on load/store instructions are within range.
+  // Stack objects with an FI operand are excluded as they can be fixed up
+  // during PEI.
+  TypeSize Scale(0U, false), Width(0U, false);
+  int64_t MinOffset, MaxOffset;
+  if (getMemOpInfo(MI.getOpcode(), Scale, Width, MinOffset, MaxOffset)) {
+    unsigned ImmIdx = getLoadStoreImmIdx(MI.getOpcode());
+    if (MI.getOperand(ImmIdx).isImm() && !MI.getOperand(ImmIdx - 1).isFI()) {
+      int64_t Imm = MI.getOperand(ImmIdx).getImm();
+      if (Imm < MinOffset || Imm > MaxOffset) {
+        ErrInfo = "Unexpected immediate on load/store instruction";
+        return false;
+      }
+    }
+  }
+  return true;
+}
+
 #define GET_INSTRINFO_HELPERS
 #define GET_INSTRMAP_INFO
 #include "AArch64GenInstrInfo.inc"
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.h b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
index 24a500aa190abbc..1d2f69bd8530263 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
@@ -583,6 +583,10 @@ class AArch64InstrInfo final : public AArch64GenInstrInfo {
   std::optional<unsigned>
   canRemovePTestInstr(MachineInstr *PTest, MachineInstr *Mask,
                       MachineInstr *Pred, const MachineRegisterInfo *MRI) const;
+
+  /// verifyInstruction - Perform target specific instruction verification.
+  bool verifyInstruction(const MachineInstr &MI,
+                         StringRef &ErrInfo) const override;
 };
 
 struct UsedNZCV {
diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-unsafe-stack-call.mir b/llvm/test/CodeGen/AArch64/machine-outliner-unsafe-stack-call.mir
index 0853b23e52916d5..679c9687e66f039 100644
--- a/llvm/test/CodeGen/AArch64/machine-outliner-unsafe-stack-call.mir
+++ b/llvm/test/CodeGen/AArch64/machine-outliner-unsafe-stack-call.mir
@@ -25,10 +25,10 @@ body:             |
     ; CHECK-DAG: baz
     $lr = ORRXri $xzr, 1
     BL @foo, implicit-def dead $lr, implicit $sp
-    $x20, $x19 = LDPXi $sp, 255
-    $x20, $x19 = LDPXi $sp, 255
-    $x20, $x19 = LDPXi $sp, 255
-    $x20, $x19 = LDPXi $sp, 255
+    $x20, $x19 = LDPXi $sp, 63
+    $x20, $x19 = LDPXi $sp, 63
+    $x20, $x19 = LDPXi $sp, 63
+    $x20, $x19 = LDPXi $sp, 63
   bb.1:
     BL @bar, implicit-def dead $lr, implicit $sp
     $x11 = ADDXri $sp, 48, 0;
@@ -55,10 +55,10 @@ body:             |
     ; CHECK-DAG: baz
     $lr = ORRXri $xzr, 1
     BL @foo, implicit-def dead $lr, implicit $sp
-    $x20, $x19 = LDPXi $sp, 255
-    $x20, $x19 = LDPXi $sp, 255
-    $x20, $x19 = LDPXi $sp, 255
-    $x20, $x19 = LDPXi $sp, 255
+    $x20, $x19 = LDPXi $sp, 63
+    $x20, $x19 = LDPXi $sp, 63
+    $x20, $x19 = LDPXi $sp, 63
+    $x20, $x19 = LDPXi $sp, 63
   bb.1:
     BL @bar, implicit-def dead $lr, implicit $sp
     $x11 = ADDXri $sp, 48, 0;
diff --git a/llvm/test/CodeGen/AArch64/verify-memop.mir b/llvm/test/CodeGen/AArch64/verify-memop.mir
new file mode 100644
index 000000000000000..90aa5e64a7f7209
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/verify-memop.mir
@@ -0,0 +1,51 @@
+# RUN: not --crash llc -mtriple=aarch64 -run-pass machineverifier -o /dev/null %s 2>&1 | FileCheck -implicit-check-not="Bad machine code"  %s
+
+# CHECK: *** Bad machine code: Unexpected immediate on load/store instruction ***
+# CHECK: - instruction: STRSui $s1, $x0, 4096
+# CHECK: *** Bad machine code: Unexpected immediate on load/store instruction ***
+# CHECK: - instruction: STRSui $s1, $x0, -1
+# CHECK: *** Bad machine code: Unexpected immediate on load/store instruction ***
+# CHECK: - instruction: early-clobber $x0, $w1 = LDRWpre $x0(tied-def 0), 256
+# CHECK: *** Bad machine code: Unexpected immediate on load/store instruction ***
+# CHECK: - instruction: early-clobber $x0, $w1 = LDRWpre $x0(tied-def 0), -257
+# CHECK: *** Bad machine code: Unexpected immediate on load/store instruction ***
+# CHECK: - instruction: early-clobber $x0, $w1 = LDRWpre $x0(tied-def 0), 256
+# CHECK: *** Bad machine code: Unexpected immediate on load/store instruction ***
+# CHECK: - instruction: early-clobber $x0, $w1 = LDRWpre $x0(tied-def 0), -257
+# CHECK: *** Bad machine code: Unexpected immediate on load/store instruction ***
+# CHECK: - instruction: STRBBui $w1, $x0, 4096
+# CHECK: *** Bad machine code: Unexpected immediate on load/store instruction ***
+# CHECK: - instruction: STRBBui $w1, $x0, -1
+# CHECK: *** Bad machine code: Unexpected immediate on load/store instruction ***
+# CHECK: - instruction: STRHHui $w1, $x0, 4096
+# CHECK: *** Bad machine code: Unexpected immediate on load/store instruction ***
+# CHECK: - instruction: STRHHui $w1, $x0, -1
+# CHECK: *** Bad machine code: Unexpected immediate on load/store instruction ***
+# CHECK: - instruction: early-clobber $x0 = STRSpost $s1, $x0(tied-def 0), 256
+# CHECK: *** Bad machine code: Unexpected immediate on load/store instruction ***
+# CHECK: - instruction: early-clobber $x0 = STRSpost $s1, $x0(tied-def 0), -257
+
+---
+name:            testoffsets
+alignment:       4
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $d0, $s1, $q2, $x0, $w1
+
+    STRSui $s1, $x0, 4095
+    STRSui $s1, $x0, 4096
+    STRSui $s1, $x0, 0
+    STRSui $s1, $x0, -1
+    $x0, $w1 = LDRWpre $x0, 256
+    $x0, $w1 = LDRWpre $x0, -257
+    $x0, $w1 = LDRWpre $x0, 256
+    $x0, $w1 = LDRWpre $x0, -257
+    STRBBui $w1, $x0, 4096
+    STRBBui $w1, $x0, -1
+    STRHHui $w1, $x0, 4096
+    STRHHui $w1, $x0, -1
+    $x0 = STRSpost $s1, $x0, 256
+    $x0 = STRSpost $s1, $x0, -257
+    RET undef $lr, implicit $x0
+...

From 33190490c667aaf8b08d5af8b8ce84524f856e80 Mon Sep 17 00:00:00 2001
From: zhongyunde 00443407 <zhongyunde@huawei.com>
Date: Fri, 10 Nov 2023 07:29:03 -0500
Subject: [PATCH 006/441] [AArch64] merge index address with large offset into
 base address

A case for this transformation, https://gcc.godbolt.org/z/nhYcWq1WE
Fold
  mov     w8, #56952
  movk    w8, #15, lsl #16
  ldrb    w0, [x0, x8]
into
  add     x0, x0, 1036288
  ldrb    w0, [x0, 3704]

Only LDRBBroX is supported for the first time.
Fix https://github.com/llvm/llvm-project/issues/71917

Note: This PR is try relanding the commit 32878c2065 with fix crash for PR79756
  this crash is exposes when there is MOVKWi instruction in the head of a block,
but without MOVZWi
---
 llvm/lib/Target/AArch64/AArch64InstrInfo.cpp  |  10 +
 llvm/lib/Target/AArch64/AArch64InstrInfo.h    |   3 +
 .../AArch64/AArch64LoadStoreOptimizer.cpp     | 232 ++++++++++++++++++
 llvm/test/CodeGen/AArch64/arm64-addrmode.ll   |  15 +-
 .../AArch64/large-offset-ldr-merge.mir        |  25 +-
 5 files changed, 273 insertions(+), 12 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index 5f8ed9cb6180a2c..ff17986208d8aab 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -4516,6 +4516,16 @@ AArch64InstrInfo::getLdStOffsetOp(const MachineInstr &MI) {
   return MI.getOperand(Idx);
 }
 
+const MachineOperand &
+AArch64InstrInfo::getLdStAmountOp(const MachineInstr &MI) {
+  switch (MI.getOpcode()) {
+  default:
+    llvm_unreachable("Unexpected opcode");
+  case AArch64::LDRBBroX:
+    return MI.getOperand(4);
+  }
+}
+
 static const TargetRegisterClass *getRegClass(const MachineInstr &MI,
                                               Register Reg) {
   if (MI.getParent() == nullptr)
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.h b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
index 1d2f69bd8530263..a1f2fbff016312a 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
@@ -254,6 +254,9 @@ class AArch64InstrInfo final : public AArch64GenInstrInfo {
   /// Returns whether the physical register is FP or NEON.
   static bool isFpOrNEON(Register Reg);
 
+  /// Returns the shift amount operator of a load/store.
+  static const MachineOperand &getLdStAmountOp(const MachineInstr &MI);
+
   /// Returns whether the instruction is FP or NEON.
   static bool isFpOrNEON(const MachineInstr &MI);
 
diff --git a/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp b/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
index ff3a21cd0da3b7c..8de3f8db84ae2b7 100644
--- a/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
+++ b/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
@@ -64,6 +64,8 @@ STATISTIC(NumZeroStoresPromoted, "Number of narrow zero stores promoted");
 STATISTIC(NumLoadsFromStoresPromoted, "Number of loads from stores promoted");
 STATISTIC(NumFailedAlignmentCheck, "Number of load/store pair transformation "
                                    "not passed the alignment check");
+STATISTIC(NumConstOffsetFolded,
+          "Number of const offset of index address folded");
 
 DEBUG_COUNTER(RegRenamingCounter, DEBUG_TYPE "-reg-renaming",
               "Controls which pairs are considered for renaming");
@@ -77,6 +79,11 @@ static cl::opt<unsigned> LdStLimit("aarch64-load-store-scan-limit",
 static cl::opt<unsigned> UpdateLimit("aarch64-update-scan-limit", cl::init(100),
                                      cl::Hidden);
 
+// The LdStConstLimit limits how far we search for const offset instructions
+// when we form index address load/store instructions.
+static cl::opt<unsigned> LdStConstLimit("aarch64-load-store-const-scan-limit",
+                                        cl::init(10), cl::Hidden);
+
 // Enable register renaming to find additional store pairing opportunities.
 static cl::opt<bool> EnableRenaming("aarch64-load-store-renaming",
                                     cl::init(true), cl::Hidden);
@@ -173,6 +180,13 @@ struct AArch64LoadStoreOpt : public MachineFunctionPass {
   findMatchingUpdateInsnForward(MachineBasicBlock::iterator I,
                                 int UnscaledOffset, unsigned Limit);
 
+  // Scan the instruction list to find a register assigned with a const
+  // value that can be combined with the current instruction (a load or store)
+  // using base addressing with writeback. Scan backwards.
+  MachineBasicBlock::iterator
+  findMatchingConstOffsetBackward(MachineBasicBlock::iterator I, unsigned Limit,
+                                  unsigned &Offset);
+
   // Scan the instruction list to find a base register update that can
   // be combined with the current instruction (a load or store) using
   // pre or post indexed addressing with writeback. Scan backwards.
@@ -184,11 +198,19 @@ struct AArch64LoadStoreOpt : public MachineFunctionPass {
   bool isMatchingUpdateInsn(MachineInstr &MemMI, MachineInstr &MI,
                             unsigned BaseReg, int Offset);
 
+  bool isMatchingMovConstInsn(MachineInstr &MemMI, MachineInstr &MI,
+                              unsigned IndexReg, unsigned &Offset);
+
   // Merge a pre- or post-index base register update into a ld/st instruction.
   MachineBasicBlock::iterator
   mergeUpdateInsn(MachineBasicBlock::iterator I,
                   MachineBasicBlock::iterator Update, bool IsPreIdx);
 
+  MachineBasicBlock::iterator
+  mergeConstOffsetInsn(MachineBasicBlock::iterator I,
+                       MachineBasicBlock::iterator Update, unsigned Offset,
+                       int Scale);
+
   // Find and merge zero store instructions.
   bool tryToMergeZeroStInst(MachineBasicBlock::iterator &MBBI);
 
@@ -201,6 +223,9 @@ struct AArch64LoadStoreOpt : public MachineFunctionPass {
   // Find and merge a base register updates before or after a ld/st instruction.
   bool tryToMergeLdStUpdate(MachineBasicBlock::iterator &MBBI);
 
+  // Find and merge an index ldr/st instruction into a base ld/st instruction.
+  bool tryToMergeIndexLdSt(MachineBasicBlock::iterator &MBBI, int Scale);
+
   bool optimizeBlock(MachineBasicBlock &MBB, bool EnableNarrowZeroStOpt);
 
   bool runOnMachineFunction(MachineFunction &Fn) override;
@@ -483,6 +508,16 @@ static unsigned getPreIndexedOpcode(unsigned Opc) {
   }
 }
 
+static unsigned getBaseAddressOpcode(unsigned Opc) {
+  // TODO: Add more index address loads/stores.
+  switch (Opc) {
+  default:
+    llvm_unreachable("Opcode has no base address equivalent!");
+  case AArch64::LDRBBroX:
+    return AArch64::LDRBBui;
+  }
+}
+
 static unsigned getPostIndexedOpcode(unsigned Opc) {
   switch (Opc) {
   default:
@@ -724,6 +759,20 @@ static bool isMergeableLdStUpdate(MachineInstr &MI) {
   }
 }
 
+// Make sure this is a reg+reg Ld/St
+static bool isMergeableIndexLdSt(MachineInstr &MI, int &Scale) {
+  unsigned Opc = MI.getOpcode();
+  switch (Opc) {
+  default:
+    return false;
+  // Scaled instructions.
+  // TODO: Add more index address loads/stores.
+  case AArch64::LDRBBroX:
+    Scale = 1;
+    return true;
+  }
+}
+
 static bool isRewritableImplicitDef(unsigned Opc) {
   switch (Opc) {
   default:
@@ -2053,6 +2102,63 @@ AArch64LoadStoreOpt::mergeUpdateInsn(MachineBasicBlock::iterator I,
   return NextI;
 }
 
+MachineBasicBlock::iterator
+AArch64LoadStoreOpt::mergeConstOffsetInsn(MachineBasicBlock::iterator I,
+                                          MachineBasicBlock::iterator Update,
+                                          unsigned Offset, int Scale) {
+  assert((Update->getOpcode() == AArch64::MOVKWi) &&
+         "Unexpected const mov instruction to merge!");
+  MachineBasicBlock::iterator E = I->getParent()->end();
+  MachineBasicBlock::iterator NextI = next_nodbg(I, E);
+  MachineBasicBlock::iterator PrevI = prev_nodbg(Update, E);
+  MachineInstr &MemMI = *I;
+  unsigned Mask = (1 << 12) * Scale - 1;
+  unsigned Low = Offset & Mask;
+  unsigned High = Offset - Low;
+  Register BaseReg = AArch64InstrInfo::getLdStBaseOp(MemMI).getReg();
+  Register IndexReg = AArch64InstrInfo::getLdStOffsetOp(MemMI).getReg();
+  MachineInstrBuilder AddMIB, MemMIB;
+
+  // Add IndexReg, BaseReg, High (the BaseReg may be SP)
+  AddMIB =
+      BuildMI(*I->getParent(), I, I->getDebugLoc(), TII->get(AArch64::ADDXri))
+          .addDef(IndexReg)
+          .addUse(BaseReg)
+          .addImm(High >> 12) // shifted value
+          .addImm(12);        // shift 12
+  (void)AddMIB;
+  // Ld/St DestReg, IndexReg, Imm12
+  unsigned NewOpc = getBaseAddressOpcode(I->getOpcode());
+  MemMIB = BuildMI(*I->getParent(), I, I->getDebugLoc(), TII->get(NewOpc))
+               .add(getLdStRegOp(MemMI))
+               .add(AArch64InstrInfo::getLdStOffsetOp(MemMI))
+               .addImm(Low / Scale)
+               .setMemRefs(I->memoperands())
+               .setMIFlags(I->mergeFlagsWith(*Update));
+  (void)MemMIB;
+
+  ++NumConstOffsetFolded;
+  LLVM_DEBUG(dbgs() << "Creating base address load/store.\n");
+  LLVM_DEBUG(dbgs() << "    Replacing instructions:\n    ");
+  LLVM_DEBUG(PrevI->print(dbgs()));
+  LLVM_DEBUG(dbgs() << "    ");
+  LLVM_DEBUG(Update->print(dbgs()));
+  LLVM_DEBUG(dbgs() << "    ");
+  LLVM_DEBUG(I->print(dbgs()));
+  LLVM_DEBUG(dbgs() << "  with instruction:\n    ");
+  LLVM_DEBUG(((MachineInstr *)AddMIB)->print(dbgs()));
+  LLVM_DEBUG(dbgs() << "    ");
+  LLVM_DEBUG(((MachineInstr *)MemMIB)->print(dbgs()));
+  LLVM_DEBUG(dbgs() << "\n");
+
+  // Erase the old instructions for the block.
+  I->eraseFromParent();
+  PrevI->eraseFromParent();
+  Update->eraseFromParent();
+
+  return NextI;
+}
+
 bool AArch64LoadStoreOpt::isMatchingUpdateInsn(MachineInstr &MemMI,
                                                MachineInstr &MI,
                                                unsigned BaseReg, int Offset) {
@@ -2100,6 +2206,34 @@ bool AArch64LoadStoreOpt::isMatchingUpdateInsn(MachineInstr &MemMI,
   return false;
 }
 
+bool AArch64LoadStoreOpt::isMatchingMovConstInsn(MachineInstr &MemMI,
+                                                 MachineInstr &MI,
+                                                 unsigned IndexReg,
+                                                 unsigned &Offset) {
+  // The update instruction source and destination register must be the
+  // same as the load/store index register.
+  if (MI.getOpcode() == AArch64::MOVKWi &&
+      TRI->isSuperOrSubRegisterEq(IndexReg, MI.getOperand(1).getReg())) {
+
+    // movz + movk hold a large offset of a Ld/St instruction.
+    MachineBasicBlock::iterator B = MI.getParent()->begin();
+    MachineBasicBlock::iterator MBBI = &MI;
+    // Skip the scene when the MI is the first instruction of a block.
+    if (MBBI == B)
+      return false;
+    MBBI = prev_nodbg(MBBI, B);
+    MachineInstr &MovzMI = *MBBI;
+    if (MovzMI.getOpcode() == AArch64::MOVZWi) {
+      unsigned Low = MovzMI.getOperand(1).getImm();
+      unsigned High = MI.getOperand(2).getImm() << MI.getOperand(3).getImm();
+      Offset = High + Low;
+      // 12-bit optionally shifted immediates are legal for adds.
+      return Offset >> 24 == 0;
+    }
+  }
+  return false;
+}
+
 MachineBasicBlock::iterator AArch64LoadStoreOpt::findMatchingUpdateInsnForward(
     MachineBasicBlock::iterator I, int UnscaledOffset, unsigned Limit) {
   MachineBasicBlock::iterator E = I->getParent()->end();
@@ -2255,6 +2389,60 @@ MachineBasicBlock::iterator AArch64LoadStoreOpt::findMatchingUpdateInsnBackward(
   return E;
 }
 
+MachineBasicBlock::iterator
+AArch64LoadStoreOpt::findMatchingConstOffsetBackward(
+    MachineBasicBlock::iterator I, unsigned Limit, unsigned &Offset) {
+  MachineBasicBlock::iterator B = I->getParent()->begin();
+  MachineBasicBlock::iterator E = I->getParent()->end();
+  MachineInstr &MemMI = *I;
+  MachineBasicBlock::iterator MBBI = I;
+
+  // If the load is the first instruction in the block, there's obviously
+  // not any matching load or store.
+  if (MBBI == B)
+    return E;
+
+  // Make sure the IndexReg is killed and the shift amount is zero.
+  // TODO: Relex this restriction to extend, simplify processing now.
+  if (!AArch64InstrInfo::getLdStOffsetOp(MemMI).isKill() ||
+      !AArch64InstrInfo::getLdStAmountOp(MemMI).isImm() ||
+      (AArch64InstrInfo::getLdStAmountOp(MemMI).getImm() != 0))
+    return E;
+
+  Register IndexReg = AArch64InstrInfo::getLdStOffsetOp(MemMI).getReg();
+
+  // Track which register units have been modified and used between the first
+  // insn (inclusive) and the second insn.
+  ModifiedRegUnits.clear();
+  UsedRegUnits.clear();
+  unsigned Count = 0;
+  do {
+    MBBI = prev_nodbg(MBBI, B);
+    MachineInstr &MI = *MBBI;
+
+    // Don't count transient instructions towards the search limit since there
+    // may be different numbers of them if e.g. debug information is present.
+    if (!MI.isTransient())
+      ++Count;
+
+    // If we found a match, return it.
+    if (isMatchingMovConstInsn(*I, MI, IndexReg, Offset)) {
+      return MBBI;
+    }
+
+    // Update the status of what the instruction clobbered and used.
+    LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits, TRI);
+
+    // Otherwise, if the index register is used or modified, we have no match,
+    // so return early.
+    if (!ModifiedRegUnits.available(IndexReg) ||
+        !UsedRegUnits.available(IndexReg))
+      return E;
+
+  } while (MBBI != B && Count < Limit);
+  return E;
+}
+
 bool AArch64LoadStoreOpt::tryToPromoteLoadFromStore(
     MachineBasicBlock::iterator &MBBI) {
   MachineInstr &MI = *MBBI;
@@ -2443,6 +2631,34 @@ bool AArch64LoadStoreOpt::tryToMergeLdStUpdate
   return false;
 }
 
+bool AArch64LoadStoreOpt::tryToMergeIndexLdSt(MachineBasicBlock::iterator &MBBI,
+                                              int Scale) {
+  MachineInstr &MI = *MBBI;
+  MachineBasicBlock::iterator E = MI.getParent()->end();
+  MachineBasicBlock::iterator Update;
+
+  // Don't know how to handle unscaled pre/post-index versions below, so bail.
+  if (TII->hasUnscaledLdStOffset(MI.getOpcode()))
+    return false;
+
+  // Look back to try to find a const offset for index LdSt instruction. For
+  // example,
+  // mov x8, #LargeImm   ; = a * (1<<12) + imm12
+  // ldr x1, [x0, x8]
+  // merged into:
+  // add x8, x0, a * (1<<12)
+  // ldr x1, [x8, imm12]
+  unsigned Offset;
+  Update = findMatchingConstOffsetBackward(MBBI, LdStConstLimit, Offset);
+  if (Update != E && (Offset & (Scale - 1)) == 0) {
+    // Merge the imm12 into the ld/st.
+    MBBI = mergeConstOffsetInsn(MBBI, Update, Offset, Scale);
+    return true;
+  }
+
+  return false;
+}
+
 bool AArch64LoadStoreOpt::optimizeBlock(MachineBasicBlock &MBB,
                                         bool EnableNarrowZeroStOpt) {
 
@@ -2521,6 +2737,22 @@ bool AArch64LoadStoreOpt::optimizeBlock(MachineBasicBlock &MBB,
       ++MBBI;
   }
 
+  // 5) Find a register assigned with a const value that can be combined with
+  // into the load or store. e.g.,
+  //        mov x8, #LargeImm   ; = a * (1<<12) + imm12
+  //        ldr x1, [x0, x8]
+  //        ; becomes
+  //        add x8, x0, a * (1<<12)
+  //        ldr x1, [x8, imm12]
+  for (MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
+       MBBI != E;) {
+    int Scale;
+    if (isMergeableIndexLdSt(*MBBI, Scale) && tryToMergeIndexLdSt(MBBI, Scale))
+      Modified = true;
+    else
+      ++MBBI;
+  }
+
   return Modified;
 }
 
diff --git a/llvm/test/CodeGen/AArch64/arm64-addrmode.ll b/llvm/test/CodeGen/AArch64/arm64-addrmode.ll
index d39029163a47aa6..2181eaaee7db686 100644
--- a/llvm/test/CodeGen/AArch64/arm64-addrmode.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-addrmode.ll
@@ -214,9 +214,8 @@ define void @t17(i64 %a) {
 define i8 @LdOffset_i8(ptr %a)  {
 ; CHECK-LABEL: LdOffset_i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #56952 // =0xde78
-; CHECK-NEXT:    movk w8, #15, lsl #16
-; CHECK-NEXT:    ldrb w0, [x0, x8]
+; CHECK-NEXT:    add x8, x0, #253, lsl #12 // =1036288
+; CHECK-NEXT:    ldrb w0, [x8, #3704]
 ; CHECK-NEXT:    ret
   %arrayidx = getelementptr inbounds i8, ptr %a, i64 1039992
   %val = load i8, ptr %arrayidx, align 1
@@ -227,9 +226,8 @@ define i8 @LdOffset_i8(ptr %a)  {
 define i32 @LdOffset_i8_zext32(ptr %a)  {
 ; CHECK-LABEL: LdOffset_i8_zext32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #56952 // =0xde78
-; CHECK-NEXT:    movk w8, #15, lsl #16
-; CHECK-NEXT:    ldrb w0, [x0, x8]
+; CHECK-NEXT:    add x8, x0, #253, lsl #12 // =1036288
+; CHECK-NEXT:    ldrb w0, [x8, #3704]
 ; CHECK-NEXT:    ret
   %arrayidx = getelementptr inbounds i8, ptr %a, i64 1039992
   %val = load i8, ptr %arrayidx, align 1
@@ -255,9 +253,8 @@ define i32 @LdOffset_i8_sext32(ptr %a)  {
 define i64 @LdOffset_i8_zext64(ptr %a)  {
 ; CHECK-LABEL: LdOffset_i8_zext64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #56952 // =0xde78
-; CHECK-NEXT:    movk w8, #15, lsl #16
-; CHECK-NEXT:    ldrb w0, [x0, x8]
+; CHECK-NEXT:    add x8, x0, #253, lsl #12 // =1036288
+; CHECK-NEXT:    ldrb w0, [x8, #3704]
 ; CHECK-NEXT:    ret
   %arrayidx = getelementptr inbounds i8, ptr %a, i64 1039992
   %val = load i8, ptr %arrayidx, align 1
diff --git a/llvm/test/CodeGen/AArch64/large-offset-ldr-merge.mir b/llvm/test/CodeGen/AArch64/large-offset-ldr-merge.mir
index 488f1ffdb52f3b2..473c74323d93998 100644
--- a/llvm/test/CodeGen/AArch64/large-offset-ldr-merge.mir
+++ b/llvm/test/CodeGen/AArch64/large-offset-ldr-merge.mir
@@ -14,9 +14,8 @@ body:             |
     ; CHECK-LABEL: name: LdOffset
     ; CHECK: liveins: $x0
     ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: renamable $w8 = MOVZWi 56952, 0
-    ; CHECK-NEXT: renamable $w8 = MOVKWi $w8, 15, 16, implicit-def $x8
-    ; CHECK-NEXT: renamable $w0 = LDRBBroX killed renamable $x0, killed renamable $x8, 0, 0
+    ; CHECK-NEXT: $x8 = ADDXri $x0, 253, 12
+    ; CHECK-NEXT: renamable $w0 = LDRBBui killed renamable $x8, 3704
     ; CHECK-NEXT: RET undef $lr, implicit $w0
     renamable $w8 = MOVZWi 56952, 0
     renamable $w8 = MOVKWi $w8, 15, 16, implicit-def $x8
@@ -46,3 +45,23 @@ body:             |
     renamable $w0 = LDRBBroX killed renamable $x0, renamable $x8, 0, 0
     RET undef $lr, implicit $w0
 ...
+
+# Negative test: No MOVZWi used for the const offset
+---
+name:            LdOffset_missing_MOVZ
+tracksRegLiveness: true
+liveins:
+  - { reg: '$x0', virtual-reg: '' }
+body:             |
+  bb.0.entry:
+    liveins: $x0
+
+    ; CHECK-LABEL: name: LdOffset_missing_MOVZ
+    ; CHECK: liveins: $x0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: renamable $w8 = MOVKWi $w8, 15, 16, implicit-def $x8
+    ; CHECK-NEXT: renamable $w0 = LDRBBroX killed renamable $x0, killed renamable $x8, 0, 0
+    ; CHECK-NEXT: RET undef $lr, implicit $w0
+    renamable $w8 = MOVKWi $w8, 15, 16, implicit-def $x8
+    renamable $w0 = LDRBBroX killed renamable $x0, killed renamable $x8, 0, 0
+    RET undef $lr, implicit $w0

From 43ffe2eed0d9f73789dbe213023733d164999306 Mon Sep 17 00:00:00 2001
From: zhongyunde 00443407 <zhongyunde@huawei.com>
Date: Mon, 18 Dec 2023 00:51:48 -0500
Subject: [PATCH 007/441] [AArch64] Fold more load.x into load.i with large
 offset

The list of load.x is refer to canFoldIntoAddrMode on D152828.
Also support LDRSroX missed in canFoldIntoAddrMode
---
 llvm/lib/Target/AArch64/AArch64InstrInfo.cpp  | 13 +++
 .../AArch64/AArch64LoadStoreOptimizer.cpp     | 51 ++++++++++-
 llvm/test/CodeGen/AArch64/arm64-addrmode.ll   | 85 ++++++++-----------
 3 files changed, 96 insertions(+), 53 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index ff17986208d8aab..d818d3ba51c59d2 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -4521,7 +4521,20 @@ AArch64InstrInfo::getLdStAmountOp(const MachineInstr &MI) {
   switch (MI.getOpcode()) {
   default:
     llvm_unreachable("Unexpected opcode");
+  case AArch64::LDRBroX:
   case AArch64::LDRBBroX:
+  case AArch64::LDRSBXroX:
+  case AArch64::LDRSBWroX:
+  case AArch64::LDRHroX:
+  case AArch64::LDRHHroX:
+  case AArch64::LDRSHXroX:
+  case AArch64::LDRSHWroX:
+  case AArch64::LDRWroX:
+  case AArch64::LDRSroX:
+  case AArch64::LDRSWroX:
+  case AArch64::LDRDroX:
+  case AArch64::LDRXroX:
+  case AArch64::LDRQroX:
     return MI.getOperand(4);
   }
 }
diff --git a/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp b/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
index 8de3f8db84ae2b7..de1727aa6ec70f7 100644
--- a/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
+++ b/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
@@ -509,12 +509,38 @@ static unsigned getPreIndexedOpcode(unsigned Opc) {
 }
 
 static unsigned getBaseAddressOpcode(unsigned Opc) {
-  // TODO: Add more index address loads/stores.
+  // TODO: Add more index address stores.
   switch (Opc) {
   default:
     llvm_unreachable("Opcode has no base address equivalent!");
+  case AArch64::LDRBroX:
+    return AArch64::LDRBui;
   case AArch64::LDRBBroX:
     return AArch64::LDRBBui;
+  case AArch64::LDRSBXroX:
+    return AArch64::LDRSBXui;
+  case AArch64::LDRSBWroX:
+    return AArch64::LDRSBWui;
+  case AArch64::LDRHroX:
+    return AArch64::LDRHui;
+  case AArch64::LDRHHroX:
+    return AArch64::LDRHHui;
+  case AArch64::LDRSHXroX:
+    return AArch64::LDRSHXui;
+  case AArch64::LDRSHWroX:
+    return AArch64::LDRSHWui;
+  case AArch64::LDRWroX:
+    return AArch64::LDRWui;
+  case AArch64::LDRSroX:
+    return AArch64::LDRSui;
+  case AArch64::LDRSWroX:
+    return AArch64::LDRSWui;
+  case AArch64::LDRDroX:
+    return AArch64::LDRDui;
+  case AArch64::LDRXroX:
+    return AArch64::LDRXui;
+  case AArch64::LDRQroX:
+    return AArch64::LDRQui;
   }
 }
 
@@ -766,10 +792,31 @@ static bool isMergeableIndexLdSt(MachineInstr &MI, int &Scale) {
   default:
     return false;
   // Scaled instructions.
-  // TODO: Add more index address loads/stores.
+  // TODO: Add more index address stores.
+  case AArch64::LDRBroX:
   case AArch64::LDRBBroX:
+  case AArch64::LDRSBXroX:
+  case AArch64::LDRSBWroX:
     Scale = 1;
     return true;
+  case AArch64::LDRHroX:
+  case AArch64::LDRHHroX:
+  case AArch64::LDRSHXroX:
+  case AArch64::LDRSHWroX:
+    Scale = 2;
+    return true;
+  case AArch64::LDRWroX:
+  case AArch64::LDRSroX:
+  case AArch64::LDRSWroX:
+    Scale = 4;
+    return true;
+  case AArch64::LDRDroX:
+  case AArch64::LDRXroX:
+    Scale = 8;
+    return true;
+  case AArch64::LDRQroX:
+    Scale = 16;
+    return true;
   }
 }
 
diff --git a/llvm/test/CodeGen/AArch64/arm64-addrmode.ll b/llvm/test/CodeGen/AArch64/arm64-addrmode.ll
index 2181eaaee7db686..bfef61abd8c129f 100644
--- a/llvm/test/CodeGen/AArch64/arm64-addrmode.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-addrmode.ll
@@ -239,9 +239,8 @@ define i32 @LdOffset_i8_zext32(ptr %a)  {
 define i32 @LdOffset_i8_sext32(ptr %a)  {
 ; CHECK-LABEL: LdOffset_i8_sext32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #56952 // =0xde78
-; CHECK-NEXT:    movk w8, #15, lsl #16
-; CHECK-NEXT:    ldrsb w0, [x0, x8]
+; CHECK-NEXT:    add x8, x0, #253, lsl #12 // =1036288
+; CHECK-NEXT:    ldrsb w0, [x8, #3704]
 ; CHECK-NEXT:    ret
   %arrayidx = getelementptr inbounds i8, ptr %a, i64 1039992
   %val = load i8, ptr %arrayidx, align 1
@@ -266,9 +265,8 @@ define i64 @LdOffset_i8_zext64(ptr %a)  {
 define i64 @LdOffset_i8_sext64(ptr %a)  {
 ; CHECK-LABEL: LdOffset_i8_sext64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #56952 // =0xde78
-; CHECK-NEXT:    movk w8, #15, lsl #16
-; CHECK-NEXT:    ldrsb x0, [x0, x8]
+; CHECK-NEXT:    add x8, x0, #253, lsl #12 // =1036288
+; CHECK-NEXT:    ldrsb x0, [x8, #3704]
 ; CHECK-NEXT:    ret
   %arrayidx = getelementptr inbounds i8, ptr %a, i64 1039992
   %val = load i8, ptr %arrayidx, align 1
@@ -280,9 +278,8 @@ define i64 @LdOffset_i8_sext64(ptr %a)  {
 define i16 @LdOffset_i16(ptr %a)  {
 ; CHECK-LABEL: LdOffset_i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #48368 // =0xbcf0
-; CHECK-NEXT:    movk w8, #31, lsl #16
-; CHECK-NEXT:    ldrh w0, [x0, x8]
+; CHECK-NEXT:    add x8, x0, #506, lsl #12 // =2072576
+; CHECK-NEXT:    ldrh w0, [x8, #7408]
 ; CHECK-NEXT:    ret
   %arrayidx = getelementptr inbounds i16, ptr %a, i64 1039992
   %val = load i16, ptr %arrayidx, align 2
@@ -293,9 +290,8 @@ define i16 @LdOffset_i16(ptr %a)  {
 define i32 @LdOffset_i16_zext32(ptr %a)  {
 ; CHECK-LABEL: LdOffset_i16_zext32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #48368 // =0xbcf0
-; CHECK-NEXT:    movk w8, #31, lsl #16
-; CHECK-NEXT:    ldrh w0, [x0, x8]
+; CHECK-NEXT:    add x8, x0, #506, lsl #12 // =2072576
+; CHECK-NEXT:    ldrh w0, [x8, #7408]
 ; CHECK-NEXT:    ret
   %arrayidx = getelementptr inbounds i16, ptr %a, i64 1039992
   %val = load i16, ptr %arrayidx, align 2
@@ -307,9 +303,8 @@ define i32 @LdOffset_i16_zext32(ptr %a)  {
 define i32 @LdOffset_i16_sext32(ptr %a)  {
 ; CHECK-LABEL: LdOffset_i16_sext32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #48368 // =0xbcf0
-; CHECK-NEXT:    movk w8, #31, lsl #16
-; CHECK-NEXT:    ldrsh w0, [x0, x8]
+; CHECK-NEXT:    add x8, x0, #506, lsl #12 // =2072576
+; CHECK-NEXT:    ldrsh w0, [x8, #7408]
 ; CHECK-NEXT:    ret
   %arrayidx = getelementptr inbounds i16, ptr %a, i64 1039992
   %val = load i16, ptr %arrayidx, align 2
@@ -321,9 +316,8 @@ define i32 @LdOffset_i16_sext32(ptr %a)  {
 define i64 @LdOffset_i16_zext64(ptr %a)  {
 ; CHECK-LABEL: LdOffset_i16_zext64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #48368 // =0xbcf0
-; CHECK-NEXT:    movk w8, #31, lsl #16
-; CHECK-NEXT:    ldrh w0, [x0, x8]
+; CHECK-NEXT:    add x8, x0, #506, lsl #12 // =2072576
+; CHECK-NEXT:    ldrh w0, [x8, #7408]
 ; CHECK-NEXT:    ret
   %arrayidx = getelementptr inbounds i16, ptr %a, i64 1039992
   %val = load i16, ptr %arrayidx, align 2
@@ -335,9 +329,8 @@ define i64 @LdOffset_i16_zext64(ptr %a)  {
 define i64 @LdOffset_i16_sext64(ptr %a)  {
 ; CHECK-LABEL: LdOffset_i16_sext64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #48368 // =0xbcf0
-; CHECK-NEXT:    movk w8, #31, lsl #16
-; CHECK-NEXT:    ldrsh x0, [x0, x8]
+; CHECK-NEXT:    add x8, x0, #506, lsl #12 // =2072576
+; CHECK-NEXT:    ldrsh x0, [x8, #7408]
 ; CHECK-NEXT:    ret
   %arrayidx = getelementptr inbounds i16, ptr %a, i64 1039992
   %val = load i16, ptr %arrayidx, align 2
@@ -349,9 +342,8 @@ define i64 @LdOffset_i16_sext64(ptr %a)  {
 define i32 @LdOffset_i32(ptr %a)  {
 ; CHECK-LABEL: LdOffset_i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #31200 // =0x79e0
-; CHECK-NEXT:    movk w8, #63, lsl #16
-; CHECK-NEXT:    ldr w0, [x0, x8]
+; CHECK-NEXT:    add x8, x0, #1012, lsl #12 // =4145152
+; CHECK-NEXT:    ldr w0, [x8, #14816]
 ; CHECK-NEXT:    ret
   %arrayidx = getelementptr inbounds i32, ptr %a, i64 1039992
   %val = load i32, ptr %arrayidx, align 4
@@ -362,9 +354,8 @@ define i32 @LdOffset_i32(ptr %a)  {
 define i64 @LdOffset_i32_zext64(ptr %a)  {
 ; CHECK-LABEL: LdOffset_i32_zext64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #31200 // =0x79e0
-; CHECK-NEXT:    movk w8, #63, lsl #16
-; CHECK-NEXT:    ldr w0, [x0, x8]
+; CHECK-NEXT:    add x8, x0, #1012, lsl #12 // =4145152
+; CHECK-NEXT:    ldr w0, [x8, #14816]
 ; CHECK-NEXT:    ret
   %arrayidx = getelementptr inbounds i32, ptr %a, i64 1039992
   %val = load i32, ptr %arrayidx, align 2
@@ -376,9 +367,8 @@ define i64 @LdOffset_i32_zext64(ptr %a)  {
 define i64 @LdOffset_i32_sext64(ptr %a)  {
 ; CHECK-LABEL: LdOffset_i32_sext64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #31200 // =0x79e0
-; CHECK-NEXT:    movk w8, #63, lsl #16
-; CHECK-NEXT:    ldrsw x0, [x0, x8]
+; CHECK-NEXT:    add x8, x0, #1012, lsl #12 // =4145152
+; CHECK-NEXT:    ldrsw x0, [x8, #14816]
 ; CHECK-NEXT:    ret
   %arrayidx = getelementptr inbounds i32, ptr %a, i64 1039992
   %val = load i32, ptr %arrayidx, align 2
@@ -390,9 +380,8 @@ define i64 @LdOffset_i32_sext64(ptr %a)  {
 define i64 @LdOffset_i64(ptr %a)  {
 ; CHECK-LABEL: LdOffset_i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #62400 // =0xf3c0
-; CHECK-NEXT:    movk w8, #126, lsl #16
-; CHECK-NEXT:    ldr x0, [x0, x8]
+; CHECK-NEXT:    add x8, x0, #2024, lsl #12 // =8290304
+; CHECK-NEXT:    ldr x0, [x8, #29632]
 ; CHECK-NEXT:    ret
   %arrayidx = getelementptr inbounds i64, ptr %a, i64 1039992
   %val = load i64, ptr %arrayidx, align 4
@@ -403,9 +392,8 @@ define i64 @LdOffset_i64(ptr %a)  {
 define <2 x i32> @LdOffset_v2i32(ptr %a)  {
 ; CHECK-LABEL: LdOffset_v2i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #62400 // =0xf3c0
-; CHECK-NEXT:    movk w8, #126, lsl #16
-; CHECK-NEXT:    ldr d0, [x0, x8]
+; CHECK-NEXT:    add x8, x0, #2024, lsl #12 // =8290304
+; CHECK-NEXT:    ldr d0, [x8, #29632]
 ; CHECK-NEXT:    ret
   %arrayidx = getelementptr inbounds <2 x i32>, ptr %a, i64 1039992
   %val = load <2 x i32>, ptr %arrayidx, align 4
@@ -416,9 +404,8 @@ define <2 x i32> @LdOffset_v2i32(ptr %a)  {
 define <2 x i64> @LdOffset_v2i64(ptr %a)  {
 ; CHECK-LABEL: LdOffset_v2i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #59264 // =0xe780
-; CHECK-NEXT:    movk w8, #253, lsl #16
-; CHECK-NEXT:    ldr q0, [x0, x8]
+; CHECK-NEXT:    add x8, x0, #4048, lsl #12 // =16580608
+; CHECK-NEXT:    ldr q0, [x8, #59264]
 ; CHECK-NEXT:    ret
   %arrayidx = getelementptr inbounds <2 x i64>, ptr %a, i64 1039992
   %val = load <2 x i64>, ptr %arrayidx, align 4
@@ -429,9 +416,8 @@ define <2 x i64> @LdOffset_v2i64(ptr %a)  {
 define double @LdOffset_i8_f64(ptr %a)  {
 ; CHECK-LABEL: LdOffset_i8_f64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #56952 // =0xde78
-; CHECK-NEXT:    movk w8, #15, lsl #16
-; CHECK-NEXT:    ldrsb w8, [x0, x8]
+; CHECK-NEXT:    add x8, x0, #253, lsl #12 // =1036288
+; CHECK-NEXT:    ldrsb w8, [x8, #3704]
 ; CHECK-NEXT:    scvtf d0, w8
 ; CHECK-NEXT:    ret
   %arrayidx = getelementptr inbounds i8, ptr %a, i64 1039992
@@ -444,9 +430,8 @@ define double @LdOffset_i8_f64(ptr %a)  {
 define double @LdOffset_i16_f64(ptr %a)  {
 ; CHECK-LABEL: LdOffset_i16_f64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #48368 // =0xbcf0
-; CHECK-NEXT:    movk w8, #31, lsl #16
-; CHECK-NEXT:    ldrsh w8, [x0, x8]
+; CHECK-NEXT:    add x8, x0, #506, lsl #12 // =2072576
+; CHECK-NEXT:    ldrsh w8, [x8, #7408]
 ; CHECK-NEXT:    scvtf d0, w8
 ; CHECK-NEXT:    ret
   %arrayidx = getelementptr inbounds i16, ptr %a, i64 1039992
@@ -459,9 +444,8 @@ define double @LdOffset_i16_f64(ptr %a)  {
 define double @LdOffset_i32_f64(ptr %a)  {
 ; CHECK-LABEL: LdOffset_i32_f64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #31200 // =0x79e0
-; CHECK-NEXT:    movk w8, #63, lsl #16
-; CHECK-NEXT:    ldr s0, [x0, x8]
+; CHECK-NEXT:    add x8, x0, #1012, lsl #12 // =4145152
+; CHECK-NEXT:    ldr s0, [x8, #14816]
 ; CHECK-NEXT:    ucvtf d0, d0
 ; CHECK-NEXT:    ret
   %arrayidx = getelementptr inbounds i32, ptr %a, i64 1039992
@@ -474,9 +458,8 @@ define double @LdOffset_i32_f64(ptr %a)  {
 define double @LdOffset_i64_f64(ptr %a)  {
 ; CHECK-LABEL: LdOffset_i64_f64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #62400 // =0xf3c0
-; CHECK-NEXT:    movk w8, #126, lsl #16
-; CHECK-NEXT:    ldr d0, [x0, x8]
+; CHECK-NEXT:    add x8, x0, #2024, lsl #12 // =8290304
+; CHECK-NEXT:    ldr d0, [x8, #29632]
 ; CHECK-NEXT:    scvtf d0, d0
 ; CHECK-NEXT:    ret
   %arrayidx = getelementptr inbounds i64, ptr %a, i64 1039992

From 79658d65c3c7a075382b74d81e74714e2ea9bd2d Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Thu, 15 Aug 2024 14:42:08 +0400
Subject: [PATCH 008/441] InferAddressSpaces: Make getPredicatedAddrSpace less
 confusing (#104052)

This takes a pointer value and the user instruction. Name them as
such, and remove the null check which should be dead.
---
 .../Transforms/Scalar/InferAddressSpaces.cpp  | 20 +++++++++----------
 1 file changed, 9 insertions(+), 11 deletions(-)

diff --git a/llvm/lib/Transforms/Scalar/InferAddressSpaces.cpp b/llvm/lib/Transforms/Scalar/InferAddressSpaces.cpp
index a5571409bba6824..0c8aee8a494c031 100644
--- a/llvm/lib/Transforms/Scalar/InferAddressSpaces.cpp
+++ b/llvm/lib/Transforms/Scalar/InferAddressSpaces.cpp
@@ -240,7 +240,8 @@ class InferAddressSpacesImpl {
       SmallVectorImpl<const Use *> *PoisonUsesToFix) const;
   unsigned joinAddressSpaces(unsigned AS1, unsigned AS2) const;
 
-  unsigned getPredicatedAddrSpace(const Value &V, Value *Opnd) const;
+  unsigned getPredicatedAddrSpace(const Value &PtrV,
+                                  const Instruction *UserCtxI) const;
 
 public:
   InferAddressSpacesImpl(AssumptionCache &AC, const DominatorTree *DT,
@@ -909,18 +910,14 @@ void InferAddressSpacesImpl::inferAddressSpaces(
   }
 }
 
-unsigned InferAddressSpacesImpl::getPredicatedAddrSpace(const Value &V,
-                                                        Value *Opnd) const {
-  const Instruction *I = dyn_cast<Instruction>(&V);
-  if (!I)
-    return UninitializedAddressSpace;
-
-  Opnd = Opnd->stripInBoundsOffsets();
-  for (auto &AssumeVH : AC.assumptionsFor(Opnd)) {
+unsigned InferAddressSpacesImpl::getPredicatedAddrSpace(
+    const Value &Ptr, const Instruction *UserCtxI) const {
+  const Value *StrippedPtr = Ptr.stripInBoundsOffsets();
+  for (auto &AssumeVH : AC.assumptionsFor(StrippedPtr)) {
     if (!AssumeVH)
       continue;
     CallInst *CI = cast<CallInst>(AssumeVH);
-    if (!isValidAssumeForContext(CI, I, DT))
+    if (!isValidAssumeForContext(CI, UserCtxI, DT))
       continue;
 
     const Value *Ptr;
@@ -989,7 +986,8 @@ bool InferAddressSpacesImpl::updateAddressSpace(
           OperandAS = PtrOperand->getType()->getPointerAddressSpace();
           if (OperandAS == FlatAddrSpace) {
             // Check AC for assumption dominating V.
-            unsigned AS = getPredicatedAddrSpace(V, PtrOperand);
+            unsigned AS =
+                getPredicatedAddrSpace(*PtrOperand, &cast<Instruction>(V));
             if (AS != UninitializedAddressSpace) {
               LLVM_DEBUG(dbgs()
                          << "  deduce operand AS from the predicate addrspace "

From 100c9c019cebf49427d9f3ea93db65f7e448a102 Mon Sep 17 00:00:00 2001
From: Sergei Barannikov <barannikov88@gmail.com>
Date: Thu, 15 Aug 2024 13:52:25 +0300
Subject: [PATCH 009/441] [DataLayout] Add helper predicates to sort
 specifications (NFC) (#104417)

---
 llvm/lib/IR/DataLayout.cpp | 47 +++++++++++++++++++-------------------
 1 file changed, 24 insertions(+), 23 deletions(-)

diff --git a/llvm/lib/IR/DataLayout.cpp b/llvm/lib/IR/DataLayout.cpp
index 44cd1e69818953d..5f9a0fbc61905fc 100644
--- a/llvm/lib/IR/DataLayout.cpp
+++ b/llvm/lib/IR/DataLayout.cpp
@@ -153,6 +153,24 @@ bool DataLayout::PointerSpec::operator==(const PointerSpec &Other) const {
          IndexBitWidth == Other.IndexBitWidth;
 }
 
+namespace {
+/// Predicate to sort primitive specs by bit width.
+struct LessPrimitiveBitWidth {
+  bool operator()(const DataLayout::PrimitiveSpec &LHS,
+                  unsigned RHSBitWidth) const {
+    return LHS.BitWidth < RHSBitWidth;
+  }
+};
+
+/// Predicate to sort pointer specs by address space number.
+struct LessPointerAddrSpace {
+  bool operator()(const DataLayout::PointerSpec &LHS,
+                  unsigned RHSAddrSpace) const {
+    return LHS.AddrSpace < RHSAddrSpace;
+  }
+};
+} // namespace
+
 const char *DataLayout::getManglingComponent(const Triple &T) {
   if (T.isOSBinFormatGOFF())
     return "-m:l";
@@ -581,15 +599,6 @@ Error DataLayout::parseSpecifier(StringRef Desc) {
   return Error::success();
 }
 
-static SmallVectorImpl<DataLayout::PrimitiveSpec>::const_iterator
-findPrimitiveSpecLowerBound(
-    const SmallVectorImpl<DataLayout::PrimitiveSpec> &Specs,
-    uint32_t BitWidth) {
-  return partition_point(Specs, [BitWidth](const DataLayout::PrimitiveSpec &E) {
-    return E.BitWidth < BitWidth;
-  });
-}
-
 Error DataLayout::setPrimitiveSpec(TypeSpecifier Specifier, uint32_t BitWidth,
                                    Align ABIAlign, Align PrefAlign) {
   // AlignmentsTy::ABIAlign and AlignmentsTy::PrefAlign were once stored as
@@ -620,9 +629,7 @@ Error DataLayout::setPrimitiveSpec(TypeSpecifier Specifier, uint32_t BitWidth,
     break;
   }
 
-  auto I = partition_point(*Specs, [BitWidth](const PrimitiveSpec &E) {
-    return E.BitWidth < BitWidth;
-  });
+  auto I = lower_bound(*Specs, BitWidth, LessPrimitiveBitWidth());
   if (I != Specs->end() && I->BitWidth == BitWidth) {
     // Update the abi, preferred alignments.
     I->ABIAlign = ABIAlign;
@@ -637,10 +644,7 @@ Error DataLayout::setPrimitiveSpec(TypeSpecifier Specifier, uint32_t BitWidth,
 const DataLayout::PointerSpec &
 DataLayout::getPointerSpec(uint32_t AddrSpace) const {
   if (AddrSpace != 0) {
-    auto I = lower_bound(PointerSpecs, AddrSpace,
-                         [](const PointerSpec &Spec, uint32_t AddrSpace) {
-                           return Spec.AddrSpace < AddrSpace;
-                         });
+    auto I = lower_bound(PointerSpecs, AddrSpace, LessPointerAddrSpace());
     if (I != PointerSpecs.end() && I->AddrSpace == AddrSpace)
       return *I;
   }
@@ -658,10 +662,7 @@ Error DataLayout::setPointerSpec(uint32_t AddrSpace, uint32_t BitWidth,
   if (IndexBitWidth > BitWidth)
     return reportError("Index width cannot be larger than pointer width");
 
-  auto I = lower_bound(PointerSpecs, AddrSpace,
-                       [](const PointerSpec &A, uint32_t AddrSpace) {
-                         return A.AddrSpace < AddrSpace;
-                       });
+  auto I = lower_bound(PointerSpecs, AddrSpace, LessPointerAddrSpace());
   if (I == PointerSpecs.end() || I->AddrSpace != AddrSpace) {
     PointerSpecs.insert(I, PointerSpec{AddrSpace, BitWidth, ABIAlign, PrefAlign,
                                        IndexBitWidth});
@@ -676,7 +677,7 @@ Error DataLayout::setPointerSpec(uint32_t AddrSpace, uint32_t BitWidth,
 
 Align DataLayout::getIntegerAlignment(uint32_t BitWidth,
                                       bool abi_or_pref) const {
-  auto I = findPrimitiveSpecLowerBound(IntSpecs, BitWidth);
+  auto I = lower_bound(IntSpecs, BitWidth, LessPrimitiveBitWidth());
   // If we don't have an exact match, use alignment of next larger integer
   // type. If there is none, use alignment of largest integer type by going
   // back one element.
@@ -792,7 +793,7 @@ Align DataLayout::getAlignment(Type *Ty, bool abi_or_pref) const {
   case Type::FP128TyID:
   case Type::X86_FP80TyID: {
     unsigned BitWidth = getTypeSizeInBits(Ty).getFixedValue();
-    auto I = findPrimitiveSpecLowerBound(FloatSpecs, BitWidth);
+    auto I = lower_bound(FloatSpecs, BitWidth, LessPrimitiveBitWidth());
     if (I != FloatSpecs.end() && I->BitWidth == BitWidth)
       return abi_or_pref ? I->ABIAlign : I->PrefAlign;
 
@@ -807,7 +808,7 @@ Align DataLayout::getAlignment(Type *Ty, bool abi_or_pref) const {
   case Type::FixedVectorTyID:
   case Type::ScalableVectorTyID: {
     unsigned BitWidth = getTypeSizeInBits(Ty).getKnownMinValue();
-    auto I = findPrimitiveSpecLowerBound(VectorSpecs, BitWidth);
+    auto I = lower_bound(VectorSpecs, BitWidth, LessPrimitiveBitWidth());
     if (I != VectorSpecs.end() && I->BitWidth == BitWidth)
       return abi_or_pref ? I->ABIAlign : I->PrefAlign;
 

From 43ba1097ee747b4ec5e757762ed0c9df6255a292 Mon Sep 17 00:00:00 2001
From: Hua Tian <akiratian@tencent.com>
Date: Thu, 15 Aug 2024 19:03:27 +0800
Subject: [PATCH 010/441] [llvm][CodeGen] Resolve issues when updating live
 intervals in window scheduler (#101945)

Corrupted live interval information can cause window scheduling to crash
in some cases. By adding the missing MBB's live interval information in the
ModuloScheduleExpander, the information can be correctly analyzed in
the window scheduler.
---
 llvm/lib/CodeGen/ModuloSchedule.cpp           |   3 +
 .../CodeGen/Hexagon/swp-ws-live-intervals.mir | 217 ++++++++++++++++++
 2 files changed, 220 insertions(+)
 create mode 100644 llvm/test/CodeGen/Hexagon/swp-ws-live-intervals.mir

diff --git a/llvm/lib/CodeGen/ModuloSchedule.cpp b/llvm/lib/CodeGen/ModuloSchedule.cpp
index dae0cb2c900e5be..78201d9bfb79a97 100644
--- a/llvm/lib/CodeGen/ModuloSchedule.cpp
+++ b/llvm/lib/CodeGen/ModuloSchedule.cpp
@@ -130,6 +130,7 @@ void ModuloScheduleExpander::generatePipelinedLoop() {
   // Generate the prolog instructions that set up the pipeline.
   generateProlog(MaxStageCount, KernelBB, VRMap, PrologBBs);
   MF.insert(BB->getIterator(), KernelBB);
+  LIS.insertMBBInMaps(KernelBB);
 
   // Rearrange the instructions to generate the new, pipelined loop,
   // and update register names as needed.
@@ -210,6 +211,7 @@ void ModuloScheduleExpander::generateProlog(unsigned LastStage,
     NewBB->transferSuccessors(PredBB);
     PredBB->addSuccessor(NewBB);
     PredBB = NewBB;
+    LIS.insertMBBInMaps(NewBB);
 
     // Generate instructions for each appropriate stage. Process instructions
     // in original program order.
@@ -283,6 +285,7 @@ void ModuloScheduleExpander::generateEpilog(
 
     PredBB->replaceSuccessor(LoopExitBB, NewBB);
     NewBB->addSuccessor(LoopExitBB);
+    LIS.insertMBBInMaps(NewBB);
 
     if (EpilogStart == LoopExitBB)
       EpilogStart = NewBB;
diff --git a/llvm/test/CodeGen/Hexagon/swp-ws-live-intervals.mir b/llvm/test/CodeGen/Hexagon/swp-ws-live-intervals.mir
new file mode 100644
index 000000000000000..7fa3cdf62d09025
--- /dev/null
+++ b/llvm/test/CodeGen/Hexagon/swp-ws-live-intervals.mir
@@ -0,0 +1,217 @@
+# REQUIRES: asserts
+# 
+# RUN: llc --march=hexagon %s -run-pass=pipeliner -debug-only=pipeliner \
+# RUN: -window-sched=force -filetype=null -window-search-num=100 \
+# RUN: -window-search-ratio=100 -window-diff-limit=0 -verify-machineinstrs \
+# RUN: 2>&1 | FileCheck %s
+
+# The bug was reported at https://github.com/llvm/llvm-project/pull/99454. 
+# It is caused by the corruption of live intervals in certain scenarios.
+#
+# We check the newly generated MBBs after successful scheduling here.
+# CHECK: Best window offset is {{[0-9]+}} and Best II is {{[0-9]+}}.
+# CHECK: prolog:
+# CHECK: bb.5:
+# CHECK: New block
+# CHECK: bb.6:
+# CHECK: epilog:
+# CHECK: bb.7:
+# CHECK: Best window offset is {{[0-9]+}} and Best II is {{[0-9]+}}.
+# CHECK: prolog:
+# CHECK: bb.8:
+# CHECK: New block
+# CHECK: bb.9:
+# CHECK: epilog:
+# CHECK: bb.10:
+
+--- |
+  target triple = "hexagon"
+  
+  @_dp_ctrl_calc_tu_temp2_fp = global i64 0
+  @_dp_ctrl_calc_tu_temp1_fp = global i32 0
+  @dp_panel_update_tu_timings___trans_tmp_5 = global i64 0
+  @_dp_ctrl_calc_tu___trans_tmp_8 = global i64 0
+  
+  declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
+  declare i8 @div64_u64_rem(i32, ptr)
+  declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
+  
+  define void @dp_ctrl_calc_tu_parameters() {
+  if.end.i:
+    %rem.i11.i = alloca i64, align 8
+    %rem.i.i = alloca i64, align 8
+    call void @llvm.lifetime.start.p0(i64 8, ptr nonnull %rem.i11.i)
+    %call.i.i = call i8 @div64_u64_rem(i32 0, ptr nonnull %rem.i11.i)
+    %conv1.i.i = zext i8 %call.i.i to i64
+    %rem.promoted.i.i = load i64, ptr %rem.i11.i, align 8
+    br label %do.body.i.i
+  
+  do.body.i.i:
+    %lsr.iv1 = phi i32 [ %lsr.iv.next2, %do.body.i.i ], [ -32, %if.end.i ]
+    %sub9.i.i = phi i64 [ %rem.promoted.i.i, %if.end.i ], [ %sub8.i.i.7, %do.body.i.i ]
+    %res_abs.0.i.i = phi i64 [ %conv1.i.i, %if.end.i ], [ %res_abs.1.i.i.7, %do.body.i.i ]
+    %cmp.not.i.i = icmp ne i64 %sub9.i.i, 0
+    %sub.i.neg.i = sext i1 %cmp.not.i.i to i64
+    %sub8.i.i = add i64 %sub9.i.i, %sub.i.neg.i
+    %0 = shl i64 %res_abs.0.i.i, 2
+    %1 = select i1 %cmp.not.i.i, i64 2, i64 0
+    %shl.i.i.5 = or disjoint i64 %0, %1
+    %cmp.not.i.i.5 = icmp ne i64 %sub8.i.i, 0
+    %sub.i.neg.i.5 = sext i1 %cmp.not.i.i.5 to i64
+    %sub8.i.i.5 = add i64 %sub8.i.i, %sub.i.neg.i.5
+    %or.i.i.5 = zext i1 %cmp.not.i.i.5 to i64
+    %res_abs.1.i.i.5 = or disjoint i64 %shl.i.i.5, %or.i.i.5
+    %cmp.not.i.i.6 = icmp ne i64 %sub8.i.i.5, 0
+    %sub.i.neg.i.6 = sext i1 %cmp.not.i.i.6 to i64
+    %sub8.i.i.6 = add i64 %sub8.i.i.5, %sub.i.neg.i.6
+    %2 = shl i64 %res_abs.1.i.i.5, 2
+    %3 = select i1 %cmp.not.i.i.6, i64 2, i64 0
+    %shl.i.i.7 = or disjoint i64 %2, %3
+    %cmp.not.i.i.7 = icmp ne i64 %sub8.i.i.6, 0
+    %sub.i.neg.i.7 = sext i1 %cmp.not.i.i.7 to i64
+    %sub8.i.i.7 = add i64 %sub8.i.i.6, %sub.i.neg.i.7
+    %or.i.i.7 = zext i1 %cmp.not.i.i.7 to i64
+    %res_abs.1.i.i.7 = or disjoint i64 %shl.i.i.7, %or.i.i.7
+    %lsr.iv.next2 = add nsw i32 %lsr.iv1, 8
+    %tobool.not.i.i.7 = icmp eq i32 %lsr.iv.next2, 0
+    br i1 %tobool.not.i.i.7, label %fec_check.i, label %do.body.i.i
+  
+  fec_check.i:
+    call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %rem.i11.i)
+    store i64 %res_abs.1.i.i.7, ptr @_dp_ctrl_calc_tu_temp2_fp, align 8
+    call void @llvm.lifetime.start.p0(i64 8, ptr nonnull %rem.i11.i)
+    %call.i12.i = call i8 @div64_u64_rem(i32 0, ptr nonnull %rem.i11.i)
+    %conv1.i13.i = zext i8 %call.i12.i to i64
+    %rem.promoted.i14.i = load i64, ptr %rem.i11.i, align 8
+    br label %do.body.i15.i
+  
+  do.body.i15.i:
+    %lsr.iv = phi i32 [ %lsr.iv.next, %do.body.i15.i ], [ -32, %fec_check.i ]
+    %sub9.i16.i = phi i64 [ %rem.promoted.i14.i, %fec_check.i ], [ %sub8.i22.i.7, %do.body.i15.i ]
+    %res_abs.0.i17.i = phi i64 [ %conv1.i13.i, %fec_check.i ], [ %res_abs.1.i24.i.7, %do.body.i15.i ]
+    %cmp.not.i20.i = icmp ugt i64 %sub9.i16.i, 999
+    %sub.i21.neg.i = select i1 %cmp.not.i20.i, i64 -1000, i64 0
+    %sub8.i22.i = add i64 %sub.i21.neg.i, %sub9.i16.i
+    %4 = shl i64 %res_abs.0.i17.i, 2
+    %5 = select i1 %cmp.not.i20.i, i64 2, i64 0
+    %shl.i19.i.7 = or disjoint i64 %4, %5
+    %cmp.not.i20.i.7 = icmp ugt i64 %sub8.i22.i, 999
+    %sub.i21.neg.i.7 = select i1 %cmp.not.i20.i.7, i64 -1000, i64 0
+    %sub8.i22.i.7 = add i64 %sub.i21.neg.i.7, %sub8.i22.i
+    %or.i23.i.7 = zext i1 %cmp.not.i20.i.7 to i64
+    %res_abs.1.i24.i.7 = or disjoint i64 %shl.i19.i.7, %or.i23.i.7
+    %lsr.iv.next = add nsw i32 %lsr.iv, 8
+    %tobool.not.i26.i.7 = icmp eq i32 %lsr.iv.next, 0
+    br i1 %tobool.not.i26.i.7, label %_dp_ctrl_calc_tu.exit, label %do.body.i15.i
+  
+  _dp_ctrl_calc_tu.exit:
+    call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %rem.i11.i)
+    %conv.i = trunc i64 %res_abs.1.i24.i.7 to i32
+    store i32 %conv.i, ptr @_dp_ctrl_calc_tu_temp1_fp, align 4
+    %conv5.i = and i64 %res_abs.1.i24.i.7, 4294967295
+    store i64 %conv5.i, ptr @dp_panel_update_tu_timings___trans_tmp_5, align 8
+    store i64 %res_abs.1.i.i.7, ptr @_dp_ctrl_calc_tu___trans_tmp_8, align 8
+    ret void
+  }
+
+...
+---
+name:            dp_ctrl_calc_tu_parameters
+tracksRegLiveness: true
+stack:
+  - { id: 0, name: rem.i11.i, type: default, offset: 0, size: 8, alignment: 8}
+body:             |
+  bb.0:
+    successors: %bb.1(0x80000000)
+  
+    %0:intregs = A2_tfrsi 0
+    %1:intregs = PS_fi %stack.0.rem.i11.i, 0
+    %2:intregs = A2_tfrsi 0
+    %3:doubleregs = A4_combineir 0, %2
+    %4:doubleregs = L2_loadrd_io %stack.0.rem.i11.i, 0
+    %5:doubleregs = A2_tfrpi 0
+    J2_loop0i %bb.1, 4, implicit-def $lc0, implicit-def $sa0, implicit-def $usr
+  
+  bb.1 (machine-block-address-taken):
+    successors: %bb.2(0x04000000), %bb.1(0x7c000000)
+  
+    %6:doubleregs = PHI %4, %bb.0, %7, %bb.1
+    %8:doubleregs = PHI %3, %bb.0, %9, %bb.1
+    %10:predregs = C2_cmpeqp %6, %5
+    %11:intregs = C2_muxii %10, 0, -1
+    %12:doubleregs = A2_addsp %11, %6
+    %13:doubleregs = S2_asl_i_p %8, 2
+    %14:intregs = S2_setbit_i %13.isub_lo, 1
+    %15:intregs = C2_mux %10, %13.isub_lo, %14
+    %16:predregs = C2_cmpeqp %12, %5
+    %17:intregs = C2_muxii %16, 0, -1
+    %18:doubleregs = A2_addsp %17, %12
+    %19:intregs = S2_setbit_i %15, 0
+    %20:intregs = C2_mux %16, %15, %19
+    %21:predregs = C2_cmpeqp %18, %5
+    %22:intregs = C2_muxii %21, 0, -1
+    %23:doubleregs = A2_addsp %22, %18
+    %24:intregs = S2_asl_i_r %20, 2
+    %25:intregs = S2_extractu %8.isub_lo, 2, 28
+    %26:intregs = S2_asl_i_r_or %25, %13.isub_hi, 2
+    %27:intregs = S2_setbit_i %24, 1
+    %28:intregs = C2_mux %21, %24, %27
+    %29:predregs = C2_cmpeqp %23, %5
+    %30:intregs = C2_muxii %29, 0, -1
+    %7:doubleregs = A2_addsp %30, %23
+    %31:intregs = S2_setbit_i %28, 0
+    %32:intregs = C2_mux %29, %28, %31
+    %9:doubleregs = REG_SEQUENCE %26, %subreg.isub_hi, %32, %subreg.isub_lo
+    ENDLOOP0 %bb.1, implicit-def $pc, implicit-def $lc0, implicit $sa0, implicit $lc0
+    J2_jump %bb.2, implicit-def dead $pc
+  
+  bb.2:
+    successors: %bb.3(0x80000000)
+  
+    S2_storerdgp @_dp_ctrl_calc_tu_temp2_fp, %9, implicit $gp
+    %33:intregs = A2_tfrsi 0
+    %34:intregs = PS_fi %stack.0.rem.i11.i, 0
+    %35:intregs = A2_tfrsi 0
+    %36:doubleregs = L2_loadrd_io %stack.0.rem.i11.i, 0
+    %37:doubleregs = A2_tfrpi 124
+    %38:intregs = A2_tfrsi -1000
+    %39:intregs = A2_tfrsi -1
+    J2_loop0i %bb.3, 4, implicit-def $lc0, implicit-def $sa0, implicit-def $usr
+  
+  bb.3 (machine-block-address-taken):
+    successors: %bb.4(0x04000000), %bb.3(0x7c000000)
+  
+    %40:doubleregs = PHI %36, %bb.2, %41, %bb.3
+    %42:intregs = PHI %35, %bb.2, %43, %bb.3
+    %44:intregs = PHI %33, %bb.2, %45, %bb.3
+    %46:doubleregs = S2_lsr_i_p %40, 3
+    %47:predregs = C2_cmpgtup %46, %37
+    %48:intregs = C2_mux %47, %38, %33
+    %49:intregs = C2_mux %47, %39, %33
+    %50:doubleregs = REG_SEQUENCE %49, %subreg.isub_hi, %48, %subreg.isub_lo
+    %51:doubleregs = A2_addp %50, %40
+    %52:intregs = S2_asl_i_r %42, 2
+    %53:intregs = S2_extractu %42, 2, 30
+    %45:intregs = S2_asl_i_r_or %53, %44, 2
+    %54:intregs = S2_setbit_i %52, 1
+    %55:intregs = C2_mux %47, %54, %52
+    %56:doubleregs = S2_lsr_i_p %51, 3
+    %57:predregs = C2_cmpgtup %56, %37
+    %58:intregs = C2_mux %57, %38, %33
+    %59:intregs = C2_mux %57, %39, %33
+    %60:doubleregs = REG_SEQUENCE %59, %subreg.isub_hi, %58, %subreg.isub_lo
+    %41:doubleregs = A2_addp %60, %51
+    %61:intregs = S2_setbit_i %55, 0
+    %43:intregs = C2_mux %57, %61, %55
+    ENDLOOP0 %bb.3, implicit-def $pc, implicit-def $lc0, implicit $sa0, implicit $lc0
+    J2_jump %bb.4, implicit-def dead $pc
+  
+  bb.4:
+    S2_storerigp @_dp_ctrl_calc_tu_temp1_fp, %43, implicit $gp
+    %62:intregs = A2_tfrsi 0
+    %63:doubleregs = REG_SEQUENCE %43, %subreg.isub_lo, %62, %subreg.isub_hi
+    S2_storerdgp @dp_panel_update_tu_timings___trans_tmp_5, %63, implicit $gp
+    S2_storerdgp @_dp_ctrl_calc_tu___trans_tmp_8, %9, implicit $gp
+    PS_jmpret $r31, implicit-def dead $pc
+
+...

From 91ffc12a820164bdebb1a388b5d7f41a6f25ce04 Mon Sep 17 00:00:00 2001
From: Timm Baeder <tbaeder@redhat.com>
Date: Thu, 15 Aug 2024 13:18:33 +0200
Subject: [PATCH 011/441] [clang][Interp] Pass callee decl to null_callee
 diagnostics (#104426)

The callee is null, not the full call expression.
---
 clang/lib/AST/Interp/Interp.h       |  4 ++--
 clang/test/AST/Interp/functions.cpp | 11 +++++++++++
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/clang/lib/AST/Interp/Interp.h b/clang/lib/AST/Interp/Interp.h
index 3eab0cfd871385a..c2d73f32f0b20c9 100644
--- a/clang/lib/AST/Interp/Interp.h
+++ b/clang/lib/AST/Interp/Interp.h
@@ -2702,9 +2702,9 @@ inline bool CallPtr(InterpState &S, CodePtr OpPC, uint32_t ArgSize,
 
   const Function *F = FuncPtr.getFunction();
   if (!F) {
-    const Expr *E = S.Current->getExpr(OpPC);
+    const auto *E = cast<CallExpr>(S.Current->getExpr(OpPC));
     S.FFDiag(E, diag::note_constexpr_null_callee)
-        << const_cast<Expr *>(E) << E->getSourceRange();
+        << const_cast<Expr *>(E->getCallee()) << E->getSourceRange();
     return false;
   }
 
diff --git a/clang/test/AST/Interp/functions.cpp b/clang/test/AST/Interp/functions.cpp
index f190262ad3ebee4..b00f59a8d8d433d 100644
--- a/clang/test/AST/Interp/functions.cpp
+++ b/clang/test/AST/Interp/functions.cpp
@@ -221,6 +221,17 @@ namespace Comparison {
   static_assert(pg == &g, "");
 }
 
+  constexpr int Double(int n) { return 2 * n; }
+  constexpr int Triple(int n) { return 3 * n; }
+  constexpr int Twice(int (*F)(int), int n) { return F(F(n)); }
+  constexpr int Quadruple(int n) { return Twice(Double, n); }
+  constexpr auto Select(int n) -> int (*)(int) {
+    return n == 2 ? &Double : n == 3 ? &Triple : n == 4 ? &Quadruple : 0;
+  }
+  constexpr int Apply(int (*F)(int), int n) { return F(n); } // both-note {{'F' evaluates to a null function pointer}}
+
+  constexpr int Invalid = Apply(Select(0), 0); // both-error {{must be initialized by a constant expression}} \
+                                               // both-note {{in call to 'Apply(nullptr, 0)'}}
 }
 
 struct F {

From 7898866065f6c9b72b5fa3e45e565baf8a5e7609 Mon Sep 17 00:00:00 2001
From: Adrian Vogelsgesang <avogelsgesang@salesforce.com>
Date: Thu, 15 Aug 2024 13:19:32 +0200
Subject: [PATCH 012/441] [lldb-dap] Expose log path in extension settings
 (#103482)

lldb-dap already supports a log file which can be enabled by setting the
`LLDBDAP_LOG` environment variable. With this commit, the log location
can be set directly through the VS-Code extension settings.

Also, this commit bumps the version number, such that the new VS Code
extension gets published to the Marketplace.
---
 lldb/tools/lldb-dap/package-lock.json   |  4 ++--
 lldb/tools/lldb-dap/package.json        |  7 ++++++-
 lldb/tools/lldb-dap/src-ts/extension.ts | 26 ++++++++++++++++++++-----
 3 files changed, 29 insertions(+), 8 deletions(-)

diff --git a/lldb/tools/lldb-dap/package-lock.json b/lldb/tools/lldb-dap/package-lock.json
index 8c70cc2d30e1443..96570e42dbfdc4b 100644
--- a/lldb/tools/lldb-dap/package-lock.json
+++ b/lldb/tools/lldb-dap/package-lock.json
@@ -1,12 +1,12 @@
 {
   "name": "lldb-dap",
-  "version": "0.2.0",
+  "version": "0.2.4",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "lldb-dap",
-      "version": "0.2.0",
+      "version": "0.2.4",
       "license": "Apache 2.0 License with LLVM exceptions",
       "devDependencies": {
         "@types/node": "^18.11.18",
diff --git a/lldb/tools/lldb-dap/package.json b/lldb/tools/lldb-dap/package.json
index 97e4efe7bac19d3..4f4261d1718c018 100644
--- a/lldb/tools/lldb-dap/package.json
+++ b/lldb/tools/lldb-dap/package.json
@@ -1,7 +1,7 @@
 {
   "name": "lldb-dap",
   "displayName": "LLDB DAP",
-  "version": "0.2.3",
+  "version": "0.2.4",
   "publisher": "llvm-vs-code-extensions",
   "homepage": "https://lldb.llvm.org",
   "description": "LLDB debugging from VSCode",
@@ -73,6 +73,11 @@
           "scope": "resource",
           "type": "string",
           "description": "The path to the lldb-dap binary."
+        },
+        "lldb-dap.log-path": {
+          "scope": "resource",
+          "type": "string",
+          "description": "The log path for lldb-dap (if any)"
         }
       }
     },
diff --git a/lldb/tools/lldb-dap/src-ts/extension.ts b/lldb/tools/lldb-dap/src-ts/extension.ts
index 791175f7b46224e..7df09f7a29dad71 100644
--- a/lldb/tools/lldb-dap/src-ts/extension.ts
+++ b/lldb/tools/lldb-dap/src-ts/extension.ts
@@ -14,13 +14,29 @@ function createDefaultLLDBDapOptions(): LLDBDapOptions {
       session: vscode.DebugSession,
       packageJSONExecutable: vscode.DebugAdapterExecutable | undefined,
     ): Promise<vscode.DebugAdapterExecutable | undefined> {
-      const path = vscode.workspace
-        .getConfiguration("lldb-dap", session.workspaceFolder)
-        .get<string>("executable-path");
+      const config = vscode.workspace
+        .getConfiguration("lldb-dap", session.workspaceFolder);
+      const path = config.get<string>("executable-path");
+      const log_path = config.get<string>("log-path");
+
+      let env : { [key: string]: string } = {};
+      if (log_path) {
+        env["LLDBDAP_LOG"] = log_path;
+      }
+
       if (path) {
-        return new vscode.DebugAdapterExecutable(path, []);
+        return new vscode.DebugAdapterExecutable(path, [], {env});
+      } else if (packageJSONExecutable) {
+        return new vscode.DebugAdapterExecutable(packageJSONExecutable.command, packageJSONExecutable.args, {
+          ...packageJSONExecutable.options,
+          env: {
+            ...packageJSONExecutable.options?.env,
+            ...env
+          }
+        });
+      } else {
+        return undefined;
       }
-      return packageJSONExecutable;
     },
   };
 }

From 57a19ac3365f1dc255e6f24fcb7afcde2ccef8f9 Mon Sep 17 00:00:00 2001
From: Pavel Labath <pavel@labath.sk>
Date: Thu, 15 Aug 2024 13:34:03 +0200
Subject: [PATCH 013/441] Reapply "[lldb] Tolerate multiple compile units with
 the same DWO ID (#100577)" (#104041)

The only change vs. the first version of the patch is that I've made
DWARFUnit linking thread-safe/unit. This was necessary because, during
the
indexing step, two skeleton units could attempt to associate themselves
with the split unit.

The original commit message was:

I ran into this when LTO completely emptied two compile units, so they
ended up with the same hash (see #100375). Although, ideally, the
compiler would try to ensure we don't end up with a hash collision even
in this case, guaranteeing their absence is practically impossible. This
patch ensures this situation does not bring down lldb.
---
 .../Plugins/SymbolFile/DWARF/DWARFUnit.cpp    |  41 ++---
 .../Plugins/SymbolFile/DWARF/DWARFUnit.h      |   4 +-
 .../SymbolFile/DWARF/x86/dwp-hash-collision.s | 142 ++++++++++++++++++
 3 files changed, 168 insertions(+), 19 deletions(-)
 create mode 100644 lldb/test/Shell/SymbolFile/DWARF/x86/dwp-hash-collision.s

diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.cpp
index 66a762bf9b68542..81f937762e35a65 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.cpp
@@ -97,12 +97,14 @@ void DWARFUnit::ExtractUnitDIEIfNeeded() {
         *m_dwo_id, m_first_die.GetOffset()));
     return; // Can't fetch the compile unit from the dwo file.
   }
-  // If the skeleton compile unit gets its unit DIE parsed first, then this
-  // will fill in the DWO file's back pointer to this skeleton compile unit.
-  // If the DWO files get parsed on their own first the skeleton back link
-  // can be done manually in DWARFUnit::GetSkeletonCompileUnit() which will
-  // do a reverse lookup and cache the result.
-  dwo_cu->SetSkeletonUnit(this);
+
+  // Link the DWO unit to this object, if it hasn't been linked already (this
+  // can happen when we have an index, and the DWO unit is parsed first).
+  if (!dwo_cu->LinkToSkeletonUnit(*this)) {
+    SetDwoError(Status::createWithFormat(
+        "multiple compile units with Dwo ID {0:x16}", *m_dwo_id));
+    return;
+  }
 
   DWARFBaseDIE dwo_cu_die = dwo_cu->GetUnitDIEOnly();
   if (!dwo_cu_die.IsValid()) {
@@ -708,23 +710,28 @@ uint8_t DWARFUnit::GetAddressByteSize(const DWARFUnit *cu) {
 uint8_t DWARFUnit::GetDefaultAddressSize() { return 4; }
 
 DWARFCompileUnit *DWARFUnit::GetSkeletonUnit() {
-  if (m_skeleton_unit == nullptr && IsDWOUnit()) {
+  if (m_skeleton_unit.load() == nullptr && IsDWOUnit()) {
     SymbolFileDWARFDwo *dwo =
         llvm::dyn_cast_or_null<SymbolFileDWARFDwo>(&GetSymbolFileDWARF());
     // Do a reverse lookup if the skeleton compile unit wasn't set.
-    if (dwo)
-      m_skeleton_unit = dwo->GetBaseSymbolFile().GetSkeletonUnit(this);
+    DWARFUnit *candidate_skeleton_unit =
+        dwo ? dwo->GetBaseSymbolFile().GetSkeletonUnit(this) : nullptr;
+    if (candidate_skeleton_unit)
+      (void)LinkToSkeletonUnit(*candidate_skeleton_unit);
+    // Linking may fail due to a race, so be sure to return the actual value.
   }
-  return llvm::dyn_cast_or_null<DWARFCompileUnit>(m_skeleton_unit);
+  return llvm::dyn_cast_or_null<DWARFCompileUnit>(m_skeleton_unit.load());
 }
 
-void DWARFUnit::SetSkeletonUnit(DWARFUnit *skeleton_unit) {
-  // If someone is re-setting the skeleton compile unit backlink, make sure
-  // it is setting it to a valid value when it wasn't valid, or if the
-  // value in m_skeleton_unit was valid, it should be the same value.
-  assert(skeleton_unit);
-  assert(m_skeleton_unit == nullptr || m_skeleton_unit == skeleton_unit);
-  m_skeleton_unit = skeleton_unit;
+bool DWARFUnit::LinkToSkeletonUnit(DWARFUnit &skeleton_unit) {
+  DWARFUnit *expected_unit = nullptr;
+  if (m_skeleton_unit.compare_exchange_strong(expected_unit, &skeleton_unit))
+    return true;
+  if (expected_unit == &skeleton_unit) {
+    // Exchange failed because it already contained the right  value.
+    return true;
+  }
+  return false; // Already linked to a different unit.
 }
 
 bool DWARFUnit::Supports_DW_AT_APPLE_objc_complete_type() {
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.h b/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.h
index 85c37971ced8e0f..148932d67b908c2 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.h
@@ -170,7 +170,7 @@ class DWARFUnit : public UserID {
   /// both cases correctly and avoids crashes.
   DWARFCompileUnit *GetSkeletonUnit();
 
-  void SetSkeletonUnit(DWARFUnit *skeleton_unit);
+  bool LinkToSkeletonUnit(DWARFUnit &skeleton_unit);
 
   bool Supports_DW_AT_APPLE_objc_complete_type();
 
@@ -308,7 +308,7 @@ class DWARFUnit : public UserID {
   const llvm::DWARFAbbreviationDeclarationSet *m_abbrevs = nullptr;
   lldb_private::CompileUnit *m_lldb_cu = nullptr;
   // If this is a DWO file, we have a backlink to our skeleton compile unit.
-  DWARFUnit *m_skeleton_unit = nullptr;
+  std::atomic<DWARFUnit *> m_skeleton_unit = nullptr;
   // The compile unit debug information entry item
   DWARFDebugInfoEntry::collection m_die_array;
   mutable llvm::sys::RWMutex m_die_array_mutex;
diff --git a/lldb/test/Shell/SymbolFile/DWARF/x86/dwp-hash-collision.s b/lldb/test/Shell/SymbolFile/DWARF/x86/dwp-hash-collision.s
new file mode 100644
index 000000000000000..d626b4602ad58f4
--- /dev/null
+++ b/lldb/test/Shell/SymbolFile/DWARF/x86/dwp-hash-collision.s
@@ -0,0 +1,142 @@
+## Test that lldb handles (mainly, that it doesn't crash) the situation where
+## two skeleton compile units have the same DWO ID (and try to claim the same
+## split unit from the DWP file. This can sometimes happen when the compile unit
+## is nearly empty (e.g. because LTO has optimized all of it away).
+
+# RUN: llvm-mc -triple=x86_64-pc-linux -filetype=obj %s --defsym MAIN=0 > %t
+# RUN: llvm-mc -triple=x86_64-pc-linux -filetype=obj %s > %t.dwp
+# RUN: %lldb %t -o "image lookup -t my_enum_type" \
+# RUN:   -o "image dump separate-debug-info" -o exit | FileCheck %s
+
+## Check that we're able to access the type within the split unit (no matter
+## which skeleton unit it ends up associated with). Completely ignoring the unit
+## might also be reasonable.
+# CHECK: image lookup -t my_enum_type
+# CHECK: 1 match found
+# CHECK:      name = "my_enum_type", byte-size = 4, compiler_type = "enum my_enum_type {
+# CHECK-NEXT: }"
+#
+## Check that we get some indication of the error.
+# CHECK: image dump separate-debug-info
+# CHECK:      Dwo ID             Err Dwo Path
+# CHECK: 0xdeadbeefbaadf00d E   multiple compile units with Dwo ID 0xdeadbeefbaadf00d
+
+.set DWO_ID, 0xdeadbeefbaadf00d
+
+## The main file.
+.ifdef MAIN
+        .section        .debug_abbrev,"",@progbits
+        .byte   1                       # Abbreviation Code
+        .byte   74                      # DW_TAG_compile_unit
+        .byte   0                       # DW_CHILDREN_no
+        .byte   0x76                    # DW_AT_dwo_name
+        .byte   8                       # DW_FORM_string
+        .byte   0                       # EOM(1)
+        .byte   0                       # EOM(2)
+        .byte   0                       # EOM(3)
+
+
+        .section        .debug_info,"",@progbits
+.irpc I,01
+.Lcu_begin\I:
+        .long   .Ldebug_info_end\I-.Ldebug_info_start\I # Length of Unit
+.Ldebug_info_start\I:
+        .short  5                       # DWARF version number
+        .byte   4                       # DWARF Unit Type
+        .byte   8                       # Address Size (in bytes)
+        .long   .debug_abbrev           # Offset Into Abbrev. Section
+        .quad   DWO_ID                  # DWO id
+        .byte   1                       # Abbrev [1] DW_TAG_compile_unit
+        .ascii  "foo"
+        .byte   '0' + \I
+        .asciz  ".dwo\0"                # DW_AT_dwo_name
+.Ldebug_info_end\I:
+.endr
+
+.else
+## DWP file starts here.
+
+        .section        .debug_abbrev.dwo,"e",@progbits
+.LAbbrevBegin:
+        .byte   1                       # Abbreviation Code
+        .byte   17                      # DW_TAG_compile_unit
+        .byte   1                       # DW_CHILDREN_yes
+        .byte   37                      # DW_AT_producer
+        .byte   8                       # DW_FORM_string
+        .byte   19                      # DW_AT_language
+        .byte   5                       # DW_FORM_data2
+        .byte   0                       # EOM(1)
+        .byte   0                       # EOM(2)
+        .byte   2                       # Abbreviation Code
+        .byte   4                       # DW_TAG_enumeration_type
+        .byte   0                       # DW_CHILDREN_no
+        .byte   3                       # DW_AT_name
+        .byte   8                       # DW_FORM_string
+        .byte   73                      # DW_AT_type
+        .byte   19                      # DW_FORM_ref4
+        .byte   11                      # DW_AT_byte_size
+        .byte   11                      # DW_FORM_data1
+        .byte   0                       # EOM(1)
+        .byte   0                       # EOM(2)
+        .byte   4                       # Abbreviation Code
+        .byte   36                      # DW_TAG_base_type
+        .byte   0                       # DW_CHILDREN_no
+        .byte   3                       # DW_AT_name
+        .byte   8                       # DW_FORM_string
+        .byte   62                      # DW_AT_encoding
+        .byte   11                      # DW_FORM_data1
+        .byte   11                      # DW_AT_byte_size
+        .byte   11                      # DW_FORM_data1
+        .byte   0                       # EOM(1)
+        .byte   0                       # EOM(2)
+        .byte   0                       # EOM(3)
+.LAbbrevEnd:
+        .section        .debug_info.dwo,"e",@progbits
+.LCUBegin:
+.Lcu_begin1:
+        .long   .Ldebug_info_end1-.Ldebug_info_start1 # Length of Unit
+.Ldebug_info_start1:
+        .short  5                       # DWARF version number
+        .byte   5                       # DWARF Unit Type
+        .byte   8                       # Address Size (in bytes)
+        .long   0                       # Offset Into Abbrev. Section
+        .quad   DWO_ID                  # DWO id
+        .byte   1                       # Abbrev [1] DW_TAG_compile_unit
+        .asciz  "Hand-written DWARF"    # DW_AT_producer
+        .short  12                      # DW_AT_language
+        .byte   2                       # Abbrev [2] DW_TAG_enumeration_type
+        .asciz  "my_enum_type"          # DW_AT_name
+        .long   .Lint-.Lcu_begin1       # DW_AT_type
+        .byte   4                       # DW_AT_byte_size
+.Lint:
+        .byte   4                       # Abbrev [4] DW_TAG_base_type
+        .asciz  "int"                   # DW_AT_name
+        .byte   5                       # DW_AT_encoding
+        .byte   4                       # DW_AT_byte_size
+        .byte   0                       # End Of Children Mark
+.Ldebug_info_end1:
+.LCUEnd:
+        .section .debug_cu_index, "", @progbits
+## Header:
+        .short 5                        # Version
+        .short 0                        # Padding
+        .long 2                         # Section count
+        .long 1                         # Unit count
+        .long 2                         # Slot count
+## Hash Table of Signatures:
+        .quad 0
+        .quad DWO_ID
+## Parallel Table of Indexes:
+        .long 0
+        .long 1
+## Table of Section Offsets:
+## Row 0:
+        .long 1                         # DW_SECT_INFO
+        .long 3                         # DW_SECT_ABBREV
+## Row 1:
+        .long 0                         # Offset in .debug_info.dwo
+        .long 0                         # Offset in .debug_abbrev.dwo
+## Table of Section Sizes:
+        .long .LCUEnd-.LCUBegin         # Size in .debug_info.dwo
+        .long .LAbbrevEnd-.LAbbrevBegin # Size in .debug_abbrev.dwo
+.endif

From 2e9f3f3b842538b55552aa22fdc0bf1966637ca8 Mon Sep 17 00:00:00 2001
From: Fraser Cormack <fraser@codeplay.com>
Date: Thu, 15 Aug 2024 12:54:26 +0100
Subject: [PATCH 014/441] [AMDGPU][llvm-split] Fix another division by zero
 (#104421)

Somehow I missed this in #98888. It requires a log file, or the debug
flag to be passed.
---
 llvm/lib/Target/AMDGPU/AMDGPUSplitModule.cpp         |  3 ++-
 .../tools/llvm-split/AMDGPU/declarations-debug.ll    | 12 ++++++++++++
 2 files changed, 14 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/tools/llvm-split/AMDGPU/declarations-debug.ll

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSplitModule.cpp b/llvm/lib/Target/AMDGPU/AMDGPUSplitModule.cpp
index bd0f0e048809bcd..df084cf41c47836 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUSplitModule.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUSplitModule.cpp
@@ -568,12 +568,13 @@ doPartitioning(SplitModuleLogger &SML, Module &M, unsigned NumParts,
   }
 
   if (SML) {
+    CostType ModuleCostOr1 = ModuleCost ? ModuleCost : 1;
     for (const auto &[Idx, Part] : enumerate(Partitions)) {
       CostType Cost = 0;
       for (auto *Fn : Part)
         Cost += FnCosts.at(Fn);
       SML << "P" << Idx << " has a total cost of " << Cost << " ("
-          << format("%0.2f", (float(Cost) / ModuleCost) * 100)
+          << format("%0.2f", (float(Cost) / ModuleCostOr1) * 100)
           << "% of source module)\n";
     }
 
diff --git a/llvm/test/tools/llvm-split/AMDGPU/declarations-debug.ll b/llvm/test/tools/llvm-split/AMDGPU/declarations-debug.ll
new file mode 100644
index 000000000000000..66b89659f5d4f98
--- /dev/null
+++ b/llvm/test/tools/llvm-split/AMDGPU/declarations-debug.ll
@@ -0,0 +1,12 @@
+; RUN: llvm-split -o %t %s -j 2 -mtriple amdgcn-amd-amdhsa --debug
+
+; REQUIRES: asserts
+
+; CHECK: --Partitioning Starts--
+; CHECK: P0 has a total cost of 0 (0.00% of source module)
+; CHECK: P1 has a total cost of 0 (0.00% of source module)
+; CHECK: --Partitioning Done--
+
+declare void @A()
+
+declare void @B()

From 2ccbf92f878e385ab0067e2f767e39b295906a47 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Thu, 15 Aug 2024 15:53:55 +0400
Subject: [PATCH 015/441] InferAddressSpaces: Restore non-instruction user
 check

Fixes regression after 79658d65c3c7a075382b74d81e74714e2ea9bd2d.
We were missing test coverage for the nested constant expression
case.
---
 llvm/lib/Transforms/Scalar/InferAddressSpaces.cpp  | 14 +++++++++-----
 .../Transforms/InferAddressSpaces/AMDGPU/basic.ll  |  8 ++++++++
 2 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/llvm/lib/Transforms/Scalar/InferAddressSpaces.cpp b/llvm/lib/Transforms/Scalar/InferAddressSpaces.cpp
index 0c8aee8a494c031..3e3e5bfe2d63327 100644
--- a/llvm/lib/Transforms/Scalar/InferAddressSpaces.cpp
+++ b/llvm/lib/Transforms/Scalar/InferAddressSpaces.cpp
@@ -241,7 +241,7 @@ class InferAddressSpacesImpl {
   unsigned joinAddressSpaces(unsigned AS1, unsigned AS2) const;
 
   unsigned getPredicatedAddrSpace(const Value &PtrV,
-                                  const Instruction *UserCtxI) const;
+                                  const Value *UserCtx) const;
 
 public:
   InferAddressSpacesImpl(AssumptionCache &AC, const DominatorTree *DT,
@@ -910,8 +910,13 @@ void InferAddressSpacesImpl::inferAddressSpaces(
   }
 }
 
-unsigned InferAddressSpacesImpl::getPredicatedAddrSpace(
-    const Value &Ptr, const Instruction *UserCtxI) const {
+unsigned
+InferAddressSpacesImpl::getPredicatedAddrSpace(const Value &Ptr,
+                                               const Value *UserCtx) const {
+  const Instruction *UserCtxI = dyn_cast<Instruction>(UserCtx);
+  if (!UserCtxI)
+    return UninitializedAddressSpace;
+
   const Value *StrippedPtr = Ptr.stripInBoundsOffsets();
   for (auto &AssumeVH : AC.assumptionsFor(StrippedPtr)) {
     if (!AssumeVH)
@@ -986,8 +991,7 @@ bool InferAddressSpacesImpl::updateAddressSpace(
           OperandAS = PtrOperand->getType()->getPointerAddressSpace();
           if (OperandAS == FlatAddrSpace) {
             // Check AC for assumption dominating V.
-            unsigned AS =
-                getPredicatedAddrSpace(*PtrOperand, &cast<Instruction>(V));
+            unsigned AS = getPredicatedAddrSpace(*PtrOperand, &V);
             if (AS != UninitializedAddressSpace) {
               LLVM_DEBUG(dbgs()
                          << "  deduce operand AS from the predicate addrspace "
diff --git a/llvm/test/Transforms/InferAddressSpaces/AMDGPU/basic.ll b/llvm/test/Transforms/InferAddressSpaces/AMDGPU/basic.ll
index e6f26aeb98b1412..eb39684a98b5f52 100644
--- a/llvm/test/Transforms/InferAddressSpaces/AMDGPU/basic.ll
+++ b/llvm/test/Transforms/InferAddressSpaces/AMDGPU/basic.ll
@@ -190,6 +190,14 @@ define i32 @atomicrmw_add_global_to_flat_preserve_amdgpu_md(ptr addrspace(1) %gl
   ret i32 %ret
 }
 
+; Make sure there's no assert
+; CHECK-LABEL: @try_infer_getelementptr_constant_null(
+; CHECK-NEXT: %ce = getelementptr i8, ptr getelementptr inbounds (i8, ptr null, i64 8), i64 0
+define ptr @try_infer_getelementptr_constant_null() {
+  %ce = getelementptr i8, ptr getelementptr inbounds (i8, ptr null, i64 8), i64 0
+  ret ptr %ce
+}
+
 attributes #0 = { nounwind }
 
 !0 = !{}

From d2c26d82b0395b8b555be384ed778361ec176c14 Mon Sep 17 00:00:00 2001
From: Timm Baeder <tbaeder@redhat.com>
Date: Thu, 15 Aug 2024 13:57:38 +0200
Subject: [PATCH 016/441] [clang][Interp] Use first field decl for Record field
 lookup (#104412)

---
 clang/lib/AST/Interp/Program.cpp    | 1 +
 clang/lib/AST/Interp/Record.cpp     | 2 +-
 clang/test/Modules/enum-codegen.cpp | 1 +
 3 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/clang/lib/AST/Interp/Program.cpp b/clang/lib/AST/Interp/Program.cpp
index 5dd59d969853cce..9109956dc652cb2 100644
--- a/clang/lib/AST/Interp/Program.cpp
+++ b/clang/lib/AST/Interp/Program.cpp
@@ -329,6 +329,7 @@ Record *Program::getOrCreateRecord(const RecordDecl *RD) {
   // Reserve space for fields.
   Record::FieldList Fields;
   for (const FieldDecl *FD : RD->fields()) {
+    assert(FD == FD->getFirstDecl());
     // Note that we DO create fields and descriptors
     // for unnamed bitfields here, even though we later ignore
     // them everywhere. That's so the FieldDecl's getFieldIndex() matches.
diff --git a/clang/lib/AST/Interp/Record.cpp b/clang/lib/AST/Interp/Record.cpp
index ac01524e1caf0d7..e7b741f2d458c84 100644
--- a/clang/lib/AST/Interp/Record.cpp
+++ b/clang/lib/AST/Interp/Record.cpp
@@ -37,7 +37,7 @@ const std::string Record::getName() const {
 }
 
 const Record::Field *Record::getField(const FieldDecl *FD) const {
-  auto It = FieldMap.find(FD);
+  auto It = FieldMap.find(FD->getFirstDecl());
   assert(It != FieldMap.end() && "Missing field");
   return It->second;
 }
diff --git a/clang/test/Modules/enum-codegen.cpp b/clang/test/Modules/enum-codegen.cpp
index 4397b457801da4e..0028ab00c15290f 100644
--- a/clang/test/Modules/enum-codegen.cpp
+++ b/clang/test/Modules/enum-codegen.cpp
@@ -1,5 +1,6 @@
 // RUN: rm -rf %t
 // RUN: %clang_cc1 -triple %itanium_abi_triple -fmodules -fmodules-cache-path=%t %s -emit-llvm -o - | FileCheck %s
+// RUN: %clang_cc1 -triple %itanium_abi_triple -fmodules -fmodules-cache-path=%t %s -emit-llvm -o - -fexperimental-new-constant-interpreter | FileCheck %s
 
 // CHECK: @{{.*var.*}} = {{.*}} %union.union_type { i8 1 },
 

From 42555cdba48b7d6d27c9a7d5c730733e66dec9a1 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Thu, 15 Aug 2024 13:05:50 +0100
Subject: [PATCH 017/441] [VPlan] Run VPlan optimizations on plans in native
 path.

Update buildVPlans (used in native path) to also run general VPlan
optimizations in another small step to align both codepaths.
---
 llvm/lib/Transforms/Vectorize/LoopVectorize.cpp      |  4 +++-
 .../AArch64/outer_loop_prefer_scalable.ll            |  2 --
 .../Transforms/LoopVectorize/dbg-outer-loop-vect.ll  | 12 ++++--------
 .../outer-loop-vec-phi-predecessor-order.ll          |  2 --
 .../LoopVectorize/outer_loop_hcfg_construction.ll    |  4 ----
 .../Transforms/LoopVectorize/outer_loop_scalable.ll  |  2 --
 .../vplan-vectorize-inner-loop-reduction.ll          |  2 --
 .../LoopVectorize/vplan-widen-call-instruction.ll    |  2 --
 8 files changed, 7 insertions(+), 23 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index fdf8f7042c4fb86..58ef665e86d65ea 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7843,7 +7843,9 @@ void LoopVectorizationPlanner::buildVPlans(ElementCount MinVF,
   auto MaxVFTimes2 = MaxVF * 2;
   for (ElementCount VF = MinVF; ElementCount::isKnownLT(VF, MaxVFTimes2);) {
     VFRange SubRange = {VF, MaxVFTimes2};
-    VPlans.push_back(buildVPlan(SubRange));
+    auto Plan = buildVPlan(SubRange);
+    VPlanTransforms::optimize(*Plan, *PSE.getSE());
+    VPlans.push_back(std::move(Plan));
     VF = SubRange.End;
   }
 }
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_prefer_scalable.ll b/llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_prefer_scalable.ll
index 933b46733f9d1b6..736be4995c575fc 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_prefer_scalable.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_prefer_scalable.ll
@@ -49,8 +49,6 @@ define void @foo() {
 ; CHECK:       vector.latch:
 ; CHECK-NEXT:    [[VEC_PHI5:%.*]] = phi <vscale x 4 x float> [ [[TMP12]], [[INNER_LOOP1]] ]
 ; CHECK-NEXT:    call void @llvm.masked.scatter.nxv4f32.nxv4p0(<vscale x 4 x float> [[VEC_PHI5]], <vscale x 4 x ptr> [[TMP10]], i32 4, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; CHECK-NEXT:    [[TMP16:%.*]] = add nuw nsw <vscale x 4 x i64> [[VEC_IND]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 1, i64 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
-; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq <vscale x 4 x i64> [[TMP16]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 1024, i64 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP19]]
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]
 ; CHECK-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
diff --git a/llvm/test/Transforms/LoopVectorize/dbg-outer-loop-vect.ll b/llvm/test/Transforms/LoopVectorize/dbg-outer-loop-vect.ll
index 3f56361409b02f1..3bf2591391121ea 100644
--- a/llvm/test/Transforms/LoopVectorize/dbg-outer-loop-vect.ll
+++ b/llvm/test/Transforms/LoopVectorize/dbg-outer-loop-vect.ll
@@ -13,7 +13,6 @@ define void @foo(ptr %h) !dbg !4 {
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]], !dbg [[DBG21]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[FOR_COND_CLEANUP32:%.*]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[FOR_COND_CLEANUP32]] ]
 ; CHECK-NEXT:    br label [[FOR_COND5_PREHEADER1:%.*]], !dbg [[DBG21]]
 ; CHECK:       for.cond5.preheader1:
 ; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, [[VECTOR_BODY]] ], [ [[TMP4:%.*]], [[FOR_COND5_PREHEADER1]] ], !dbg [[DBG21]]
@@ -30,10 +29,7 @@ define void @foo(ptr %h) !dbg !4 {
 ; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i1> [[TMP5]], i32 0, !dbg [[DBG26:![0-9]+]]
 ; CHECK-NEXT:    br i1 [[TMP6]], label [[FOR_COND_CLEANUP32]], label [[FOR_COND5_PREHEADER1]], !dbg [[DBG26]]
 ; CHECK:       vector.latch:
-; CHECK-NEXT:    [[TMP7:%.*]] = add nuw nsw <4 x i64> [[VEC_IND]], <i64 1, i64 1, i64 1, i64 1>, !dbg [[DBG27:![0-9]+]]
-; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq <4 x i64> [[TMP7]], <i64 23, i64 23, i64 23, i64 23>, !dbg [[DBG28:![0-9]+]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], <i64 4, i64 4, i64 4, i64 4>
 ; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
 ; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
 ; CHECK:       middle.block:
@@ -59,9 +55,9 @@ define void @foo(ptr %h) !dbg !4 {
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC10]], 5, !dbg [[DBG25]]
 ; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP3]], label [[FOR_COND5_PREHEADER]], !dbg [[DBG26]]
 ; CHECK:       for.cond.cleanup3:
-; CHECK-NEXT:    [[INC13]] = add nuw nsw i64 [[I_023]], 1, !dbg [[DBG27]]
+; CHECK-NEXT:    [[INC13]] = add nuw nsw i64 [[I_023]], 1, !dbg [[DBG27:![0-9]+]]
 ; CHECK-NEXT:      #dbg_value(i64 [[INC13]], [[META11]], !DIExpression(), [[META20]])
-; CHECK-NEXT:    [[EXITCOND24_NOT:%.*]] = icmp eq i64 [[INC13]], 23, !dbg [[DBG28]]
+; CHECK-NEXT:    [[EXITCOND24_NOT:%.*]] = icmp eq i64 [[INC13]], 23, !dbg [[DBG28:![0-9]+]]
 ; CHECK-NEXT:    br i1 [[EXITCOND24_NOT]], label [[EXIT]], label [[FOR_COND1_PREHEADER]], !dbg [[DBG21]], !llvm.loop [[LOOP34:![0-9]+]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void, !dbg [[DBG35:![0-9]+]]
@@ -164,13 +160,13 @@ declare void @llvm.dbg.value(metadata, metadata, metadata)
 ; CHECK: [[DBG24]] = !DILocation(line: 11, column: 32, scope: [[META19]])
 ; CHECK: [[DBG25]] = !DILocation(line: 11, column: 26, scope: [[META19]])
 ; CHECK: [[DBG26]] = !DILocation(line: 11, column: 5, scope: [[META15]])
-; CHECK: [[DBG27]] = !DILocation(line: 10, column: 30, scope: [[META16]])
-; CHECK: [[DBG28]] = !DILocation(line: 10, column: 24, scope: [[META16]])
 ; CHECK: [[LOOP29]] = distinct !{[[LOOP29]], [[DBG21]], [[META30:![0-9]+]], [[META31:![0-9]+]], [[META32:![0-9]+]]}
 ; CHECK: [[META30]] = !DILocation(line: 13, column: 13, scope: [[META12]])
 ; CHECK: [[META31]] = !{!"llvm.loop.isvectorized", i32 1}
 ; CHECK: [[META32]] = !{!"llvm.loop.unroll.runtime.disable"}
 ; CHECK: [[DBG33]] = !DILocation(line: 13, column: 2, scope: [[META23]])
+; CHECK: [[DBG27]] = !DILocation(line: 10, column: 30, scope: [[META16]])
+; CHECK: [[DBG28]] = !DILocation(line: 10, column: 24, scope: [[META16]])
 ; CHECK: [[LOOP34]] = distinct !{[[LOOP34]], [[DBG21]], [[META30]], [[META31]]}
 ; CHECK: [[DBG35]] = !DILocation(line: 14, column: 1, scope: [[DBG4]])
 ;.
diff --git a/llvm/test/Transforms/LoopVectorize/outer-loop-vec-phi-predecessor-order.ll b/llvm/test/Transforms/LoopVectorize/outer-loop-vec-phi-predecessor-order.ll
index 260f59c5c97e274..b6f2ae3c14893a5 100644
--- a/llvm/test/Transforms/LoopVectorize/outer-loop-vec-phi-predecessor-order.ll
+++ b/llvm/test/Transforms/LoopVectorize/outer-loop-vec-phi-predecessor-order.ll
@@ -41,8 +41,6 @@ define void @test(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x i1> [[TMP6]], i32 0
 ; CHECK-NEXT:    br i1 [[TMP7]], label [[LOOP_1_LATCH5]], label [[LOOP_2_HEADER1]]
 ; CHECK:       vector.latch:
-; CHECK-NEXT:    [[TMP8:%.*]] = add nuw nsw <4 x i64> [[VEC_IND]], <i64 1, i64 1, i64 1, i64 1>
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq <4 x i64> [[TMP8]], [[BROADCAST_SPLAT]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], <i64 4, i64 4, i64 4, i64 4>
 ; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
diff --git a/llvm/test/Transforms/LoopVectorize/outer_loop_hcfg_construction.ll b/llvm/test/Transforms/LoopVectorize/outer_loop_hcfg_construction.ll
index 986a6082a6891db..5df4fce2190a3a6 100644
--- a/llvm/test/Transforms/LoopVectorize/outer_loop_hcfg_construction.ll
+++ b/llvm/test/Transforms/LoopVectorize/outer_loop_hcfg_construction.ll
@@ -52,8 +52,6 @@ define void @non_outermost_loop_hcfg_construction(i64 %n, ptr %a) {
 ; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x i1> [[TMP6]], i32 0
 ; CHECK-NEXT:    br i1 [[TMP7]], label [[MIDDLE_LOOP_LATCH4]], label [[INNERMOST_LOOP1]]
 ; CHECK:       vector.latch:
-; CHECK-NEXT:    [[TMP8:%.*]] = add nuw nsw <4 x i64> [[VEC_IND]], <i64 1, i64 1, i64 1, i64 1>
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq <4 x i64> [[TMP8]], [[BROADCAST_SPLAT3]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], <i64 4, i64 4, i64 4, i64 4>
 ; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -197,8 +195,6 @@ define void @non_outermost_loop_hcfg_construction_other_loops_at_same_level(i64
 ; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x i1> [[TMP6]], i32 0
 ; CHECK-NEXT:    br i1 [[TMP7]], label [[MIDDLE_LOOP_J0_CLEANUP4]], label [[INNERMOST_LOOP1]]
 ; CHECK:       vector.latch:
-; CHECK-NEXT:    [[TMP8:%.*]] = add nuw nsw <4 x i64> [[VEC_IND]], <i64 1, i64 1, i64 1, i64 1>
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq <4 x i64> [[TMP8]], [[BROADCAST_SPLAT3]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], <i64 4, i64 4, i64 4, i64 4>
 ; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
diff --git a/llvm/test/Transforms/LoopVectorize/outer_loop_scalable.ll b/llvm/test/Transforms/LoopVectorize/outer_loop_scalable.ll
index 6cd7e3714201b7a..08f2b823815b1ce 100644
--- a/llvm/test/Transforms/LoopVectorize/outer_loop_scalable.ll
+++ b/llvm/test/Transforms/LoopVectorize/outer_loop_scalable.ll
@@ -55,8 +55,6 @@ define void @foo() {
 ; CHECK:       vector.latch:
 ; CHECK-NEXT:    [[VEC_PHI5:%.*]] = phi <vscale x 4 x float> [ [[TMP12]], [[INNER_LOOP1]] ]
 ; CHECK-NEXT:    call void @llvm.masked.scatter.nxv4f32.nxv4p0(<vscale x 4 x float> [[VEC_PHI5]], <vscale x 4 x ptr> [[TMP10]], i32 4, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
-; CHECK-NEXT:    [[TMP16:%.*]] = add nuw nsw <vscale x 4 x i64> [[VEC_IND]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 1, i64 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
-; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq <vscale x 4 x i64> [[TMP16]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 1024, i64 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP19]]
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]
 ; CHECK-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-vectorize-inner-loop-reduction.ll b/llvm/test/Transforms/LoopVectorize/vplan-vectorize-inner-loop-reduction.ll
index fdf73963e86464a..1552af3140e3837 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-vectorize-inner-loop-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-vectorize-inner-loop-reduction.ll
@@ -40,8 +40,6 @@ define void @inner_loop_reduction(ptr noalias nocapture readonly %a.in, ptr noal
 ; CHECK-NEXT: %[[REDUCTION:.*]] = phi <4 x double> [ %[[REDUCTION_NEXT]], %[[FOR2_HEADER]] ]
 ; CHECK-NEXT: %[[C_PTR:.*]] = getelementptr inbounds double, ptr %c.out, <4 x i64> %[[VEC_INDEX]]
 ; CHECK-NEXT: call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> %[[REDUCTION]], <4 x ptr> %[[C_PTR]], i32 8, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
-; CHECK-NEXT: %[[VEC_INDEX_NEXT:.*]] = add nuw nsw <4 x i64> %[[VEC_INDEX]], <i64 1, i64 1, i64 1, i64 1>
-; CHECK-NEXT: %[[VEC_PTR:.*]] = icmp eq <4 x i64> %[[VEC_INDEX_NEXT]], <i64 1000, i64 1000, i64 1000, i64 1000>
 ; CHECK-NEXT: %[[FOR1_INDEX_NEXT:.*]] = add nuw i64 %[[FOR1_INDEX]], 4
 ; CHECK-NEXT: %{{.*}} = add <4 x i64> %[[VEC_INDEX]], <i64 4, i64 4, i64 4, i64 4>
 ; CHECK-NEXT: %[[EXIT_COND:.*]] = icmp eq i64 %[[FOR1_INDEX_NEXT]], 1000
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-widen-call-instruction.ll b/llvm/test/Transforms/LoopVectorize/vplan-widen-call-instruction.ll
index 3335c21c8d74571..4241409d3935fd6 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-widen-call-instruction.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-widen-call-instruction.ll
@@ -30,8 +30,6 @@ define void @widen_call_instruction(ptr noalias nocapture readonly %a.in, ptr no
 ; CHECK-NEXT: %[[REDUCTION:.*]] = phi <4 x double> [ %[[REDUCTION_NEXT]], %[[FOR2_HEADER]] ]
 ; CHECK-NEXT: %[[C_PTR:.*]] = getelementptr inbounds double, ptr %c.out, <4 x i64> %[[VEC_INDEX]]
 ; CHECK-NEXT: call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> %[[REDUCTION]], <4 x ptr> %[[C_PTR]], i32 8, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
-; CHECK-NEXT: %[[VEC_INDEX_NEXT:.*]] = add nuw nsw <4 x i64> %[[VEC_INDEX]], <i64 1, i64 1, i64 1, i64 1>
-; CHECK-NEXT: %[[VEC_PTR:.*]] = icmp eq <4 x i64> %[[VEC_INDEX_NEXT]], <i64 1000, i64 1000, i64 1000, i64 1000>
 ; CHECK-NEXT: %[[FOR1_INDEX_NEXT:.*]] = add nuw i64 %[[FOR1_INDEX]], 4
 ; CHECK-NEXT: %{{.*}} = add <4 x i64> %[[VEC_INDEX]], <i64 4, i64 4, i64 4, i64 4>
 ; CHECK-NEXT: %[[EXIT_COND:.*]] = icmp eq i64 %[[FOR1_INDEX_NEXT]], 1000

From f71b63865140cf3c286baf3a77ba3e467f929504 Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Thu, 15 Aug 2024 07:11:31 -0500
Subject: [PATCH 018/441] [libcxx] Use `aligned_alloc` for testing instead of
 `posix_memalign` (#101748)

Summary:
The `aligned_alloc` function is the C11 replacement for
`posix_memalign`. We should favor the C standard over the POSIX standard
so more C library implementations can run the tests.
---
 .../support.dynamic/new_faligned_allocation.pass.cpp |  2 +-
 libcxx/test/support/count_new.h                      | 12 ++++++++++++
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/libcxx/test/libcxx/language.support/support.dynamic/new_faligned_allocation.pass.cpp b/libcxx/test/libcxx/language.support/support.dynamic/new_faligned_allocation.pass.cpp
index 69c46f00fb65dc7..87f4783e12973e5 100644
--- a/libcxx/test/libcxx/language.support/support.dynamic/new_faligned_allocation.pass.cpp
+++ b/libcxx/test/libcxx/language.support/support.dynamic/new_faligned_allocation.pass.cpp
@@ -76,7 +76,7 @@ int main(int, char**) {
   test_allocations(64, 64);
   // Size being a multiple of alignment also needs to be supported.
   test_allocations(64, 32);
-  // When aligned allocation is implemented using posix_memalign,
+  // When aligned allocation is implemented using aligned_alloc,
   // that function requires a minimum alignment of sizeof(void*).
   // Check that we can also create overaligned allocations with
   // an alignment argument less than sizeof(void*).
diff --git a/libcxx/test/support/count_new.h b/libcxx/test/support/count_new.h
index 2298c4fd63e845b..61c8ca16ab0d043 100644
--- a/libcxx/test/support/count_new.h
+++ b/libcxx/test/support/count_new.h
@@ -455,11 +455,23 @@ void operator delete[](void* p, std::nothrow_t const&) TEST_NOEXCEPT {
 #      define USE_ALIGNED_ALLOC
 #    endif
 
+#    if defined(__APPLE__)
+#      if (defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) &&                                                   \
+           __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 101500)
+#        define TEST_HAS_NO_C11_ALIGNED_ALLOC
+#      endif
+#    elif defined(__ANDROID__) && __ANDROID_API__ < 28
+#      define TEST_HAS_NO_C11_ALIGNED_ALLOC
+#    endif
+
 inline void* allocate_aligned_impl(std::size_t size, std::align_val_t align) {
   const std::size_t alignment = static_cast<std::size_t>(align);
   void* ret                   = nullptr;
 #    ifdef USE_ALIGNED_ALLOC
   ret = _aligned_malloc(size, alignment);
+#    elif TEST_STD_VER >= 17 && !defined(TEST_HAS_NO_C11_ALIGNED_ALLOC)
+  size_t rounded_size = (size + alignment - 1) & ~(alignment - 1);
+  ret                 = aligned_alloc(alignment, size > rounded_size ? size : rounded_size);
 #    else
   assert(posix_memalign(&ret, std::max(alignment, sizeof(void*)), size) != EINVAL);
 #    endif

From 846f790216e1a0c40f8890d489904c3d716cc998 Mon Sep 17 00:00:00 2001
From: Tobias Stadler <mail@stadler-tobias.de>
Date: Thu, 15 Aug 2024 14:40:16 +0200
Subject: [PATCH 019/441] [GlobalISel] Combiner: Observer-based DCE and
 retrying of combines

Continues the work for disabling fixed-point iteration in the Combiner
(#94291).

This introduces improved Observer-based heuristics in the
GISel Combiner to retry combining defs/uses of modified instructions and
for performing sparse dead code elimination.

I have experimented a lot with the heuristics and this seems to be the
minimal set of heuristics that allows disabling fixed-point iteration
for AArch64 CTMark O2 without regressions.
Enabling this globally would pass all regression tests for all official
targets (apart from small benign diffs), but I have made this fully
opt-in for now, because I can't quantify the impact for other targets.

This should mostly be on-par with how the WorkList-aware functions
in the InstCombiner and DAGCombiner handle rescheduling instructions
for recombining.

For performance numbers see my follow-up patch for AArch64 (#102167)

Pull Request: https://github.com/llvm/llvm-project/pull/102163
---
 .../llvm/CodeGen/GlobalISel/Combiner.h        |  10 +-
 .../llvm/CodeGen/GlobalISel/CombinerInfo.h    |  25 ++
 llvm/lib/CodeGen/GlobalISel/Combiner.cpp      | 219 +++++++++++++++---
 3 files changed, 214 insertions(+), 40 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/GlobalISel/Combiner.h b/llvm/include/llvm/CodeGen/GlobalISel/Combiner.h
index f826601544932d1..fa6a7be6cf6c321 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/Combiner.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/Combiner.h
@@ -15,13 +15,13 @@
 #ifndef LLVM_CODEGEN_GLOBALISEL_COMBINER_H
 #define LLVM_CODEGEN_GLOBALISEL_COMBINER_H
 
+#include "llvm/CodeGen/GlobalISel/CombinerInfo.h"
 #include "llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h"
 #include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
 
 namespace llvm {
 class MachineRegisterInfo;
-struct CombinerInfo;
 class GISelCSEInfo;
 class TargetPassConfig;
 class MachineFunction;
@@ -33,8 +33,12 @@ class MachineIRBuilder;
 /// TODO: Is it worth making this module-wide?
 class Combiner : public GIMatchTableExecutor {
 private:
+  using WorkListTy = GISelWorkList<512>;
+
   class WorkListMaintainer;
-  GISelWorkList<512> WorkList;
+  template <CombinerInfo::ObserverLevel Lvl> class WorkListMaintainerImpl;
+
+  WorkListTy WorkList;
 
   // We have a little hack here where keep the owned pointers private, and only
   // expose a reference. This has two purposes:
@@ -48,6 +52,8 @@ class Combiner : public GIMatchTableExecutor {
 
   bool HasSetupMF = false;
 
+  static bool tryDCE(MachineInstr &MI, MachineRegisterInfo &MRI);
+
 public:
   /// If CSEInfo is not null, then the Combiner will use CSEInfo as the observer
   /// and also create a CSEMIRBuilder. Pass nullptr if CSE is not needed.
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerInfo.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerInfo.h
index 2b0eb71f8808277..67f95c962c58246 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerInfo.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerInfo.h
@@ -53,6 +53,31 @@ struct CombinerInfo {
   /// The maximum number of times the Combiner will iterate over the
   /// MachineFunction. Setting this to 0 enables fixed-point iteration.
   unsigned MaxIterations = 0;
+
+  enum class ObserverLevel {
+    /// Only retry combining created/changed instructions.
+    /// This replicates the legacy default Observer behavior for use with
+    /// fixed-point iteration.
+    Basic,
+    /// Enables Observer-based detection of dead instructions. This can save
+    /// some compile-time if full disabling of fixed-point iteration is not
+    /// desired. If the input IR doesn't contain dead instructions, consider
+    /// disabling \p EnableFullDCE.
+    DCE,
+    /// Enables Observer-based DCE and additional heuristics that retry
+    /// combining defined and used instructions of modified instructions.
+    /// This provides a good balance between compile-time and completeness of
+    /// combining without needing fixed-point iteration.
+    SinglePass,
+  };
+
+  /// Select how the Combiner acts on MIR changes.
+  ObserverLevel ObserverLvl = ObserverLevel::Basic;
+
+  /// Whether dead code elimination is performed before each Combiner iteration.
+  /// If Observer-based DCE is enabled, this controls if a full DCE pass is
+  /// performed before the first Combiner iteration.
+  bool EnableFullDCE = true;
 };
 } // namespace llvm
 
diff --git a/llvm/lib/CodeGen/GlobalISel/Combiner.cpp b/llvm/lib/CodeGen/GlobalISel/Combiner.cpp
index 49842e5fd65da6f..75b2525e368af68 100644
--- a/llvm/lib/CodeGen/GlobalISel/Combiner.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/Combiner.cpp
@@ -45,61 +45,190 @@ cl::OptionCategory GICombinerOptionCategory(
 );
 } // end namespace llvm
 
-/// This class acts as the glue the joins the CombinerHelper to the overall
+/// This class acts as the glue that joins the CombinerHelper to the overall
 /// Combine algorithm. The CombinerHelper is intended to report the
 /// modifications it makes to the MIR to the GISelChangeObserver and the
-/// observer subclass will act on these events. In this case, instruction
-/// erasure will cancel any future visits to the erased instruction and
-/// instruction creation will schedule that instruction for a future visit.
-/// Other Combiner implementations may require more complex behaviour from
-/// their GISelChangeObserver subclass.
+/// observer subclass will act on these events.
 class Combiner::WorkListMaintainer : public GISelChangeObserver {
-  using WorkListTy = GISelWorkList<512>;
-  WorkListTy &WorkList;
+protected:
+#ifndef NDEBUG
   /// The instructions that have been created but we want to report once they
   /// have their operands. This is only maintained if debug output is requested.
-#ifndef NDEBUG
-  SetVector<const MachineInstr *> CreatedInstrs;
+  SmallSetVector<const MachineInstr *, 32> CreatedInstrs;
 #endif
+  using Level = CombinerInfo::ObserverLevel;
 
 public:
-  WorkListMaintainer(WorkListTy &WorkList) : WorkList(WorkList) {}
+  static std::unique_ptr<WorkListMaintainer>
+  create(Level Lvl, WorkListTy &WorkList, MachineRegisterInfo &MRI);
+
   virtual ~WorkListMaintainer() = default;
 
+  void reportFullyCreatedInstrs() {
+    LLVM_DEBUG({
+      for (auto *MI : CreatedInstrs) {
+        dbgs() << "Created: " << *MI;
+      }
+      CreatedInstrs.clear();
+    });
+  }
+
+  virtual void reset() = 0;
+  virtual void appliedCombine() = 0;
+};
+
+/// A configurable WorkListMaintainer implementation.
+/// The ObserverLevel determines how the WorkListMaintainer reacts to MIR
+/// changes.
+template <CombinerInfo::ObserverLevel Lvl>
+class Combiner::WorkListMaintainerImpl : public Combiner::WorkListMaintainer {
+  WorkListTy &WorkList;
+  MachineRegisterInfo &MRI;
+
+  // Defer handling these instructions until the combine finishes.
+  SmallSetVector<MachineInstr *, 32> DeferList;
+
+  // Track VRegs that (might) have lost a use.
+  SmallSetVector<Register, 32> LostUses;
+
+public:
+  WorkListMaintainerImpl(WorkListTy &WorkList, MachineRegisterInfo &MRI)
+      : WorkList(WorkList), MRI(MRI) {}
+
+  virtual ~WorkListMaintainerImpl() = default;
+
+  void reset() override {
+    DeferList.clear();
+    LostUses.clear();
+  }
+
   void erasingInstr(MachineInstr &MI) override {
-    LLVM_DEBUG(dbgs() << "Erasing: " << MI << "\n");
+    // MI will become dangling, remove it from all lists.
+    LLVM_DEBUG(dbgs() << "Erasing: " << MI; CreatedInstrs.remove(&MI));
     WorkList.remove(&MI);
+    if constexpr (Lvl != Level::Basic) {
+      DeferList.remove(&MI);
+      noteLostUses(MI);
+    }
   }
+
   void createdInstr(MachineInstr &MI) override {
-    LLVM_DEBUG(dbgs() << "Creating: " << MI << "\n");
-    WorkList.insert(&MI);
-    LLVM_DEBUG(CreatedInstrs.insert(&MI));
+    LLVM_DEBUG(dbgs() << "Creating: " << MI; CreatedInstrs.insert(&MI));
+    if constexpr (Lvl == Level::Basic)
+      WorkList.insert(&MI);
+    else
+      // Defer handling newly created instructions, because they don't have
+      // operands yet. We also insert them into the WorkList in reverse
+      // order so that they will be combined top down.
+      DeferList.insert(&MI);
   }
+
   void changingInstr(MachineInstr &MI) override {
-    LLVM_DEBUG(dbgs() << "Changing: " << MI << "\n");
-    WorkList.insert(&MI);
+    LLVM_DEBUG(dbgs() << "Changing: " << MI);
+    // Some uses might get dropped when MI is changed.
+    // For now, overapproximate by assuming all uses will be dropped.
+    // TODO: Is a more precise heuristic or manual tracking of use count
+    // decrements worth it?
+    if constexpr (Lvl != Level::Basic)
+      noteLostUses(MI);
   }
+
   void changedInstr(MachineInstr &MI) override {
-    LLVM_DEBUG(dbgs() << "Changed: " << MI << "\n");
-    WorkList.insert(&MI);
+    LLVM_DEBUG(dbgs() << "Changed: " << MI);
+    if constexpr (Lvl == Level::Basic)
+      WorkList.insert(&MI);
+    else
+      // Defer this for DCE
+      DeferList.insert(&MI);
   }
 
-  void reportFullyCreatedInstrs() {
-    LLVM_DEBUG(for (const auto *MI
-                    : CreatedInstrs) {
-      dbgs() << "Created: ";
-      MI->print(dbgs());
-    });
-    LLVM_DEBUG(CreatedInstrs.clear());
+  // Only track changes during the combine and then walk the def/use-chains once
+  // the combine is finished, because:
+  // - instructions might have multiple defs during the combine.
+  // - use counts aren't accurate during the combine.
+  void appliedCombine() override {
+    if constexpr (Lvl == Level::Basic)
+      return;
+
+    // DCE deferred instructions and add them to the WorkList bottom up.
+    while (!DeferList.empty()) {
+      MachineInstr &MI = *DeferList.pop_back_val();
+      if (tryDCE(MI, MRI))
+        continue;
+
+      if constexpr (Lvl >= Level::SinglePass)
+        addUsersToWorkList(MI);
+
+      WorkList.insert(&MI);
+    }
+
+    // Handle instructions that have lost a user.
+    while (!LostUses.empty()) {
+      Register Use = LostUses.pop_back_val();
+      MachineInstr *UseMI = MRI.getVRegDef(Use);
+      if (!UseMI)
+        continue;
+
+      // If DCE succeeds, UseMI's uses are added back to LostUses by
+      // erasingInstr.
+      if (tryDCE(*UseMI, MRI))
+        continue;
+
+      if constexpr (Lvl >= Level::SinglePass) {
+        // OneUse checks are relatively common, so we might be able to combine
+        // the single remaining user of this Reg.
+        if (MRI.hasOneNonDBGUser(Use))
+          WorkList.insert(&*MRI.use_instr_nodbg_begin(Use));
+
+        WorkList.insert(UseMI);
+      }
+    }
+  }
+
+  void noteLostUses(MachineInstr &MI) {
+    for (auto &Use : MI.explicit_uses()) {
+      if (!Use.isReg() || !Use.getReg().isVirtual())
+        continue;
+      LostUses.insert(Use.getReg());
+    }
+  }
+
+  void addUsersToWorkList(MachineInstr &MI) {
+    for (auto &Def : MI.defs()) {
+      Register DefReg = Def.getReg();
+      if (!DefReg.isVirtual())
+        continue;
+      for (auto &UseMI : MRI.use_nodbg_instructions(DefReg)) {
+        WorkList.insert(&UseMI);
+      }
+    }
   }
 };
 
+std::unique_ptr<Combiner::WorkListMaintainer>
+Combiner::WorkListMaintainer::create(Level Lvl, WorkListTy &WorkList,
+                                     MachineRegisterInfo &MRI) {
+  switch (Lvl) {
+  case Level::Basic:
+    return std::make_unique<WorkListMaintainerImpl<Level::Basic>>(WorkList,
+                                                                  MRI);
+  case Level::DCE:
+    return std::make_unique<WorkListMaintainerImpl<Level::DCE>>(WorkList, MRI);
+  case Level::SinglePass:
+    return std::make_unique<WorkListMaintainerImpl<Level::SinglePass>>(WorkList,
+                                                                       MRI);
+  default:
+    llvm_unreachable("Illegal ObserverLevel");
+  }
+}
+
 Combiner::Combiner(MachineFunction &MF, CombinerInfo &CInfo,
                    const TargetPassConfig *TPC, GISelKnownBits *KB,
                    GISelCSEInfo *CSEInfo)
     : Builder(CSEInfo ? std::make_unique<CSEMIRBuilder>()
                       : std::make_unique<MachineIRBuilder>()),
-      WLObserver(std::make_unique<WorkListMaintainer>(WorkList)),
+      WLObserver(WorkListMaintainer::create(CInfo.ObserverLvl, WorkList,
+                                            MF.getRegInfo())),
       ObserverWrapper(std::make_unique<GISelObserverWrapper>()), CInfo(CInfo),
       Observer(*ObserverWrapper), B(*Builder), MF(MF), MRI(MF.getRegInfo()),
       KB(KB), TPC(TPC), CSEInfo(CSEInfo) {
@@ -115,6 +244,15 @@ Combiner::Combiner(MachineFunction &MF, CombinerInfo &CInfo,
 
 Combiner::~Combiner() = default;
 
+bool Combiner::tryDCE(MachineInstr &MI, MachineRegisterInfo &MRI) {
+  if (!isTriviallyDead(MI, MRI))
+    return false;
+  LLVM_DEBUG(dbgs() << "Dead: " << MI);
+  llvm::salvageDebugInfo(MRI, MI);
+  MI.eraseFromParent();
+  return true;
+}
+
 bool Combiner::combineMachineInstrs() {
   // If the ISel pipeline failed, do not bother running this pass.
   // FIXME: Should this be here or in individual combiner passes.
@@ -141,27 +279,29 @@ bool Combiner::combineMachineInstrs() {
     ++Iteration;
     LLVM_DEBUG(dbgs() << "\n\nCombiner iteration #" << Iteration << '\n');
 
+    Changed = false;
     WorkList.clear();
+    WLObserver->reset();
     ObserverWrapper->clearObservers();
     if (CSEInfo)
       ObserverWrapper->addObserver(CSEInfo);
 
+    // If Observer-based DCE is enabled, perform full DCE only before the first
+    // iteration.
+    bool EnableDCE = CInfo.ObserverLvl >= CombinerInfo::ObserverLevel::DCE
+                         ? CInfo.EnableFullDCE && Iteration == 1
+                         : CInfo.EnableFullDCE;
+
     // Collect all instructions. Do a post order traversal for basic blocks and
     // insert with list bottom up, so while we pop_back_val, we'll traverse top
     // down RPOT.
-    Changed = false;
-
     RAIIMFObsDelInstaller DelInstall(MF, *ObserverWrapper);
     for (MachineBasicBlock *MBB : post_order(&MF)) {
       for (MachineInstr &CurMI :
            llvm::make_early_inc_range(llvm::reverse(*MBB))) {
         // Erase dead insts before even adding to the list.
-        if (isTriviallyDead(CurMI, MRI)) {
-          LLVM_DEBUG(dbgs() << CurMI << "Is dead; erasing.\n");
-          llvm::salvageDebugInfo(MRI, CurMI);
-          CurMI.eraseFromParent();
+        if (EnableDCE && tryDCE(CurMI, MRI))
           continue;
-        }
         WorkList.deferred_insert(&CurMI);
       }
     }
@@ -171,10 +311,13 @@ bool Combiner::combineMachineInstrs() {
     ObserverWrapper->addObserver(WLObserver.get());
     // Main Loop. Process the instructions here.
     while (!WorkList.empty()) {
-      MachineInstr *CurrInst = WorkList.pop_back_val();
-      LLVM_DEBUG(dbgs() << "\nTry combining " << *CurrInst;);
-      Changed |= tryCombineAll(*CurrInst);
-      WLObserver->reportFullyCreatedInstrs();
+      MachineInstr &CurrInst = *WorkList.pop_back_val();
+      LLVM_DEBUG(dbgs() << "\nTry combining " << CurrInst);
+      bool AppliedCombine = tryCombineAll(CurrInst);
+      LLVM_DEBUG(WLObserver->reportFullyCreatedInstrs());
+      Changed |= AppliedCombine;
+      if (AppliedCombine)
+        WLObserver->appliedCombine();
     }
     MFChanged |= Changed;
 

From bfce1aae76de05dd6ac9fbbd997295fe5aa280c0 Mon Sep 17 00:00:00 2001
From: Janek van Oirschot <janek.vanoirschot@amd.com>
Date: Thu, 15 Aug 2024 13:43:13 +0100
Subject: [PATCH 020/441] [AMDGPU] MCExpr printing helper with KnownBits
 support (#95951)

Walks over the MCExpr and uses KnownBits to deduce whether an expression
is known and if so, prints said known value. Should support the most
common MCExpr cases for AMDGPU metadata.
---
 llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp   |  10 +-
 .../AMDGPU/MCTargetDesc/AMDGPUMCExpr.cpp      | 374 ++++++++++++++++++
 .../Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.h |  13 +
 .../MCTargetDesc/AMDGPUTargetStreamer.cpp     |  28 +-
 .../AMDGPU/Utils/AMDKernelCodeTUtils.cpp      |  46 +--
 .../Target/AMDGPU/Utils/AMDKernelCodeTUtils.h |   5 +-
 llvm/test/MC/AMDGPU/amd_kernel_code_t.s       |  36 +-
 llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx10.s     |  64 +--
 llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx11.s     |  60 +--
 llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx12.s     |  58 +--
 llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx7.s      |  54 +--
 llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx8.s      |  54 +--
 llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx90a.s    |  46 +--
 13 files changed, 611 insertions(+), 237 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
index e64e28e01d3d185..b90d245b7bd3944 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
@@ -395,12 +395,10 @@ void AMDGPUAsmPrinter::emitCommonFunctionComments(
 SmallString<128> AMDGPUAsmPrinter::getMCExprStr(const MCExpr *Value) {
   SmallString<128> Str;
   raw_svector_ostream OSS(Str);
-  int64_t IVal;
-  if (Value->evaluateAsAbsolute(IVal)) {
-    OSS << static_cast<uint64_t>(IVal);
-  } else {
-    Value->print(OSS, MAI);
-  }
+  auto &Streamer = getTargetStreamer()->getStreamer();
+  auto &Context = Streamer.getContext();
+  const MCExpr *New = foldAMDGPUMCExpr(Value, Context);
+  printAMDGPUMCExpr(New, OSS, MAI);
   return Str;
 }
 
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.cpp
index b2a9667f7c23ea2..4fbd7d0f889457d 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.cpp
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.cpp
@@ -10,12 +10,14 @@
 #include "GCNSubtarget.h"
 #include "Utils/AMDGPUBaseInfo.h"
 #include "llvm/IR/Function.h"
+#include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/MC/MCValue.h"
 #include "llvm/Support/Allocator.h"
+#include "llvm/Support/KnownBits.h"
 #include "llvm/Support/raw_ostream.h"
 #include <optional>
 
@@ -303,3 +305,375 @@ const AMDGPUMCExpr *AMDGPUMCExpr::createOccupancy(unsigned InitOcc,
                  CreateExpr(InitOcc), NumSGPRs, NumVGPRs},
                 Ctx);
 }
+
+static KnownBits fromOptionalToKnownBits(std::optional<bool> CompareResult) {
+  static constexpr unsigned BitWidth = 64;
+  const APInt True(BitWidth, 1);
+  const APInt False(BitWidth, 0);
+  if (CompareResult) {
+    return *CompareResult ? KnownBits::makeConstant(True)
+                          : KnownBits::makeConstant(False);
+  }
+
+  KnownBits UnknownBool(/*BitWidth=*/1);
+  return UnknownBool.zext(BitWidth);
+}
+
+using KnownBitsMap = DenseMap<const MCExpr *, KnownBits>;
+static void knownBitsMapHelper(const MCExpr *Expr, KnownBitsMap &KBM,
+                               unsigned Depth = 0);
+
+static void binaryOpKnownBitsMapHelper(const MCExpr *Expr, KnownBitsMap &KBM,
+                                       unsigned Depth) {
+  static constexpr unsigned BitWidth = 64;
+  const MCBinaryExpr *BExpr = cast<MCBinaryExpr>(Expr);
+  const MCExpr *LHS = BExpr->getLHS();
+  const MCExpr *RHS = BExpr->getRHS();
+
+  knownBitsMapHelper(LHS, KBM, Depth + 1);
+  knownBitsMapHelper(RHS, KBM, Depth + 1);
+  KnownBits LHSKnown = KBM[LHS];
+  KnownBits RHSKnown = KBM[RHS];
+
+  switch (BExpr->getOpcode()) {
+  default:
+    KBM[Expr] = KnownBits(BitWidth);
+    return;
+  case MCBinaryExpr::Opcode::Add:
+    KBM[Expr] = KnownBits::add(LHSKnown, RHSKnown);
+    return;
+  case MCBinaryExpr::Opcode::And:
+    KBM[Expr] = LHSKnown & RHSKnown;
+    return;
+  case MCBinaryExpr::Opcode::Div:
+    KBM[Expr] = KnownBits::sdiv(LHSKnown, RHSKnown);
+    return;
+  case MCBinaryExpr::Opcode::EQ: {
+    std::optional<bool> CompareRes = KnownBits::eq(LHSKnown, RHSKnown);
+    KBM[Expr] = fromOptionalToKnownBits(CompareRes);
+    return;
+  }
+  case MCBinaryExpr::Opcode::NE: {
+    std::optional<bool> CompareRes = KnownBits::ne(LHSKnown, RHSKnown);
+    KBM[Expr] = fromOptionalToKnownBits(CompareRes);
+    return;
+  }
+  case MCBinaryExpr::Opcode::GT: {
+    std::optional<bool> CompareRes = KnownBits::sgt(LHSKnown, RHSKnown);
+    KBM[Expr] = fromOptionalToKnownBits(CompareRes);
+    return;
+  }
+  case MCBinaryExpr::Opcode::GTE: {
+    std::optional<bool> CompareRes = KnownBits::sge(LHSKnown, RHSKnown);
+    KBM[Expr] = fromOptionalToKnownBits(CompareRes);
+    return;
+  }
+  case MCBinaryExpr::Opcode::LAnd: {
+    std::optional<bool> CompareRes;
+    const APInt False(BitWidth, 0);
+    std::optional<bool> LHSBool =
+        KnownBits::ne(LHSKnown, KnownBits::makeConstant(False));
+    std::optional<bool> RHSBool =
+        KnownBits::ne(RHSKnown, KnownBits::makeConstant(False));
+    if (LHSBool && RHSBool)
+      CompareRes = *LHSBool && *RHSBool;
+    KBM[Expr] = fromOptionalToKnownBits(CompareRes);
+    return;
+  }
+  case MCBinaryExpr::Opcode::LOr: {
+    const APInt False(BitWidth, 0);
+    KnownBits Bits = LHSKnown | RHSKnown;
+    std::optional<bool> CompareRes =
+        KnownBits::ne(Bits, KnownBits::makeConstant(False));
+    KBM[Expr] = fromOptionalToKnownBits(CompareRes);
+    return;
+  }
+  case MCBinaryExpr::Opcode::LT: {
+    std::optional<bool> CompareRes = KnownBits::slt(LHSKnown, RHSKnown);
+    KBM[Expr] = fromOptionalToKnownBits(CompareRes);
+    return;
+  }
+  case MCBinaryExpr::Opcode::LTE: {
+    std::optional<bool> CompareRes = KnownBits::sle(LHSKnown, RHSKnown);
+    KBM[Expr] = fromOptionalToKnownBits(CompareRes);
+    return;
+  }
+  case MCBinaryExpr::Opcode::Mod:
+    KBM[Expr] = KnownBits::srem(LHSKnown, RHSKnown);
+    return;
+  case MCBinaryExpr::Opcode::Mul:
+    KBM[Expr] = KnownBits::mul(LHSKnown, RHSKnown);
+    return;
+  case MCBinaryExpr::Opcode::Or:
+    KBM[Expr] = LHSKnown | RHSKnown;
+    return;
+  case MCBinaryExpr::Opcode::Shl:
+    KBM[Expr] = KnownBits::shl(LHSKnown, RHSKnown);
+    return;
+  case MCBinaryExpr::Opcode::AShr:
+    KBM[Expr] = KnownBits::ashr(LHSKnown, RHSKnown);
+    return;
+  case MCBinaryExpr::Opcode::LShr:
+    KBM[Expr] = KnownBits::lshr(LHSKnown, RHSKnown);
+    return;
+  case MCBinaryExpr::Opcode::Sub:
+    KBM[Expr] = KnownBits::sub(LHSKnown, RHSKnown);
+    return;
+  case MCBinaryExpr::Opcode::Xor:
+    KBM[Expr] = LHSKnown ^ RHSKnown;
+    return;
+  }
+}
+
+static void unaryOpKnownBitsMapHelper(const MCExpr *Expr, KnownBitsMap &KBM,
+                                      unsigned Depth) {
+  static constexpr unsigned BitWidth = 64;
+  const MCUnaryExpr *UExpr = cast<MCUnaryExpr>(Expr);
+  knownBitsMapHelper(UExpr->getSubExpr(), KBM, Depth + 1);
+  KnownBits KB = KBM[UExpr->getSubExpr()];
+
+  switch (UExpr->getOpcode()) {
+  default:
+    KBM[Expr] = KnownBits(BitWidth);
+    return;
+  case MCUnaryExpr::Opcode::Minus: {
+    KB.makeNegative();
+    KBM[Expr] = KB;
+    return;
+  }
+  case MCUnaryExpr::Opcode::Not: {
+    KnownBits AllOnes(BitWidth);
+    AllOnes.setAllOnes();
+    KBM[Expr] = KB ^ AllOnes;
+    return;
+  }
+  case MCUnaryExpr::Opcode::Plus: {
+    KB.makeNonNegative();
+    KBM[Expr] = KB;
+    return;
+  }
+  }
+}
+
+static void targetOpKnownBitsMapHelper(const MCExpr *Expr, KnownBitsMap &KBM,
+                                       unsigned Depth) {
+  static constexpr unsigned BitWidth = 64;
+  const AMDGPUMCExpr *AGVK = cast<AMDGPUMCExpr>(Expr);
+
+  switch (AGVK->getKind()) {
+  default:
+    KBM[Expr] = KnownBits(BitWidth);
+    return;
+  case AMDGPUMCExpr::VariantKind::AGVK_Or: {
+    knownBitsMapHelper(AGVK->getSubExpr(0), KBM, Depth + 1);
+    KnownBits KB = KBM[AGVK->getSubExpr(0)];
+    for (const MCExpr *Arg : AGVK->getArgs()) {
+      knownBitsMapHelper(Arg, KBM, Depth + 1);
+      KB |= KBM[Arg];
+    }
+    KBM[Expr] = KB;
+    return;
+  }
+  case AMDGPUMCExpr::VariantKind::AGVK_Max: {
+    knownBitsMapHelper(AGVK->getSubExpr(0), KBM, Depth + 1);
+    KnownBits KB = KBM[AGVK->getSubExpr(0)];
+    for (const MCExpr *Arg : AGVK->getArgs()) {
+      knownBitsMapHelper(Arg, KBM, Depth + 1);
+      KB = KnownBits::umax(KB, KBM[Arg]);
+    }
+    KBM[Expr] = KB;
+    return;
+  }
+  case AMDGPUMCExpr::VariantKind::AGVK_ExtraSGPRs:
+  case AMDGPUMCExpr::VariantKind::AGVK_TotalNumVGPRs:
+  case AMDGPUMCExpr::VariantKind::AGVK_AlignTo:
+  case AMDGPUMCExpr::VariantKind::AGVK_Occupancy: {
+    int64_t Val;
+    if (AGVK->evaluateAsAbsolute(Val)) {
+      APInt APValue(BitWidth, Val);
+      KBM[Expr] = KnownBits::makeConstant(APValue);
+      return;
+    }
+    KBM[Expr] = KnownBits(BitWidth);
+    return;
+  }
+  }
+}
+
+static void knownBitsMapHelper(const MCExpr *Expr, KnownBitsMap &KBM,
+                               unsigned Depth) {
+  static constexpr unsigned BitWidth = 64;
+
+  int64_t Val;
+  if (Expr->evaluateAsAbsolute(Val)) {
+    APInt APValue(BitWidth, Val, /*isSigned=*/true);
+    KBM[Expr] = KnownBits::makeConstant(APValue);
+    return;
+  }
+
+  if (Depth == 16) {
+    KBM[Expr] = KnownBits(BitWidth);
+    return;
+  }
+
+  switch (Expr->getKind()) {
+  case MCExpr::ExprKind::Binary: {
+    binaryOpKnownBitsMapHelper(Expr, KBM, Depth);
+    return;
+  }
+  case MCExpr::ExprKind::Constant: {
+    const MCConstantExpr *CE = cast<MCConstantExpr>(Expr);
+    APInt APValue(BitWidth, CE->getValue(), /*isSigned=*/true);
+    KBM[Expr] = KnownBits::makeConstant(APValue);
+    return;
+  }
+  case MCExpr::ExprKind::SymbolRef: {
+    const MCSymbolRefExpr *RExpr = cast<MCSymbolRefExpr>(Expr);
+    const MCSymbol &Sym = RExpr->getSymbol();
+    if (!Sym.isVariable()) {
+      KBM[Expr] = KnownBits(BitWidth);
+      return;
+    }
+
+    // Variable value retrieval is not for actual use but only for knownbits
+    // analysis.
+    knownBitsMapHelper(Sym.getVariableValue(/*SetUsed=*/false), KBM, Depth + 1);
+    KBM[Expr] = KBM[Sym.getVariableValue(/*SetUsed=*/false)];
+    return;
+  }
+  case MCExpr::ExprKind::Unary: {
+    unaryOpKnownBitsMapHelper(Expr, KBM, Depth);
+    return;
+  }
+  case MCExpr::ExprKind::Target: {
+    targetOpKnownBitsMapHelper(Expr, KBM, Depth);
+    return;
+  }
+  }
+}
+
+static const MCExpr *tryFoldHelper(const MCExpr *Expr, KnownBitsMap &KBM,
+                                   MCContext &Ctx) {
+  if (!KBM.count(Expr))
+    return Expr;
+
+  auto ValueCheckKnownBits = [](KnownBits &KB, unsigned Value) -> bool {
+    if (!KB.isConstant())
+      return false;
+
+    return Value == KB.getConstant();
+  };
+
+  if (Expr->getKind() == MCExpr::ExprKind::Constant)
+    return Expr;
+
+  // Resolving unary operations to constants may make the value more ambiguous.
+  // For example, `~62` becomes `-63`; however, to me it's more ambiguous if a
+  // bit mask value is represented through a negative number.
+  if (Expr->getKind() != MCExpr::ExprKind::Unary) {
+    if (KBM[Expr].isConstant()) {
+      APInt ConstVal = KBM[Expr].getConstant();
+      return MCConstantExpr::create(ConstVal.getSExtValue(), Ctx);
+    }
+
+    int64_t EvalValue;
+    if (Expr->evaluateAsAbsolute(EvalValue))
+      return MCConstantExpr::create(EvalValue, Ctx);
+  }
+
+  switch (Expr->getKind()) {
+  default:
+    return Expr;
+  case MCExpr::ExprKind::Binary: {
+    const MCBinaryExpr *BExpr = cast<MCBinaryExpr>(Expr);
+    const MCExpr *LHS = BExpr->getLHS();
+    const MCExpr *RHS = BExpr->getRHS();
+
+    switch (BExpr->getOpcode()) {
+    default:
+      return Expr;
+    case MCBinaryExpr::Opcode::Sub: {
+      if (ValueCheckKnownBits(KBM[RHS], 0))
+        return tryFoldHelper(LHS, KBM, Ctx);
+      break;
+    }
+    case MCBinaryExpr::Opcode::Add:
+    case MCBinaryExpr::Opcode::Or: {
+      if (ValueCheckKnownBits(KBM[LHS], 0))
+        return tryFoldHelper(RHS, KBM, Ctx);
+      if (ValueCheckKnownBits(KBM[RHS], 0))
+        return tryFoldHelper(LHS, KBM, Ctx);
+      break;
+    }
+    case MCBinaryExpr::Opcode::Mul: {
+      if (ValueCheckKnownBits(KBM[LHS], 1))
+        return tryFoldHelper(RHS, KBM, Ctx);
+      if (ValueCheckKnownBits(KBM[RHS], 1))
+        return tryFoldHelper(LHS, KBM, Ctx);
+      break;
+    }
+    case MCBinaryExpr::Opcode::Shl:
+    case MCBinaryExpr::Opcode::AShr:
+    case MCBinaryExpr::Opcode::LShr: {
+      if (ValueCheckKnownBits(KBM[RHS], 0))
+        return tryFoldHelper(LHS, KBM, Ctx);
+      if (ValueCheckKnownBits(KBM[LHS], 0))
+        return MCConstantExpr::create(0, Ctx);
+      break;
+    }
+    case MCBinaryExpr::Opcode::And: {
+      if (ValueCheckKnownBits(KBM[LHS], 0) || ValueCheckKnownBits(KBM[RHS], 0))
+        return MCConstantExpr::create(0, Ctx);
+      break;
+    }
+    }
+    const MCExpr *NewLHS = tryFoldHelper(LHS, KBM, Ctx);
+    const MCExpr *NewRHS = tryFoldHelper(RHS, KBM, Ctx);
+    if (NewLHS != LHS || NewRHS != RHS)
+      return MCBinaryExpr::create(BExpr->getOpcode(), NewLHS, NewRHS, Ctx,
+                                  BExpr->getLoc());
+    return Expr;
+  }
+  case MCExpr::ExprKind::Unary: {
+    const MCUnaryExpr *UExpr = cast<MCUnaryExpr>(Expr);
+    const MCExpr *SubExpr = UExpr->getSubExpr();
+    const MCExpr *NewSubExpr = tryFoldHelper(SubExpr, KBM, Ctx);
+    if (SubExpr != NewSubExpr)
+      return MCUnaryExpr::create(UExpr->getOpcode(), NewSubExpr, Ctx,
+                                 UExpr->getLoc());
+    return Expr;
+  }
+  case MCExpr::ExprKind::Target: {
+    const AMDGPUMCExpr *AGVK = cast<AMDGPUMCExpr>(Expr);
+    SmallVector<const MCExpr *, 8> NewArgs;
+    bool Changed = false;
+    for (const MCExpr *Arg : AGVK->getArgs()) {
+      const MCExpr *NewArg = tryFoldHelper(Arg, KBM, Ctx);
+      NewArgs.push_back(NewArg);
+      Changed |= Arg != NewArg;
+    }
+    return Changed ? AMDGPUMCExpr::create(AGVK->getKind(), NewArgs, Ctx) : Expr;
+  }
+  }
+  return Expr;
+}
+
+const MCExpr *llvm::AMDGPU::foldAMDGPUMCExpr(const MCExpr *Expr,
+                                             MCContext &Ctx) {
+  KnownBitsMap KBM;
+  knownBitsMapHelper(Expr, KBM);
+  const MCExpr *NewExpr = tryFoldHelper(Expr, KBM, Ctx);
+
+  return Expr != NewExpr ? NewExpr : Expr;
+}
+
+void llvm::AMDGPU::printAMDGPUMCExpr(const MCExpr *Expr, raw_ostream &OS,
+                                     const MCAsmInfo *MAI) {
+  int64_t Val;
+  if (Expr->evaluateAsAbsolute(Val)) {
+    OS << Val;
+    return;
+  }
+
+  Expr->print(OS, MAI);
+}
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.h b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.h
index 970802dab69b60a..a16843f404b8f62 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.h
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCExpr.h
@@ -90,6 +90,7 @@ class AMDGPUMCExpr : public MCTargetExpr {
                                              const GCNSubtarget &STM,
                                              MCContext &Ctx);
 
+  ArrayRef<const MCExpr *> getArgs() const { return Args; }
   VariantKind getKind() const { return Kind; }
   const MCExpr *getSubExpr(size_t Index) const;
 
@@ -105,6 +106,18 @@ class AMDGPUMCExpr : public MCTargetExpr {
   }
 };
 
+namespace AMDGPU {
+// Tries to leverage KnownBits for MCExprs to reduce and limit any composed
+// MCExprs printing. E.g., for an expression such as
+// ((unevaluatable_sym | 1) & 1) won't evaluate due to unevaluatable_sym and
+// would verbosely print the full expression; however, KnownBits should deduce
+// the value to be 1. Particularly useful for AMDGPU metadata MCExprs.
+void printAMDGPUMCExpr(const MCExpr *Expr, raw_ostream &OS,
+                       const MCAsmInfo *MAI);
+
+const MCExpr *foldAMDGPUMCExpr(const MCExpr *Expr, MCContext &Ctx);
+
+} // end namespace AMDGPU
 } // end namespace llvm
 
 #endif // LLVM_LIB_TARGET_AMDGPU_MCTARGETDESC_AMDGPUMCEXPR_H
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp
index 8c78db8e83c42d6..73d466abc66f7b7 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp
@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "AMDGPUTargetStreamer.h"
+#include "AMDGPUMCExpr.h"
 #include "AMDGPUMCKernelDescriptor.h"
 #include "AMDGPUPTNote.h"
 #include "Utils/AMDGPUBaseInfo.h"
@@ -244,8 +245,13 @@ void AMDGPUTargetAsmStreamer::EmitDirectiveAMDHSACodeObjectVersion(
 }
 
 void AMDGPUTargetAsmStreamer::EmitAMDKernelCodeT(AMDGPUMCKernelCodeT &Header) {
+  auto FoldAndPrint = [&](const MCExpr *Expr, raw_ostream &OS,
+                          const MCAsmInfo *MAI) {
+    printAMDGPUMCExpr(foldAMDGPUMCExpr(Expr, getContext()), OS, MAI);
+  };
+
   OS << "\t.amd_kernel_code_t\n";
-  Header.EmitKernelCodeT(OS, getContext());
+  Header.EmitKernelCodeT(OS, getContext(), FoldAndPrint);
   OS << "\t.end_amd_kernel_code_t\n";
 }
 
@@ -329,24 +335,17 @@ void AMDGPUTargetAsmStreamer::EmitAmdhsaKernelDescriptor(
 
   auto PrintField = [&](const MCExpr *Expr, uint32_t Shift, uint32_t Mask,
                         StringRef Directive) {
-    int64_t IVal;
     OS << "\t\t" << Directive << ' ';
-    const MCExpr *pgm_rsrc1_bits =
+    const MCExpr *ShiftedAndMaskedExpr =
         MCKernelDescriptor::bits_get(Expr, Shift, Mask, getContext());
-    if (pgm_rsrc1_bits->evaluateAsAbsolute(IVal))
-      OS << static_cast<uint64_t>(IVal);
-    else
-      pgm_rsrc1_bits->print(OS, MAI);
+    const MCExpr *New = foldAMDGPUMCExpr(ShiftedAndMaskedExpr, getContext());
+    printAMDGPUMCExpr(New, OS, MAI);
     OS << '\n';
   };
 
   auto EmitMCExpr = [&](const MCExpr *Value) {
-    int64_t evaluatableValue;
-    if (Value->evaluateAsAbsolute(evaluatableValue)) {
-      OS << static_cast<uint64_t>(evaluatableValue);
-    } else {
-      Value->print(OS, MAI);
-    }
+    const MCExpr *NewExpr = foldAMDGPUMCExpr(Value, getContext());
+    printAMDGPUMCExpr(NewExpr, OS, MAI);
   };
 
   OS << "\t\t.amdhsa_group_segment_fixed_size ";
@@ -462,7 +461,8 @@ void AMDGPUTargetAsmStreamer::EmitAmdhsaKernelDescriptor(
     accum_bits = MCBinaryExpr::createMul(
         accum_bits, MCConstantExpr::create(4, getContext()), getContext());
     OS << "\t\t.amdhsa_accum_offset ";
-    EmitMCExpr(accum_bits);
+    const MCExpr *New = foldAMDGPUMCExpr(accum_bits, getContext());
+    printAMDGPUMCExpr(New, OS, MAI);
     OS << '\n';
   }
 
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDKernelCodeTUtils.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDKernelCodeTUtils.cpp
index 720d5a1853dbbd7..7b88ddb7b0e95d1 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDKernelCodeTUtils.cpp
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDKernelCodeTUtils.cpp
@@ -226,35 +226,35 @@ class PrintField {
   template <typename T, T AMDGPUMCKernelCodeT::*ptr,
             typename std::enable_if_t<!std::is_integral_v<T>, T> * = nullptr>
   static void printField(StringRef Name, const AMDGPUMCKernelCodeT &C,
-                         raw_ostream &OS, MCContext &Ctx) {
+                         raw_ostream &OS, MCContext &Ctx,
+                         AMDGPUMCKernelCodeT::PrintHelper Helper) {
     OS << Name << " = ";
     const MCExpr *Value = C.*ptr;
-    int64_t Val;
-    if (Value->evaluateAsAbsolute(Val))
-      OS << Val;
-    else
-      Value->print(OS, Ctx.getAsmInfo());
+    Helper(Value, OS, Ctx.getAsmInfo());
   }
 
   template <typename T, T AMDGPUMCKernelCodeT::*ptr,
             typename std::enable_if_t<std::is_integral_v<T>, T> * = nullptr>
   static void printField(StringRef Name, const AMDGPUMCKernelCodeT &C,
-                         raw_ostream &OS, MCContext &) {
+                         raw_ostream &OS, MCContext &,
+                         AMDGPUMCKernelCodeT::PrintHelper) {
     OS << Name << " = " << (int)(C.*ptr);
   }
 };
 
 template <typename T, T AMDGPUMCKernelCodeT::*ptr, int shift, int width = 1>
 static void printBitField(StringRef Name, const AMDGPUMCKernelCodeT &C,
-                          raw_ostream &OS, MCContext &) {
+                          raw_ostream &OS, MCContext &,
+                          AMDGPUMCKernelCodeT::PrintHelper) {
   const auto Mask = (static_cast<T>(1) << width) - 1;
   OS << Name << " = " << (int)((C.*ptr >> shift) & Mask);
 }
 
 using PrintFx = void (*)(StringRef, const AMDGPUMCKernelCodeT &, raw_ostream &,
-                         MCContext &);
+                         MCContext &, AMDGPUMCKernelCodeT::PrintHelper Helper);
 
-static ArrayRef<PrintFx> getPrinterTable() {
+static ArrayRef<PrintFx>
+getPrinterTable(AMDGPUMCKernelCodeT::PrintHelper Helper) {
   static const PrintFx Table[] = {
 #define COMPPGM1(name, aname, AccMacro)                                        \
   COMPPGM(name, aname, C_00B848_##AccMacro, S_00B848_##AccMacro, 0)
@@ -263,7 +263,7 @@ static ArrayRef<PrintFx> getPrinterTable() {
 #define PRINTFIELD(sname, aname, name) PrintField::printField<FLD_T(name)>
 #define PRINTCOMP(Complement, PGMType)                                         \
   [](StringRef Name, const AMDGPUMCKernelCodeT &C, raw_ostream &OS,            \
-     MCContext &Ctx) {                                                         \
+     MCContext &Ctx, AMDGPUMCKernelCodeT::PrintHelper Helper) {                \
     OS << Name << " = ";                                                       \
     auto [Shift, Mask] = getShiftMask(Complement);                             \
     const MCExpr *Value;                                                       \
@@ -274,11 +274,7 @@ static ArrayRef<PrintFx> getPrinterTable() {
       Value =                                                                  \
           maskShiftGet(C.compute_pgm_resource2_registers, Mask, Shift, Ctx);   \
     }                                                                          \
-    int64_t Val;                                                               \
-    if (Value->evaluateAsAbsolute(Val))                                        \
-      OS << Val;                                                               \
-    else                                                                       \
-      Value->print(OS, Ctx.getAsmInfo());                                      \
+    Helper(Value, OS, Ctx.getAsmInfo());                                       \
   }
 #define RECORD(name, altName, print, parse) print
 #include "Utils/AMDKernelCodeTInfo.h"
@@ -379,10 +375,11 @@ static ArrayRef<ParseFx> getParserTable() {
 }
 
 static void printAmdKernelCodeField(const AMDGPUMCKernelCodeT &C, int FldIndex,
-                                    raw_ostream &OS, MCContext &Ctx) {
-  auto Printer = getPrinterTable()[FldIndex];
+                                    raw_ostream &OS, MCContext &Ctx,
+                                    AMDGPUMCKernelCodeT::PrintHelper Helper) {
+  auto Printer = getPrinterTable(Helper)[FldIndex];
   if (Printer)
-    Printer(get_amd_kernel_code_t_FldNames()[FldIndex + 1], C, OS, Ctx);
+    Printer(get_amd_kernel_code_t_FldNames()[FldIndex + 1], C, OS, Ctx, Helper);
 }
 
 void AMDGPUMCKernelCodeT::initDefault(const MCSubtargetInfo *STI,
@@ -459,20 +456,17 @@ bool AMDGPUMCKernelCodeT::ParseKernelCodeT(StringRef ID, MCAsmParser &MCParser,
   return Parser ? Parser(*this, MCParser, Err) : false;
 }
 
-void AMDGPUMCKernelCodeT::EmitKernelCodeT(raw_ostream &OS, MCContext &Ctx) {
+void AMDGPUMCKernelCodeT::EmitKernelCodeT(raw_ostream &OS, MCContext &Ctx,
+                                          PrintHelper Helper) {
   const int Size = hasMCExprVersionTable().size();
   for (int i = 0; i < Size; ++i) {
     OS << "\t\t";
     if (hasMCExprVersionTable()[i]) {
       OS << get_amd_kernel_code_t_FldNames()[i + 1] << " = ";
-      int64_t Val;
       const MCExpr *Value = getMCExprForIndex(i);
-      if (Value->evaluateAsAbsolute(Val))
-        OS << Val;
-      else
-        Value->print(OS, Ctx.getAsmInfo());
+      Helper(Value, OS, Ctx.getAsmInfo());
     } else {
-      printAmdKernelCodeField(*this, i, OS, Ctx);
+      printAmdKernelCodeField(*this, i, OS, Ctx, Helper);
     }
     OS << '\n';
   }
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDKernelCodeTUtils.h b/llvm/lib/Target/AMDGPU/Utils/AMDKernelCodeTUtils.h
index 6aeb98f1ce147b8..39acd5c43053bf5 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDKernelCodeTUtils.h
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDKernelCodeTUtils.h
@@ -27,6 +27,7 @@ class MCExpr;
 class MCStreamer;
 class MCSubtargetInfo;
 class raw_ostream;
+class MCAsmInfo;
 namespace AMDGPU {
 
 struct AMDGPUMCKernelCodeT {
@@ -79,8 +80,10 @@ struct AMDGPUMCKernelCodeT {
 
   const MCExpr *&getMCExprForIndex(int Index);
 
+  using PrintHelper =
+      function_ref<void(const MCExpr *, raw_ostream &, const MCAsmInfo *)>;
   bool ParseKernelCodeT(StringRef ID, MCAsmParser &MCParser, raw_ostream &Err);
-  void EmitKernelCodeT(raw_ostream &OS, MCContext &Ctx);
+  void EmitKernelCodeT(raw_ostream &OS, MCContext &Ctx, PrintHelper Helper);
   void EmitKernelCodeT(MCStreamer &OS, MCContext &Ctx);
 };
 
diff --git a/llvm/test/MC/AMDGPU/amd_kernel_code_t.s b/llvm/test/MC/AMDGPU/amd_kernel_code_t.s
index 052ec0bfabb849d..507001a2657c5f5 100644
--- a/llvm/test/MC/AMDGPU/amd_kernel_code_t.s
+++ b/llvm/test/MC/AMDGPU/amd_kernel_code_t.s
@@ -131,17 +131,17 @@ unknown_workitem_private_segment_byte_size:
 	s_endpgm
 
 ; ASM-LABEL: unknown_granulated_workitem_vgpr_count:
-; ASM: granulated_workitem_vgpr_count = ((0&4294967232)|(unknown&63))&63
-; ASM: granulated_wavefront_sgpr_count = (((0&4294967232)|(unknown&63))>>6)&15
-; ASM: priority = (((0&4294967232)|(unknown&63))>>10)&3
-; ASM: float_mode = (((0&4294967232)|(unknown&63))>>12)&255
-; ASM: priv = (((0&4294967232)|(unknown&63))>>20)&1
-; ASM: enable_dx10_clamp = (((0&4294967232)|(unknown&63))>>21)&1
-; ASM: debug_mode = (((0&4294967232)|(unknown&63))>>22)&1
-; ASM: enable_ieee_mode = (((0&4294967232)|(unknown&63))>>23)&1
-; ASM: enable_wgp_mode = (((0&4294967232)|(unknown&63))>>29)&1
-; ASM: enable_mem_ordered = (((0&4294967232)|(unknown&63))>>30)&1
-; ASM: enable_fwd_progress = (((0&4294967232)|(unknown&63))>>31)&1
+; ASM: granulated_workitem_vgpr_count = (unknown&63)&63
+; ASM: granulated_wavefront_sgpr_count = 0
+; ASM: priority = 0
+; ASM: float_mode = 0
+; ASM: priv = 0
+; ASM: enable_dx10_clamp = 0
+; ASM: debug_mode = 0
+; ASM: enable_ieee_mode = 0
+; ASM: enable_wgp_mode = 0
+; ASM: enable_mem_ordered = 0
+; ASM: enable_fwd_progress = 0
 .section .unknown_granulated_workitem_vgpr_count
 unknown_granulated_workitem_vgpr_count:
 	.amd_kernel_code_t
@@ -150,17 +150,9 @@ unknown_granulated_workitem_vgpr_count:
 	s_endpgm
 
 ; ASM-LABEL: unknown_enable_sgpr_workgroup_id_x:
-; ASM: enable_sgpr_private_segment_wave_byte_offset = ((0&4294967167)|((unknown&1)<<7))&1
-; ASM: user_sgpr_count = (((0&4294967167)|((unknown&1)<<7))>>1)&31
-; ASM: enable_trap_handler = (((0&4294967167)|((unknown&1)<<7))>>6)&1
-; ASM: enable_sgpr_workgroup_id_x = (((0&4294967167)|((unknown&1)<<7))>>7)&1
-; ASM: enable_sgpr_workgroup_id_y = (((0&4294967167)|((unknown&1)<<7))>>8)&1
-; ASM: enable_sgpr_workgroup_id_z = (((0&4294967167)|((unknown&1)<<7))>>9)&1
-; ASM: enable_sgpr_workgroup_info = (((0&4294967167)|((unknown&1)<<7))>>10)&1
-; ASM: enable_vgpr_workitem_id = (((0&4294967167)|((unknown&1)<<7))>>11)&3
-; ASM: enable_exception_msb = (((0&4294967167)|((unknown&1)<<7))>>13)&3
-; ASM: granulated_lds_size = (((0&4294967167)|((unknown&1)<<7))>>15)&511
-; ASM: enable_exception = (((0&4294967167)|((unknown&1)<<7))>>24)&127
+; ASM: enable_sgpr_workgroup_id_x = (((unknown&1)<<7)>>7)&1
+; ASM: enable_sgpr_workgroup_id_y = 0
+; ASM: enable_sgpr_workgroup_id_z = 0
 .section .unknown_enable_sgpr_workgroup_id_x
 unknown_enable_sgpr_workgroup_id_x:
 	.amd_kernel_code_t
diff --git a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx10.s b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx10.s
index af4cb1a008f9d92..bec717e4137df2b 100644
--- a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx10.s
+++ b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx10.s
@@ -106,44 +106,44 @@ expr_defined:
 // ASM-NEXT: .amdhsa_group_segment_fixed_size defined_value+2
 // ASM-NEXT: .amdhsa_private_segment_fixed_size defined_value+3
 // ASM-NEXT: .amdhsa_kernarg_size 0
-// ASM-NEXT: .amdhsa_user_sgpr_count (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&62)>>1
-// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&1)>>0
-// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&2)>>1
-// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&4)>>2
-// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&8)>>3
-// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&16)>>4
-// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&32)>>5
-// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&64)>>6
-// ASM-NEXT: .amdhsa_wavefront_size32 (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&1024)>>10
-// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1)>>0
-// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&128)>>7
-// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&256)>>8
-// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&512)>>9
-// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1024)>>10
-// ASM-NEXT: .amdhsa_system_vgpr_workitem_id (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&6144)>>11
+// ASM-NEXT: .amdhsa_user_sgpr_count 0
+// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer 0
+// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0
+// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 0
+// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0
+// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id 0
+// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init 0
+// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 0
+// ASM-NEXT: .amdhsa_wavefront_size32 1
+// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset ((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&1
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&128)>>7
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&256)>>8
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&512)>>9
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&1024)>>10
+// ASM-NEXT: .amdhsa_system_vgpr_workitem_id (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&6144)>>11
 // ASM-NEXT: .amdhsa_next_free_vgpr defined_value+4
 // ASM-NEXT: .amdhsa_next_free_sgpr defined_value+5
 // ASM-NEXT: .amdhsa_reserve_vcc defined_boolean
 // ASM-NEXT: .amdhsa_reserve_flat_scratch defined_boolean
 // ASM-NEXT: .amdhsa_reserve_xnack_mask 1
-// ASM-NEXT: .amdhsa_float_round_mode_32 (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|((((alignto(max(defined_value+4, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max(0, 1), 8))/8)-1)<<6))&12288)>>12
-// ASM-NEXT: .amdhsa_float_round_mode_16_64 (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|((((alignto(max(defined_value+4, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max(0, 1), 8))/8)-1)<<6))&49152)>>14
-// ASM-NEXT: .amdhsa_float_denorm_mode_32 (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|((((alignto(max(defined_value+4, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max(0, 1), 8))/8)-1)<<6))&196608)>>16
-// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|((((alignto(max(defined_value+4, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max(0, 1), 8))/8)-1)<<6))&786432)>>18
-// ASM-NEXT: .amdhsa_dx10_clamp (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|((((alignto(max(defined_value+4, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max(0, 1), 8))/8)-1)<<6))&2097152)>>21
-// ASM-NEXT: .amdhsa_ieee_mode (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|((((alignto(max(defined_value+4, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max(0, 1), 8))/8)-1)<<6))&8388608)>>23
-// ASM-NEXT: .amdhsa_fp16_overflow (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|((((alignto(max(defined_value+4, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max(0, 1), 8))/8)-1)<<6))&67108864)>>26
-// ASM-NEXT: .amdhsa_workgroup_processor_mode (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|((((alignto(max(defined_value+4, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max(0, 1), 8))/8)-1)<<6))&536870912)>>29
-// ASM-NEXT: .amdhsa_memory_ordered (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|((((alignto(max(defined_value+4, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max(0, 1), 8))/8)-1)<<6))&1073741824)>>30
-// ASM-NEXT: .amdhsa_forward_progress (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|((((alignto(max(defined_value+4, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max(0, 1), 8))/8)-1)<<6))&2147483648)>>31
+// ASM-NEXT: .amdhsa_float_round_mode_32 (((((((((((((((((((1621884928|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&12288)>>12
+// ASM-NEXT: .amdhsa_float_round_mode_16_64 (((((((((((((((((((1621884928|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&49152)>>14
+// ASM-NEXT: .amdhsa_float_denorm_mode_32 (((((((((((((((((((1621884928|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&196608)>>16
+// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 (((((((((((((((((((1621884928|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&786432)>>18
+// ASM-NEXT: .amdhsa_dx10_clamp (((((((((((((((((((1621884928|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&2097152)>>21
+// ASM-NEXT: .amdhsa_ieee_mode (((((((((((((((((((1621884928|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&8388608)>>23
+// ASM-NEXT: .amdhsa_fp16_overflow (((((((((((((((((((1621884928|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&67108864)>>26
+// ASM-NEXT: .amdhsa_workgroup_processor_mode (((((((((((((((((((1621884928|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&536870912)>>29
+// ASM-NEXT: .amdhsa_memory_ordered (((((((((((((((((((1621884928|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&1073741824)>>30
+// ASM-NEXT: .amdhsa_forward_progress (((((((((((((((((((1621884928|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&2147483648)>>31
 // ASM-NEXT: .amdhsa_shared_vgpr_count 0
-// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&16777216)>>24
-// ASM-NEXT: .amdhsa_exception_fp_denorm_src (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&33554432)>>25
-// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&67108864)>>26
-// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&134217728)>>27
-// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&268435456)>>28
-// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&536870912)>>29
-// ASM-NEXT: .amdhsa_exception_int_div_zero (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1073741824)>>30
+// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&16777216)>>24
+// ASM-NEXT: .amdhsa_exception_fp_denorm_src (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&33554432)>>25
+// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&67108864)>>26
+// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&134217728)>>27
+// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&268435456)>>28
+// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&536870912)>>29
+// ASM-NEXT: .amdhsa_exception_int_div_zero (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&1073741824)>>30
 // ASM-NEXT: .end_amdhsa_kernel
 
 // ASM:       .set defined_value, 41
diff --git a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx11.s b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx11.s
index b6e4ddde3d7f9e2..85a7ad05b00f484 100644
--- a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx11.s
+++ b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx11.s
@@ -106,40 +106,40 @@ expr_defined:
 // ASM-NEXT: .amdhsa_group_segment_fixed_size defined_value+2
 // ASM-NEXT: .amdhsa_private_segment_fixed_size defined_value+3
 // ASM-NEXT: .amdhsa_kernarg_size 0
-// ASM-NEXT: .amdhsa_user_sgpr_count (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&62)>>1
-// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&2)>>1
-// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&4)>>2
-// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&8)>>3
-// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&16)>>4
-// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&64)>>6
-// ASM-NEXT: .amdhsa_wavefront_size32 (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&1024)>>10
-// ASM-NEXT: .amdhsa_enable_private_segment (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1)>>0
-// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&128)>>7
-// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&256)>>8
-// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&512)>>9
-// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1024)>>10
-// ASM-NEXT: .amdhsa_system_vgpr_workitem_id (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&6144)>>11
+// ASM-NEXT: .amdhsa_user_sgpr_count 0
+// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0
+// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 0
+// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0
+// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id 0
+// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 0
+// ASM-NEXT: .amdhsa_wavefront_size32 1
+// ASM-NEXT: .amdhsa_enable_private_segment ((((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|defined_boolean)&(~62))&1
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x (((((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|defined_boolean)&(~62))&128)>>7
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y (((((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|defined_boolean)&(~62))&256)>>8
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z (((((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|defined_boolean)&(~62))&512)>>9
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info (((((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|defined_boolean)&(~62))&1024)>>10
+// ASM-NEXT: .amdhsa_system_vgpr_workitem_id (((((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|defined_boolean)&(~62))&6144)>>11
 // ASM-NEXT: .amdhsa_next_free_vgpr defined_value+4
 // ASM-NEXT: .amdhsa_next_free_sgpr defined_value+5
 // ASM-NEXT: .amdhsa_reserve_vcc defined_boolean
-// ASM-NEXT: .amdhsa_float_round_mode_32 (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|((((alignto(max(defined_value+4, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max(0, 1), 8))/8)-1)<<6))&12288)>>12
-// ASM-NEXT: .amdhsa_float_round_mode_16_64 (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|((((alignto(max(defined_value+4, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max(0, 1), 8))/8)-1)<<6))&49152)>>14
-// ASM-NEXT: .amdhsa_float_denorm_mode_32 (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|((((alignto(max(defined_value+4, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max(0, 1), 8))/8)-1)<<6))&196608)>>16
-// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|((((alignto(max(defined_value+4, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max(0, 1), 8))/8)-1)<<6))&786432)>>18
-// ASM-NEXT: .amdhsa_dx10_clamp (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|((((alignto(max(defined_value+4, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max(0, 1), 8))/8)-1)<<6))&2097152)>>21
-// ASM-NEXT: .amdhsa_ieee_mode (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|((((alignto(max(defined_value+4, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max(0, 1), 8))/8)-1)<<6))&8388608)>>23
-// ASM-NEXT: .amdhsa_fp16_overflow (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|((((alignto(max(defined_value+4, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max(0, 1), 8))/8)-1)<<6))&67108864)>>26
-// ASM-NEXT: .amdhsa_workgroup_processor_mode (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|((((alignto(max(defined_value+4, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max(0, 1), 8))/8)-1)<<6))&536870912)>>29
-// ASM-NEXT: .amdhsa_memory_ordered (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|((((alignto(max(defined_value+4, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max(0, 1), 8))/8)-1)<<6))&1073741824)>>30
-// ASM-NEXT: .amdhsa_forward_progress (((((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|((((alignto(max(defined_value+4, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max(0, 1), 8))/8)-1)<<6))&2147483648)>>31
+// ASM-NEXT: .amdhsa_float_round_mode_32 (((((((((((((((((((1621884928|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&12288)>>12
+// ASM-NEXT: .amdhsa_float_round_mode_16_64 (((((((((((((((((((1621884928|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&49152)>>14
+// ASM-NEXT: .amdhsa_float_denorm_mode_32 (((((((((((((((((((1621884928|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&196608)>>16
+// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 (((((((((((((((((((1621884928|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&786432)>>18
+// ASM-NEXT: .amdhsa_dx10_clamp (((((((((((((((((((1621884928|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&2097152)>>21
+// ASM-NEXT: .amdhsa_ieee_mode (((((((((((((((((((1621884928|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&8388608)>>23
+// ASM-NEXT: .amdhsa_fp16_overflow (((((((((((((((((((1621884928|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&67108864)>>26
+// ASM-NEXT: .amdhsa_workgroup_processor_mode (((((((((((((((((((1621884928|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&536870912)>>29
+// ASM-NEXT: .amdhsa_memory_ordered (((((((((((((((((((1621884928|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&1073741824)>>30
+// ASM-NEXT: .amdhsa_forward_progress (((((((((((((((((((1621884928|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&2147483648)>>31
 // ASM-NEXT: .amdhsa_shared_vgpr_count 0
-// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&16777216)>>24
-// ASM-NEXT: .amdhsa_exception_fp_denorm_src (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&33554432)>>25
-// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&67108864)>>26
-// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&134217728)>>27
-// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&268435456)>>28
-// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&536870912)>>29
-// ASM-NEXT: .amdhsa_exception_int_div_zero (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1073741824)>>30
+// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op (((((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|defined_boolean)&(~62))&16777216)>>24
+// ASM-NEXT: .amdhsa_exception_fp_denorm_src (((((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|defined_boolean)&(~62))&33554432)>>25
+// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero (((((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|defined_boolean)&(~62))&67108864)>>26
+// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow (((((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|defined_boolean)&(~62))&134217728)>>27
+// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow (((((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|defined_boolean)&(~62))&268435456)>>28
+// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact (((((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|defined_boolean)&(~62))&536870912)>>29
+// ASM-NEXT: .amdhsa_exception_int_div_zero (((((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|defined_boolean)&(~62))&1073741824)>>30
 // ASM-NEXT: .end_amdhsa_kernel
 
 // ASM:       .set defined_value, 41
diff --git a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx12.s b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx12.s
index a80000dc44dac0d..51d0fb30b320c52 100644
--- a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx12.s
+++ b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx12.s
@@ -108,38 +108,38 @@ expr_defined:
 // ASM-NEXT: .amdhsa_group_segment_fixed_size defined_value+2
 // ASM-NEXT: .amdhsa_private_segment_fixed_size defined_value+3
 // ASM-NEXT: .amdhsa_kernarg_size 0
-// ASM-NEXT: .amdhsa_user_sgpr_count (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&62)>>1
-// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&2)>>1
-// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&4)>>2
-// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&8)>>3
-// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&16)>>4
-// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&64)>>6
-// ASM-NEXT: .amdhsa_wavefront_size32 (((((0&(~1024))|(1<<10))&(~2048))|(defined_boolean<<11))&1024)>>10
-// ASM-NEXT: .amdhsa_enable_private_segment (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1)>>0
-// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&128)>>7
-// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&256)>>8
-// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&512)>>9
-// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1024)>>10
-// ASM-NEXT: .amdhsa_system_vgpr_workitem_id (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&6144)>>11
+// ASM-NEXT: .amdhsa_user_sgpr_count 0
+// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0
+// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 0
+// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0
+// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id 0
+// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 0
+// ASM-NEXT: .amdhsa_wavefront_size32 1
+// ASM-NEXT: .amdhsa_enable_private_segment ((((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|defined_boolean)&(~62))&1
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x (((((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|defined_boolean)&(~62))&128)>>7
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y (((((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|defined_boolean)&(~62))&256)>>8
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z (((((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|defined_boolean)&(~62))&512)>>9
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info (((((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|defined_boolean)&(~62))&1024)>>10
+// ASM-NEXT: .amdhsa_system_vgpr_workitem_id (((((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|defined_boolean)&(~62))&6144)>>11
 // ASM-NEXT: .amdhsa_next_free_vgpr defined_value+4
 // ASM-NEXT: .amdhsa_next_free_sgpr defined_value+5
 // ASM-NEXT: .amdhsa_reserve_vcc defined_boolean
-// ASM-NEXT: .amdhsa_float_round_mode_32 (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|((((alignto(max(defined_value+4, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max(0, 1), 8))/8)-1)<<6))&12288)>>12
-// ASM-NEXT: .amdhsa_float_round_mode_16_64 (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|((((alignto(max(defined_value+4, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max(0, 1), 8))/8)-1)<<6))&49152)>>14
-// ASM-NEXT: .amdhsa_float_denorm_mode_32 (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|((((alignto(max(defined_value+4, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max(0, 1), 8))/8)-1)<<6))&196608)>>16
-// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|((((alignto(max(defined_value+4, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max(0, 1), 8))/8)-1)<<6))&786432)>>18
-// ASM-NEXT: .amdhsa_fp16_overflow (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|((((alignto(max(defined_value+4, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max(0, 1), 8))/8)-1)<<6))&67108864)>>26
-// ASM-NEXT: .amdhsa_workgroup_processor_mode (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|((((alignto(max(defined_value+4, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max(0, 1), 8))/8)-1)<<6))&536870912)>>29
-// ASM-NEXT: .amdhsa_memory_ordered (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|((((alignto(max(defined_value+4, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max(0, 1), 8))/8)-1)<<6))&1073741824)>>30
-// ASM-NEXT: .amdhsa_forward_progress (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|((((alignto(max(defined_value+4, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max(0, 1), 8))/8)-1)<<6))&2147483648)>>31
-// ASM-NEXT: .amdhsa_round_robin_scheduling (((((((((((((((((((((((((((((0&(~786432))|(3<<18))&(~536870912))|(1<<29))&(~1073741824))|(1<<30))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|((((alignto(max(defined_value+4, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max(0, 1), 8))/8)-1)<<6))&2097152)>>21
-// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&16777216)>>24
-// ASM-NEXT: .amdhsa_exception_fp_denorm_src (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&33554432)>>25
-// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&67108864)>>26
-// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&134217728)>>27
-// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&268435456)>>28
-// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&536870912)>>29
-// ASM-NEXT: .amdhsa_exception_int_div_zero (((((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1073741824)>>30
+// ASM-NEXT: .amdhsa_float_round_mode_32 (((((((((((((((((((((1611399168|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&12288)>>12
+// ASM-NEXT: .amdhsa_float_round_mode_16_64 (((((((((((((((((((((1611399168|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&49152)>>14
+// ASM-NEXT: .amdhsa_float_denorm_mode_32 (((((((((((((((((((((1611399168|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&196608)>>16
+// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 (((((((((((((((((((((1611399168|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&786432)>>18
+// ASM-NEXT: .amdhsa_fp16_overflow (((((((((((((((((((((1611399168|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&67108864)>>26
+// ASM-NEXT: .amdhsa_workgroup_processor_mode (((((((((((((((((((((1611399168|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&536870912)>>29
+// ASM-NEXT: .amdhsa_memory_ordered (((((((((((((((((((((1611399168|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&1073741824)>>30
+// ASM-NEXT: .amdhsa_forward_progress (((((((((((((((((((((1611399168|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&2147483648)>>31
+// ASM-NEXT: .amdhsa_round_robin_scheduling (((((((((((((((((((((1611399168|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~67108864))|(defined_boolean<<26))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~2147483648))|(defined_boolean<<31))&(~2097152))|(defined_boolean<<21))&(~63))|(((alignto(max(defined_value+4, 1), 8))/8)-1))&(~960))&2097152)>>21
+// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op (((((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|defined_boolean)&(~62))&16777216)>>24
+// ASM-NEXT: .amdhsa_exception_fp_denorm_src (((((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|defined_boolean)&(~62))&33554432)>>25
+// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero (((((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|defined_boolean)&(~62))&67108864)>>26
+// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow (((((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|defined_boolean)&(~62))&134217728)>>27
+// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow (((((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|defined_boolean)&(~62))&268435456)>>28
+// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact (((((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|defined_boolean)&(~62))&536870912)>>29
+// ASM-NEXT: .amdhsa_exception_int_div_zero (((((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~1))|defined_boolean)&(~62))&1073741824)>>30
 // ASM-NEXT: .end_amdhsa_kernel
 
 // ASM:       .set defined_value, 41
diff --git a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx7.s b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx7.s
index 7ab2e2b28a0e6ec..485f48c695c4dea 100644
--- a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx7.s
+++ b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx7.s
@@ -98,37 +98,37 @@ expr_defined:
 // ASM-NEXT: .amdhsa_group_segment_fixed_size defined_value+2
 // ASM-NEXT: .amdhsa_private_segment_fixed_size defined_value+3
 // ASM-NEXT: .amdhsa_kernarg_size 0
-// ASM-NEXT: .amdhsa_user_sgpr_count (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&62)>>1
-// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer (((0&(~2048))|(defined_boolean<<11))&1)>>0
-// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr (((0&(~2048))|(defined_boolean<<11))&2)>>1
-// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr (((0&(~2048))|(defined_boolean<<11))&4)>>2
-// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr (((0&(~2048))|(defined_boolean<<11))&8)>>3
-// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id (((0&(~2048))|(defined_boolean<<11))&16)>>4
-// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init (((0&(~2048))|(defined_boolean<<11))&32)>>5
-// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size (((0&(~2048))|(defined_boolean<<11))&64)>>6
-// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1)>>0
-// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&128)>>7
-// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&256)>>8
-// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&512)>>9
-// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1024)>>10
-// ASM-NEXT: .amdhsa_system_vgpr_workitem_id (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&6144)>>11
+// ASM-NEXT: .amdhsa_user_sgpr_count 0
+// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer 0
+// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0
+// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 0
+// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0
+// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id 0
+// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init 0
+// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 0
+// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset ((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&1
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&128)>>7
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&256)>>8
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&512)>>9
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&1024)>>10
+// ASM-NEXT: .amdhsa_system_vgpr_workitem_id (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&6144)>>11
 // ASM-NEXT: .amdhsa_next_free_vgpr defined_value+4
 // ASM-NEXT: .amdhsa_next_free_sgpr defined_value+5
 // ASM-NEXT: .amdhsa_reserve_vcc defined_boolean
 // ASM-NEXT: .amdhsa_reserve_flat_scratch defined_boolean
-// ASM-NEXT: .amdhsa_float_round_mode_32 (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|((((alignto(max(defined_value+4, 1), 4))/4)-1)<<0))&(~960))|((((alignto(max((defined_value+5)+(extrasgprs(defined_boolean, defined_boolean, 0)), 1), 8))/8)-1)<<6))&12288)>>12
-// ASM-NEXT: .amdhsa_float_round_mode_16_64 (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|((((alignto(max(defined_value+4, 1), 4))/4)-1)<<0))&(~960))|((((alignto(max((defined_value+5)+(extrasgprs(defined_boolean, defined_boolean, 0)), 1), 8))/8)-1)<<6))&49152)>>14
-// ASM-NEXT: .amdhsa_float_denorm_mode_32 (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|((((alignto(max(defined_value+4, 1), 4))/4)-1)<<0))&(~960))|((((alignto(max((defined_value+5)+(extrasgprs(defined_boolean, defined_boolean, 0)), 1), 8))/8)-1)<<6))&196608)>>16
-// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|((((alignto(max(defined_value+4, 1), 4))/4)-1)<<0))&(~960))|((((alignto(max((defined_value+5)+(extrasgprs(defined_boolean, defined_boolean, 0)), 1), 8))/8)-1)<<6))&786432)>>18
-// ASM-NEXT: .amdhsa_dx10_clamp (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|((((alignto(max(defined_value+4, 1), 4))/4)-1)<<0))&(~960))|((((alignto(max((defined_value+5)+(extrasgprs(defined_boolean, defined_boolean, 0)), 1), 8))/8)-1)<<6))&2097152)>>21
-// ASM-NEXT: .amdhsa_ieee_mode (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|((((alignto(max(defined_value+4, 1), 4))/4)-1)<<0))&(~960))|((((alignto(max((defined_value+5)+(extrasgprs(defined_boolean, defined_boolean, 0)), 1), 8))/8)-1)<<6))&8388608)>>23
-// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&16777216)>>24
-// ASM-NEXT: .amdhsa_exception_fp_denorm_src (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&33554432)>>25
-// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&67108864)>>26
-// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&134217728)>>27
-// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&268435456)>>28
-// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&536870912)>>29
-// ASM-NEXT: .amdhsa_exception_int_div_zero (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1073741824)>>30
+// ASM-NEXT: .amdhsa_float_round_mode_32 ((((((((((((11272192|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(((alignto(max(defined_value+4, 1), 4))/4)-1))&(~960))|((((alignto(max((defined_value+5)+(extrasgprs(defined_boolean, defined_boolean, 0)), 1), 8))/8)-1)<<6))&12288)>>12
+// ASM-NEXT: .amdhsa_float_round_mode_16_64 ((((((((((((11272192|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(((alignto(max(defined_value+4, 1), 4))/4)-1))&(~960))|((((alignto(max((defined_value+5)+(extrasgprs(defined_boolean, defined_boolean, 0)), 1), 8))/8)-1)<<6))&49152)>>14
+// ASM-NEXT: .amdhsa_float_denorm_mode_32 ((((((((((((11272192|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(((alignto(max(defined_value+4, 1), 4))/4)-1))&(~960))|((((alignto(max((defined_value+5)+(extrasgprs(defined_boolean, defined_boolean, 0)), 1), 8))/8)-1)<<6))&196608)>>16
+// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 ((((((((((((11272192|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(((alignto(max(defined_value+4, 1), 4))/4)-1))&(~960))|((((alignto(max((defined_value+5)+(extrasgprs(defined_boolean, defined_boolean, 0)), 1), 8))/8)-1)<<6))&786432)>>18
+// ASM-NEXT: .amdhsa_dx10_clamp 1
+// ASM-NEXT: .amdhsa_ieee_mode 1
+// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&16777216)>>24
+// ASM-NEXT: .amdhsa_exception_fp_denorm_src (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&33554432)>>25
+// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&67108864)>>26
+// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&134217728)>>27
+// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&268435456)>>28
+// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&536870912)>>29
+// ASM-NEXT: .amdhsa_exception_int_div_zero (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&1073741824)>>30
 // ASM-NEXT: .end_amdhsa_kernel
 
 // ASM:       .set defined_value, 41
diff --git a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx8.s b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx8.s
index caccde7ba0e3b8e..0d2e066113ee861 100644
--- a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx8.s
+++ b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx8.s
@@ -99,38 +99,38 @@ expr_defined:
 // ASM-NEXT: .amdhsa_group_segment_fixed_size defined_value+2
 // ASM-NEXT: .amdhsa_private_segment_fixed_size defined_value+3
 // ASM-NEXT: .amdhsa_kernarg_size 0
-// ASM-NEXT: .amdhsa_user_sgpr_count (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&62)>>1
-// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer (((0&(~2048))|(defined_boolean<<11))&1)>>0
-// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr (((0&(~2048))|(defined_boolean<<11))&2)>>1
-// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr (((0&(~2048))|(defined_boolean<<11))&4)>>2
-// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr (((0&(~2048))|(defined_boolean<<11))&8)>>3
-// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id (((0&(~2048))|(defined_boolean<<11))&16)>>4
-// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init (((0&(~2048))|(defined_boolean<<11))&32)>>5
-// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size (((0&(~2048))|(defined_boolean<<11))&64)>>6
-// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1)>>0
-// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&128)>>7
-// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&256)>>8
-// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&512)>>9
-// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1024)>>10
-// ASM-NEXT: .amdhsa_system_vgpr_workitem_id (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&6144)>>11
+// ASM-NEXT: .amdhsa_user_sgpr_count 0
+// ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer 0
+// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0
+// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 0
+// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0
+// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id 0
+// ASM-NEXT: .amdhsa_user_sgpr_flat_scratch_init 0
+// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 0
+// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset ((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&1
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&128)>>7
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&256)>>8
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&512)>>9
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&1024)>>10
+// ASM-NEXT: .amdhsa_system_vgpr_workitem_id (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&6144)>>11
 // ASM-NEXT: .amdhsa_next_free_vgpr defined_value+4
 // ASM-NEXT: .amdhsa_next_free_sgpr defined_value+5
 // ASM-NEXT: .amdhsa_reserve_vcc defined_boolean
 // ASM-NEXT: .amdhsa_reserve_flat_scratch defined_boolean
 // ASM-NEXT: .amdhsa_reserve_xnack_mask 1
-// ASM-NEXT: .amdhsa_float_round_mode_32 (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|((((alignto(max(defined_value+4, 1), 4))/4)-1)<<0))&(~960))|((((alignto(max((defined_value+5)+(extrasgprs(defined_boolean, defined_boolean, 1)), 1), 8))/8)-1)<<6))&12288)>>12
-// ASM-NEXT: .amdhsa_float_round_mode_16_64 (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|((((alignto(max(defined_value+4, 1), 4))/4)-1)<<0))&(~960))|((((alignto(max((defined_value+5)+(extrasgprs(defined_boolean, defined_boolean, 1)), 1), 8))/8)-1)<<6))&49152)>>14
-// ASM-NEXT: .amdhsa_float_denorm_mode_32 (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|((((alignto(max(defined_value+4, 1), 4))/4)-1)<<0))&(~960))|((((alignto(max((defined_value+5)+(extrasgprs(defined_boolean, defined_boolean, 1)), 1), 8))/8)-1)<<6))&196608)>>16
-// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|((((alignto(max(defined_value+4, 1), 4))/4)-1)<<0))&(~960))|((((alignto(max((defined_value+5)+(extrasgprs(defined_boolean, defined_boolean, 1)), 1), 8))/8)-1)<<6))&786432)>>18
-// ASM-NEXT: .amdhsa_dx10_clamp (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|((((alignto(max(defined_value+4, 1), 4))/4)-1)<<0))&(~960))|((((alignto(max((defined_value+5)+(extrasgprs(defined_boolean, defined_boolean, 1)), 1), 8))/8)-1)<<6))&2097152)>>21
-// ASM-NEXT: .amdhsa_ieee_mode (((((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~12288))|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|((((alignto(max(defined_value+4, 1), 4))/4)-1)<<0))&(~960))|((((alignto(max((defined_value+5)+(extrasgprs(defined_boolean, defined_boolean, 1)), 1), 8))/8)-1)<<6))&8388608)>>23
-// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&16777216)>>24
-// ASM-NEXT: .amdhsa_exception_fp_denorm_src (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&33554432)>>25
-// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&67108864)>>26
-// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&134217728)>>27
-// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&268435456)>>28
-// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&536870912)>>29
-// ASM-NEXT: .amdhsa_exception_int_div_zero (((((((((((((((((((((((((((((0&(~128))|(1<<7))&(~6144))|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))|(0<<1))&1073741824)>>30
+// ASM-NEXT: .amdhsa_float_round_mode_32 ((((((((((((11272192|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(((alignto(max(defined_value+4, 1), 4))/4)-1))&(~960))|((((alignto(max((defined_value+5)+(extrasgprs(defined_boolean, defined_boolean, 1)), 1), 8))/8)-1)<<6))&12288)>>12
+// ASM-NEXT: .amdhsa_float_round_mode_16_64 ((((((((((((11272192|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(((alignto(max(defined_value+4, 1), 4))/4)-1))&(~960))|((((alignto(max((defined_value+5)+(extrasgprs(defined_boolean, defined_boolean, 1)), 1), 8))/8)-1)<<6))&49152)>>14
+// ASM-NEXT: .amdhsa_float_denorm_mode_32 ((((((((((((11272192|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(((alignto(max(defined_value+4, 1), 4))/4)-1))&(~960))|((((alignto(max((defined_value+5)+(extrasgprs(defined_boolean, defined_boolean, 1)), 1), 8))/8)-1)<<6))&196608)>>16
+// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 ((((((((((((11272192|(defined_2_bits<<12))&(~49152))|(defined_2_bits<<14))&(~196608))|(defined_2_bits<<16))&(~786432))|(defined_2_bits<<18))&(~63))|(((alignto(max(defined_value+4, 1), 4))/4)-1))&(~960))|((((alignto(max((defined_value+5)+(extrasgprs(defined_boolean, defined_boolean, 1)), 1), 8))/8)-1)<<6))&786432)>>18
+// ASM-NEXT: .amdhsa_dx10_clamp 1
+// ASM-NEXT: .amdhsa_ieee_mode 1
+// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&16777216)>>24
+// ASM-NEXT: .amdhsa_exception_fp_denorm_src (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&33554432)>>25
+// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&67108864)>>26
+// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&134217728)>>27
+// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&268435456)>>28
+// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&536870912)>>29
+// ASM-NEXT: .amdhsa_exception_int_div_zero (((((((((((((((((((((((((128|(defined_2_bits<<11))&(~128))|(defined_boolean<<7))&(~256))|(defined_boolean<<8))&(~512))|(defined_boolean<<9))&(~1024))|(defined_boolean<<10))&(~16777216))|(defined_boolean<<24))&(~33554432))|(defined_boolean<<25))&(~67108864))|(defined_boolean<<26))&(~134217728))|(defined_boolean<<27))&(~268435456))|(defined_boolean<<28))&(~536870912))|(defined_boolean<<29))&(~1073741824))|(defined_boolean<<30))&(~62))&1073741824)>>30
 // ASM-NEXT: .end_amdhsa_kernel
 
 // ASM:       .set defined_value, 41
diff --git a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx90a.s b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx90a.s
index 216ae4c42a3d9a9..88b5e23a6f2c5f1 100644
--- a/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx90a.s
+++ b/llvm/test/MC/AMDGPU/hsa-sym-exprs-gfx90a.s
@@ -70,7 +70,7 @@ expr_defined:
 // ASM-NEXT: .amdhsa_group_segment_fixed_size 0
 // ASM-NEXT: .amdhsa_private_segment_fixed_size 0
 // ASM-NEXT: .amdhsa_kernarg_size 0
-// ASM-NEXT: .amdhsa_user_sgpr_count (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&62)>>1
+// ASM-NEXT: .amdhsa_user_sgpr_count 0
 // ASM-NEXT: .amdhsa_user_sgpr_private_segment_buffer 0
 // ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0
 // ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 0
@@ -80,33 +80,33 @@ expr_defined:
 // ASM-NEXT: .amdhsa_user_sgpr_kernarg_preload_length 0
 // ASM-NEXT: .amdhsa_user_sgpr_kernarg_preload_offset 0
 // ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 0
-// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1)>>0
-// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&128)>>7
-// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&256)>>8
-// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&512)>>9
-// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1024)>>10
-// ASM-NEXT: .amdhsa_system_vgpr_workitem_id (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&6144)>>11
+// ASM-NEXT: .amdhsa_system_sgpr_private_segment_wavefront_offset ((128|defined_boolean)&(~62))&1
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x 1
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y (((128|defined_boolean)&(~62))&256)>>8
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z (((128|defined_boolean)&(~62))&512)>>9
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info (((128|defined_boolean)&(~62))&1024)>>10
+// ASM-NEXT: .amdhsa_system_vgpr_workitem_id (((128|defined_boolean)&(~62))&6144)>>11
 // ASM-NEXT: .amdhsa_next_free_vgpr defined_boolean+1
 // ASM-NEXT: .amdhsa_next_free_sgpr defined_boolean+2
-// ASM-NEXT: .amdhsa_accum_offset (((((((0&(~65536))|(defined_boolean<<16))&(~63))|(((4/4)-1)<<0))&63)>>0)+1)*4
+// ASM-NEXT: .amdhsa_accum_offset 4
 // ASM-NEXT: .amdhsa_reserve_vcc defined_boolean
 // ASM-NEXT: .amdhsa_reserve_flat_scratch defined_boolean
 // ASM-NEXT: .amdhsa_reserve_xnack_mask 1
-// ASM-NEXT: .amdhsa_float_round_mode_32 (((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~2097152))|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|((((alignto(max(defined_boolean+1, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max((defined_boolean+2)+(extrasgprs(defined_boolean, defined_boolean, 1)), 1), 8))/8)-1)<<6))&12288)>>12
-// ASM-NEXT: .amdhsa_float_round_mode_16_64 (((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~2097152))|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|((((alignto(max(defined_boolean+1, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max((defined_boolean+2)+(extrasgprs(defined_boolean, defined_boolean, 1)), 1), 8))/8)-1)<<6))&49152)>>14
-// ASM-NEXT: .amdhsa_float_denorm_mode_32 (((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~2097152))|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|((((alignto(max(defined_boolean+1, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max((defined_boolean+2)+(extrasgprs(defined_boolean, defined_boolean, 1)), 1), 8))/8)-1)<<6))&196608)>>16
-// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 (((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~2097152))|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|((((alignto(max(defined_boolean+1, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max((defined_boolean+2)+(extrasgprs(defined_boolean, defined_boolean, 1)), 1), 8))/8)-1)<<6))&786432)>>18
-// ASM-NEXT: .amdhsa_dx10_clamp (((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~2097152))|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|((((alignto(max(defined_boolean+1, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max((defined_boolean+2)+(extrasgprs(defined_boolean, defined_boolean, 1)), 1), 8))/8)-1)<<6))&2097152)>>21
-// ASM-NEXT: .amdhsa_ieee_mode (((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~2097152))|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|((((alignto(max(defined_boolean+1, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max((defined_boolean+2)+(extrasgprs(defined_boolean, defined_boolean, 1)), 1), 8))/8)-1)<<6))&8388608)>>23
-// ASM-NEXT: .amdhsa_fp16_overflow (((((((((((((((((0&(~786432))|(3<<18))&(~2097152))|(1<<21))&(~8388608))|(1<<23))&(~2097152))|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|((((alignto(max(defined_boolean+1, 1), 8))/8)-1)<<0))&(~960))|((((alignto(max((defined_boolean+2)+(extrasgprs(defined_boolean, defined_boolean, 1)), 1), 8))/8)-1)<<6))&67108864)>>26
-// ASM-NEXT: .amdhsa_tg_split (((((0&(~65536))|(defined_boolean<<16))&(~63))|(((4/4)-1)<<0))&65536)>>16 
-// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&16777216)>>24
-// ASM-NEXT: .amdhsa_exception_fp_denorm_src (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&33554432)>>25
-// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&67108864)>>26
-// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&134217728)>>27
-// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&268435456)>>28
-// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&536870912)>>29
-// ASM-NEXT: .amdhsa_exception_int_div_zero (((((((0&(~128))|(1<<7))&(~1))|(defined_boolean<<0))&(~62))|(0<<1))&1073741824)>>30
+// ASM-NEXT: .amdhsa_float_round_mode_32 ((((((((((9175040|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|(((alignto(max(defined_boolean+1, 1), 8))/8)-1))&(~960))|((((alignto(max((defined_boolean+2)+(extrasgprs(defined_boolean, defined_boolean, 1)), 1), 8))/8)-1)<<6))&12288)>>12
+// ASM-NEXT: .amdhsa_float_round_mode_16_64 ((((((((((9175040|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|(((alignto(max(defined_boolean+1, 1), 8))/8)-1))&(~960))|((((alignto(max((defined_boolean+2)+(extrasgprs(defined_boolean, defined_boolean, 1)), 1), 8))/8)-1)<<6))&49152)>>14
+// ASM-NEXT: .amdhsa_float_denorm_mode_32 ((((((((((9175040|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|(((alignto(max(defined_boolean+1, 1), 8))/8)-1))&(~960))|((((alignto(max((defined_boolean+2)+(extrasgprs(defined_boolean, defined_boolean, 1)), 1), 8))/8)-1)<<6))&196608)>>16
+// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 3
+// ASM-NEXT: .amdhsa_dx10_clamp ((((((((((9175040|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|(((alignto(max(defined_boolean+1, 1), 8))/8)-1))&(~960))|((((alignto(max((defined_boolean+2)+(extrasgprs(defined_boolean, defined_boolean, 1)), 1), 8))/8)-1)<<6))&2097152)>>21
+// ASM-NEXT: .amdhsa_ieee_mode ((((((((((9175040|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|(((alignto(max(defined_boolean+1, 1), 8))/8)-1))&(~960))|((((alignto(max((defined_boolean+2)+(extrasgprs(defined_boolean, defined_boolean, 1)), 1), 8))/8)-1)<<6))&8388608)>>23
+// ASM-NEXT: .amdhsa_fp16_overflow ((((((((((9175040|(defined_boolean<<21))&(~8388608))|(defined_boolean<<23))&(~67108864))|(defined_boolean<<26))&(~63))|(((alignto(max(defined_boolean+1, 1), 8))/8)-1))&(~960))|((((alignto(max((defined_boolean+2)+(extrasgprs(defined_boolean, defined_boolean, 1)), 1), 8))/8)-1)<<6))&67108864)>>26
+// ASM-NEXT: .amdhsa_tg_split (((defined_boolean<<16)&(~63))&65536)>>16
+// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op (((128|defined_boolean)&(~62))&16777216)>>24
+// ASM-NEXT: .amdhsa_exception_fp_denorm_src (((128|defined_boolean)&(~62))&33554432)>>25
+// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero (((128|defined_boolean)&(~62))&67108864)>>26
+// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow (((128|defined_boolean)&(~62))&134217728)>>27
+// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow (((128|defined_boolean)&(~62))&268435456)>>28
+// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact (((128|defined_boolean)&(~62))&536870912)>>29
+// ASM-NEXT: .amdhsa_exception_int_div_zero (((128|defined_boolean)&(~62))&1073741824)>>30
 // ASM-NEXT: .end_amdhsa_kernel
 
 // ASM:       .set defined_boolean, 1

From d7aeea626dac64449fc67cf8ddf8f326a0157d91 Mon Sep 17 00:00:00 2001
From: Lukacma <Marian.Lukac@arm.com>
Date: Thu, 15 Aug 2024 13:52:35 +0100
Subject: [PATCH 021/441] [AArch64] optimise SVE prefetch intrinsics with no
 active lanes (#103052)

This patch extends https://github.com/llvm/llvm-project/pull/73964 and
optimises away SVE prefetch intrinsics when predicate is zero.
---
 .../AArch64/AArch64TargetTransformInfo.cpp    |  18 ++
 .../sve-intrinsic-comb-no-active-lanes-prf.ll | 156 ++++++++++++++++++
 2 files changed, 174 insertions(+)
 create mode 100644 llvm/test/Transforms/InstCombine/AArch64/sve-intrinsic-comb-no-active-lanes-prf.ll

diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index a416565392eabec..a782c9c43512379 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -2165,6 +2165,24 @@ AArch64TTIImpl::instCombineIntrinsic(InstCombiner &IC,
   case Intrinsic::aarch64_sve_ldnt1_gather_scalar_offset:
   case Intrinsic::aarch64_sve_ldnt1_gather_uxtw:
     return instCombineSVENoActiveUnaryZero(IC, II);
+  case Intrinsic::aarch64_sve_prf:
+  case Intrinsic::aarch64_sve_prfb_gather_index:
+  case Intrinsic::aarch64_sve_prfb_gather_scalar_offset:
+  case Intrinsic::aarch64_sve_prfb_gather_sxtw_index:
+  case Intrinsic::aarch64_sve_prfb_gather_uxtw_index:
+  case Intrinsic::aarch64_sve_prfd_gather_index:
+  case Intrinsic::aarch64_sve_prfd_gather_scalar_offset:
+  case Intrinsic::aarch64_sve_prfd_gather_sxtw_index:
+  case Intrinsic::aarch64_sve_prfd_gather_uxtw_index:
+  case Intrinsic::aarch64_sve_prfh_gather_index:
+  case Intrinsic::aarch64_sve_prfh_gather_scalar_offset:
+  case Intrinsic::aarch64_sve_prfh_gather_sxtw_index:
+  case Intrinsic::aarch64_sve_prfh_gather_uxtw_index:
+  case Intrinsic::aarch64_sve_prfw_gather_index:
+  case Intrinsic::aarch64_sve_prfw_gather_scalar_offset:
+  case Intrinsic::aarch64_sve_prfw_gather_sxtw_index:
+  case Intrinsic::aarch64_sve_prfw_gather_uxtw_index:
+    return instCombineSVENoActiveUnaryErase(IC, II, 0);
   case Intrinsic::aarch64_neon_fmaxnm:
   case Intrinsic::aarch64_neon_fminnm:
     return instCombineMaxMinNM(IC, II);
diff --git a/llvm/test/Transforms/InstCombine/AArch64/sve-intrinsic-comb-no-active-lanes-prf.ll b/llvm/test/Transforms/InstCombine/AArch64/sve-intrinsic-comb-no-active-lanes-prf.ll
new file mode 100644
index 000000000000000..1de582dff52dad5
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/AArch64/sve-intrinsic-comb-no-active-lanes-prf.ll
@@ -0,0 +1,156 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -passes=instcombine < %s | FileCheck %s
+target triple = "aarch64-unknown-linux-gnu"
+
+define void @test_prf(ptr %base){
+; CHECK-LABEL: define void @test_prf(
+; CHECK-SAME: ptr [[BASE:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  tail call void @llvm.aarch64.sve.prf.nxv16i1(<vscale x 16 x i1> zeroinitializer, ptr %base, i32 1)
+  ret void
+}
+
+define void @test_prfb_gather_index(ptr %base, <vscale x 2 x i64> %indexes){
+; CHECK-LABEL: define void @test_prfb_gather_index(
+; CHECK-SAME: ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDEXES:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.aarch64.sve.prfb.gather.index.nx2vi64(<vscale x 2 x i1> zeroinitializer, ptr %base, <vscale x 2 x i64> %indexes, i32 1)
+  ret void
+}
+
+define void @test_prfb_gather_scalar_offset(<vscale x 4 x i32> %bases){
+; CHECK-LABEL: define void @test_prfb_gather_scalar_offset(
+; CHECK-SAME: <vscale x 4 x i32> [[BASES:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.aarch64.sve.prfb.gather.scalar.offset.nx4vi32(<vscale x 4 x i1> zeroinitializer, <vscale x 4 x i32> %bases, i64 7, i32 1)
+  ret void
+}
+
+define void @test_prfb_gather_sxtw_index(ptr %base, <vscale x 4 x i32> %indexes){
+; CHECK-LABEL: define void @test_prfb_gather_sxtw_index(
+; CHECK-SAME: ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDEXES:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.aarch64.sve.prfb.gather.sxtw.index.nx4vi32(<vscale x 4 x i1> zeroinitializer, ptr %base, <vscale x 4 x i32> %indexes, i32 1)
+  ret void
+}
+
+define void @test_prfb_gather_uxtw_index(ptr %base, <vscale x 4 x i32> %indexes){
+; CHECK-LABEL: define void @test_prfb_gather_uxtw_index(
+; CHECK-SAME: ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDEXES:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.aarch64.sve.prfb.gather.uxtw.index.nx4vi32(<vscale x 4 x i1> zeroinitializer, ptr %base, <vscale x 4 x i32> %indexes, i32 1)
+  ret void
+}
+
+define void @test_prfd_gather_index(ptr %base, <vscale x 2 x i64> %indexes){
+; CHECK-LABEL: define void @test_prfd_gather_index(
+; CHECK-SAME: ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDEXES:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.aarch64.sve.prfd.gather.index.nx2vi64(<vscale x 2 x i1> zeroinitializer, ptr %base, <vscale x 2 x i64> %indexes, i32 1)
+  ret void
+}
+
+define void @test_prfd_gather_scalar_offset(<vscale x 4 x i32> %bases){
+; CHECK-LABEL: define void @test_prfd_gather_scalar_offset(
+; CHECK-SAME: <vscale x 4 x i32> [[BASES:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.aarch64.sve.prfd.gather.scalar.offset.nx4vi32(<vscale x 4 x i1> zeroinitializer, <vscale x 4 x i32> %bases, i64 7, i32 1)
+  ret void
+}
+
+define void @test_prfd_gather_sxtw_index(ptr %base, <vscale x 4 x i32> %indexes){
+; CHECK-LABEL: define void @test_prfd_gather_sxtw_index(
+; CHECK-SAME: ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDEXES:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.aarch64.sve.prfd.gather.sxtw.index.nx4vi32(<vscale x 4 x i1> zeroinitializer, ptr %base, <vscale x 4 x i32> %indexes, i32 1)
+  ret void
+}
+
+define void @test_prfd_gather_uxtw_index(ptr %base, <vscale x 4 x i32> %indexes){
+; CHECK-LABEL: define void @test_prfd_gather_uxtw_index(
+; CHECK-SAME: ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDEXES:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.aarch64.sve.prfd.gather.uxtw.index.nx4vi32(<vscale x 4 x i1> zeroinitializer, ptr %base, <vscale x 4 x i32> %indexes, i32 1)
+  ret void
+}
+
+define void @test_prfh_gather_index(ptr %base, <vscale x 2 x i64> %indexes){
+; CHECK-LABEL: define void @test_prfh_gather_index(
+; CHECK-SAME: ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDEXES:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.aarch64.sve.prfh.gather.index.nx2vi64(<vscale x 2 x i1> zeroinitializer, ptr %base, <vscale x 2 x i64> %indexes, i32 1)
+  ret void
+}
+
+define void @test_prfh_gather_scalar_offset(<vscale x 4 x i32> %bases){
+; CHECK-LABEL: define void @test_prfh_gather_scalar_offset(
+; CHECK-SAME: <vscale x 4 x i32> [[BASES:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.aarch64.sve.prfh.gather.scalar.offset.nx4vi32(<vscale x 4 x i1> zeroinitializer, <vscale x 4 x i32> %bases, i64 7, i32 1)
+  ret void
+}
+
+define void @test_prfh_gather_sxtw_index(ptr %base, <vscale x 4 x i32> %indexes){
+; CHECK-LABEL: define void @test_prfh_gather_sxtw_index(
+; CHECK-SAME: ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDEXES:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.aarch64.sve.prfh.gather.sxtw.index.nx4vi32(<vscale x 4 x i1> zeroinitializer, ptr %base, <vscale x 4 x i32> %indexes, i32 1)
+  ret void
+}
+
+define void @test_prfh_gather_uxtw_index(ptr %base, <vscale x 4 x i32> %indexes){
+; CHECK-LABEL: define void @test_prfh_gather_uxtw_index(
+; CHECK-SAME: ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDEXES:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.aarch64.sve.prfh.gather.uxtw.index.nx4vi32(<vscale x 4 x i1> zeroinitializer, ptr %base, <vscale x 4 x i32> %indexes, i32 1)
+  ret void
+}
+
+define void @test_prfw_gather_index(ptr %base, <vscale x 2 x i64> %indexes){
+; CHECK-LABEL: define void @test_prfw_gather_index(
+; CHECK-SAME: ptr [[BASE:%.*]], <vscale x 2 x i64> [[INDEXES:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.aarch64.sve.prfw.gather.index.nx2vi64(<vscale x 2 x i1> zeroinitializer, ptr %base, <vscale x 2 x i64> %indexes, i32 1)
+  ret void
+}
+
+define void @test_prfw_gather_scalar_offset(<vscale x 4 x i32> %bases){
+; CHECK-LABEL: define void @test_prfw_gather_scalar_offset(
+; CHECK-SAME: <vscale x 4 x i32> [[BASES:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.aarch64.sve.prfw.gather.scalar.offset.nx4vi32(<vscale x 4 x i1> zeroinitializer, <vscale x 4 x i32> %bases, i64 7, i32 1)
+  ret void
+}
+
+define void @test_prfw_gather_sxtw_index(ptr %base, <vscale x 4 x i32> %indexes){
+; CHECK-LABEL: define void @test_prfw_gather_sxtw_index(
+; CHECK-SAME: ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDEXES:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.aarch64.sve.prfw.gather.sxtw.index.nx4vi32(<vscale x 4 x i1> zeroinitializer, ptr %base, <vscale x 4 x i32> %indexes, i32 1)
+  ret void
+}
+
+define void @test_prfw_gather_uxtw_index(ptr %base, <vscale x 4 x i32> %indexes){
+; CHECK-LABEL: define void @test_prfw_gather_uxtw_index(
+; CHECK-SAME: ptr [[BASE:%.*]], <vscale x 4 x i32> [[INDEXES:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.aarch64.sve.prfw.gather.uxtw.index.nx4vi32(<vscale x 4 x i1> zeroinitializer, ptr %base, <vscale x 4 x i32> %indexes, i32 1)
+  ret void
+}

From 82cf6558e50ea7fe024264cc2fb76ca20450fb82 Mon Sep 17 00:00:00 2001
From: WANG Rui <wangrui@loongson.cn>
Date: Thu, 15 Aug 2024 17:14:48 +0800
Subject: [PATCH 022/441] [LoongArch] Pre-commit tests for validating the merge
 base offset in vecotrs. NFC

---
 .../CodeGen/LoongArch/merge-base-offset.ll    | 201 ++++++++++++++++--
 1 file changed, 188 insertions(+), 13 deletions(-)

diff --git a/llvm/test/CodeGen/LoongArch/merge-base-offset.ll b/llvm/test/CodeGen/LoongArch/merge-base-offset.ll
index 32a4c4bdd1508a9..58a8e5d77c63fe7 100644
--- a/llvm/test/CodeGen/LoongArch/merge-base-offset.ll
+++ b/llvm/test/CodeGen/LoongArch/merge-base-offset.ll
@@ -1,9 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc --mtriple=loongarch32 --mattr=+d --verify-machineinstrs < %s \
+; RUN: llc --mtriple=loongarch32 --mattr=+lasx --verify-machineinstrs < %s \
 ; RUN:   | FileCheck --check-prefix=LA32 %s
-; RUN: llc --mtriple=loongarch64 --mattr=+d --verify-machineinstrs < %s \
+; RUN: llc --mtriple=loongarch64 --mattr=+lasx --verify-machineinstrs < %s \
 ; RUN:   | FileCheck --check-prefix=LA64 %s
-; RUN: llc --mtriple=loongarch64 --mattr=+d --verify-machineinstrs \
+; RUN: llc --mtriple=loongarch64 --mattr=+lasx --verify-machineinstrs \
 ; RUN:   --code-model=large < %s | FileCheck --check-prefix=LA64-LARGE %s
 
 @g_i8 = dso_local global i8 0
@@ -405,10 +405,7 @@ define dso_local void @store_f64() nounwind {
 ; LA32-LABEL: store_f64:
 ; LA32:       # %bb.0: # %entry
 ; LA32-NEXT:    pcalau12i $a0, %pc_hi20(g_f64)
-; LA32-NEXT:    addi.w $a1, $zero, 1
-; LA32-NEXT:    movgr2fr.w $fa0, $a1
-; LA32-NEXT:    ffint.s.w $fa0, $fa0
-; LA32-NEXT:    fcvt.d.s $fa0, $fa0
+; LA32-NEXT:    vldi $vr0, -912
 ; LA32-NEXT:    fst.d $fa0, $a0, %pc_lo12(g_f64)
 ; LA32-NEXT:    ret
 ;
@@ -538,6 +535,184 @@ entry:
   ret void
 }
 
+@g_i32x4_src = dso_local global [4 x i32] zeroinitializer, align 16
+@g_i32x4_dst = dso_local global [4 x i32] zeroinitializer, align 16
+
+define dso_local void @copy_i32x4() nounwind {
+; LA32-LABEL: copy_i32x4:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    pcalau12i $a0, %pc_hi20(g_i32x4_src)
+; LA32-NEXT:    addi.w $a0, $a0, %pc_lo12(g_i32x4_src)
+; LA32-NEXT:    vld $vr0, $a0, 0
+; LA32-NEXT:    pcalau12i $a0, %pc_hi20(g_i32x4_dst)
+; LA32-NEXT:    addi.w $a0, $a0, %pc_lo12(g_i32x4_dst)
+; LA32-NEXT:    vst $vr0, $a0, 0
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: copy_i32x4:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    pcalau12i $a0, %pc_hi20(g_i32x4_src)
+; LA64-NEXT:    addi.d $a0, $a0, %pc_lo12(g_i32x4_src)
+; LA64-NEXT:    vld $vr0, $a0, 0
+; LA64-NEXT:    pcalau12i $a0, %pc_hi20(g_i32x4_dst)
+; LA64-NEXT:    addi.d $a0, $a0, %pc_lo12(g_i32x4_dst)
+; LA64-NEXT:    vst $vr0, $a0, 0
+; LA64-NEXT:    ret
+;
+; LA64-LARGE-LABEL: copy_i32x4:
+; LA64-LARGE:       # %bb.0: # %entry
+; LA64-LARGE-NEXT:    pcalau12i $a0, %pc_hi20(g_i32x4_src)
+; LA64-LARGE-NEXT:    addi.d $a1, $zero, %pc_lo12(g_i32x4_src)
+; LA64-LARGE-NEXT:    lu32i.d $a1, %pc64_lo20(g_i32x4_src)
+; LA64-LARGE-NEXT:    lu52i.d $a1, $a1, %pc64_hi12(g_i32x4_src)
+; LA64-LARGE-NEXT:    add.d $a0, $a1, $a0
+; LA64-LARGE-NEXT:    vld $vr0, $a0, 0
+; LA64-LARGE-NEXT:    pcalau12i $a0, %pc_hi20(g_i32x4_dst)
+; LA64-LARGE-NEXT:    addi.d $a1, $zero, %pc_lo12(g_i32x4_dst)
+; LA64-LARGE-NEXT:    lu32i.d $a1, %pc64_lo20(g_i32x4_dst)
+; LA64-LARGE-NEXT:    lu52i.d $a1, $a1, %pc64_hi12(g_i32x4_dst)
+; LA64-LARGE-NEXT:    add.d $a0, $a1, $a0
+; LA64-LARGE-NEXT:    vst $vr0, $a0, 0
+; LA64-LARGE-NEXT:    ret
+entry:
+  %0 = load <4 x i32>, ptr @g_i32x4_src, align 16
+  store <4 x i32> %0, ptr @g_i32x4_dst, align 16
+  ret void
+}
+
+@g_i32x8_src = dso_local global [8 x i32] zeroinitializer, align 32
+@g_i32x8_dst = dso_local global [8 x i32] zeroinitializer, align 32
+
+define dso_local void @copy_i32x8() nounwind {
+; LA32-LABEL: copy_i32x8:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    pcalau12i $a0, %pc_hi20(g_i32x4_src)
+; LA32-NEXT:    addi.w $a0, $a0, %pc_lo12(g_i32x4_src)
+; LA32-NEXT:    xvld $xr0, $a0, 0
+; LA32-NEXT:    pcalau12i $a0, %pc_hi20(g_i32x4_dst)
+; LA32-NEXT:    addi.w $a0, $a0, %pc_lo12(g_i32x4_dst)
+; LA32-NEXT:    xvst $xr0, $a0, 0
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: copy_i32x8:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    pcalau12i $a0, %pc_hi20(g_i32x4_src)
+; LA64-NEXT:    addi.d $a0, $a0, %pc_lo12(g_i32x4_src)
+; LA64-NEXT:    xvld $xr0, $a0, 0
+; LA64-NEXT:    pcalau12i $a0, %pc_hi20(g_i32x4_dst)
+; LA64-NEXT:    addi.d $a0, $a0, %pc_lo12(g_i32x4_dst)
+; LA64-NEXT:    xvst $xr0, $a0, 0
+; LA64-NEXT:    ret
+;
+; LA64-LARGE-LABEL: copy_i32x8:
+; LA64-LARGE:       # %bb.0: # %entry
+; LA64-LARGE-NEXT:    pcalau12i $a0, %pc_hi20(g_i32x4_src)
+; LA64-LARGE-NEXT:    addi.d $a1, $zero, %pc_lo12(g_i32x4_src)
+; LA64-LARGE-NEXT:    lu32i.d $a1, %pc64_lo20(g_i32x4_src)
+; LA64-LARGE-NEXT:    lu52i.d $a1, $a1, %pc64_hi12(g_i32x4_src)
+; LA64-LARGE-NEXT:    add.d $a0, $a1, $a0
+; LA64-LARGE-NEXT:    xvld $xr0, $a0, 0
+; LA64-LARGE-NEXT:    pcalau12i $a0, %pc_hi20(g_i32x4_dst)
+; LA64-LARGE-NEXT:    addi.d $a1, $zero, %pc_lo12(g_i32x4_dst)
+; LA64-LARGE-NEXT:    lu32i.d $a1, %pc64_lo20(g_i32x4_dst)
+; LA64-LARGE-NEXT:    lu52i.d $a1, $a1, %pc64_hi12(g_i32x4_dst)
+; LA64-LARGE-NEXT:    add.d $a0, $a1, $a0
+; LA64-LARGE-NEXT:    xvst $xr0, $a0, 0
+; LA64-LARGE-NEXT:    ret
+entry:
+  %0 = load <8 x i32>, ptr @g_i32x4_src, align 32
+  store <8 x i32> %0, ptr @g_i32x4_dst, align 32
+  ret void
+}
+
+@g_i8x16 = dso_local global <16 x i8> zeroinitializer, align 16
+
+define void @copy_i8_to_i8x16() {
+; LA32-LABEL: copy_i8_to_i8x16:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    pcalau12i $a0, %pc_hi20(g_i8)
+; LA32-NEXT:    addi.w $a0, $a0, %pc_lo12(g_i8)
+; LA32-NEXT:    vldrepl.b $vr0, $a0, 0
+; LA32-NEXT:    pcalau12i $a0, %pc_hi20(g_i8x16)
+; LA32-NEXT:    addi.w $a0, $a0, %pc_lo12(g_i8x16)
+; LA32-NEXT:    vst $vr0, $a0, 0
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: copy_i8_to_i8x16:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    pcalau12i $a0, %pc_hi20(g_i8)
+; LA64-NEXT:    addi.d $a0, $a0, %pc_lo12(g_i8)
+; LA64-NEXT:    vldrepl.b $vr0, $a0, 0
+; LA64-NEXT:    pcalau12i $a0, %pc_hi20(g_i8x16)
+; LA64-NEXT:    addi.d $a0, $a0, %pc_lo12(g_i8x16)
+; LA64-NEXT:    vst $vr0, $a0, 0
+; LA64-NEXT:    ret
+;
+; LA64-LARGE-LABEL: copy_i8_to_i8x16:
+; LA64-LARGE:       # %bb.0: # %entry
+; LA64-LARGE-NEXT:    pcalau12i $a0, %pc_hi20(g_i8)
+; LA64-LARGE-NEXT:    addi.d $a1, $zero, %pc_lo12(g_i8)
+; LA64-LARGE-NEXT:    lu32i.d $a1, %pc64_lo20(g_i8)
+; LA64-LARGE-NEXT:    lu52i.d $a1, $a1, %pc64_hi12(g_i8)
+; LA64-LARGE-NEXT:    add.d $a0, $a1, $a0
+; LA64-LARGE-NEXT:    vldrepl.b $vr0, $a0, 0
+; LA64-LARGE-NEXT:    pcalau12i $a0, %pc_hi20(g_i8x16)
+; LA64-LARGE-NEXT:    addi.d $a1, $zero, %pc_lo12(g_i8x16)
+; LA64-LARGE-NEXT:    lu32i.d $a1, %pc64_lo20(g_i8x16)
+; LA64-LARGE-NEXT:    lu52i.d $a1, $a1, %pc64_hi12(g_i8x16)
+; LA64-LARGE-NEXT:    add.d $a0, $a1, $a0
+; LA64-LARGE-NEXT:    vst $vr0, $a0, 0
+; LA64-LARGE-NEXT:    ret
+entry:
+  %0 = call <16 x i8> @llvm.loongarch.lsx.vldrepl.b(ptr @g_i8, i32 0)
+  store <16 x i8> %0, ptr @g_i8x16, align 16
+  ret void
+}
+
+@g_i8x32 = dso_local global <32 x i8> zeroinitializer, align 32
+
+define void @copy_i8_to_i8x32() {
+; LA32-LABEL: copy_i8_to_i8x32:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    pcalau12i $a0, %pc_hi20(g_i8)
+; LA32-NEXT:    addi.w $a0, $a0, %pc_lo12(g_i8)
+; LA32-NEXT:    xvldrepl.b $xr0, $a0, 0
+; LA32-NEXT:    pcalau12i $a0, %pc_hi20(g_i8x32)
+; LA32-NEXT:    addi.w $a0, $a0, %pc_lo12(g_i8x32)
+; LA32-NEXT:    xvst $xr0, $a0, 0
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: copy_i8_to_i8x32:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    pcalau12i $a0, %pc_hi20(g_i8)
+; LA64-NEXT:    addi.d $a0, $a0, %pc_lo12(g_i8)
+; LA64-NEXT:    xvldrepl.b $xr0, $a0, 0
+; LA64-NEXT:    pcalau12i $a0, %pc_hi20(g_i8x32)
+; LA64-NEXT:    addi.d $a0, $a0, %pc_lo12(g_i8x32)
+; LA64-NEXT:    xvst $xr0, $a0, 0
+; LA64-NEXT:    ret
+;
+; LA64-LARGE-LABEL: copy_i8_to_i8x32:
+; LA64-LARGE:       # %bb.0: # %entry
+; LA64-LARGE-NEXT:    pcalau12i $a0, %pc_hi20(g_i8)
+; LA64-LARGE-NEXT:    addi.d $a1, $zero, %pc_lo12(g_i8)
+; LA64-LARGE-NEXT:    lu32i.d $a1, %pc64_lo20(g_i8)
+; LA64-LARGE-NEXT:    lu52i.d $a1, $a1, %pc64_hi12(g_i8)
+; LA64-LARGE-NEXT:    add.d $a0, $a1, $a0
+; LA64-LARGE-NEXT:    xvldrepl.b $xr0, $a0, 0
+; LA64-LARGE-NEXT:    pcalau12i $a0, %pc_hi20(g_i8x32)
+; LA64-LARGE-NEXT:    addi.d $a1, $zero, %pc_lo12(g_i8x32)
+; LA64-LARGE-NEXT:    lu32i.d $a1, %pc64_lo20(g_i8x32)
+; LA64-LARGE-NEXT:    lu52i.d $a1, $a1, %pc64_hi12(g_i8x32)
+; LA64-LARGE-NEXT:    add.d $a0, $a1, $a0
+; LA64-LARGE-NEXT:    xvst $xr0, $a0, 0
+; LA64-LARGE-NEXT:    ret
+entry:
+  %0 = call <32 x i8> @llvm.loongarch.lasx.xvldrepl.b(ptr @g_i8, i32 0)
+  store <32 x i8> %0, ptr @g_i8x32, align 32
+  ret void
+}
+
 @g_rmw = dso_local global i64 0
 
 define dso_local void @rmw() nounwind {
@@ -659,11 +834,11 @@ define dso_local void @control_flow_with_mem_access() nounwind {
 ; LA32-NEXT:    pcalau12i $a0, %pc_hi20(g_a32+4)
 ; LA32-NEXT:    ld.w $a1, $a0, %pc_lo12(g_a32+4)
 ; LA32-NEXT:    ori $a2, $zero, 1
-; LA32-NEXT:    blt $a1, $a2, .LBB21_2
+; LA32-NEXT:    blt $a1, $a2, .LBB25_2
 ; LA32-NEXT:  # %bb.1: # %if.then
 ; LA32-NEXT:    ori $a1, $zero, 10
 ; LA32-NEXT:    st.w $a1, $a0, %pc_lo12(g_a32+4)
-; LA32-NEXT:  .LBB21_2: # %if.end
+; LA32-NEXT:  .LBB25_2: # %if.end
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: control_flow_with_mem_access:
@@ -671,11 +846,11 @@ define dso_local void @control_flow_with_mem_access() nounwind {
 ; LA64-NEXT:    pcalau12i $a0, %pc_hi20(g_a32+4)
 ; LA64-NEXT:    ld.w $a1, $a0, %pc_lo12(g_a32+4)
 ; LA64-NEXT:    ori $a2, $zero, 1
-; LA64-NEXT:    blt $a1, $a2, .LBB21_2
+; LA64-NEXT:    blt $a1, $a2, .LBB25_2
 ; LA64-NEXT:  # %bb.1: # %if.then
 ; LA64-NEXT:    ori $a1, $zero, 10
 ; LA64-NEXT:    st.w $a1, $a0, %pc_lo12(g_a32+4)
-; LA64-NEXT:  .LBB21_2: # %if.end
+; LA64-NEXT:  .LBB25_2: # %if.end
 ; LA64-NEXT:    ret
 ;
 ; LA64-LARGE-LABEL: control_flow_with_mem_access:
@@ -686,7 +861,7 @@ define dso_local void @control_flow_with_mem_access() nounwind {
 ; LA64-LARGE-NEXT:    lu52i.d $a1, $a1, %pc64_hi12(g_a32+4)
 ; LA64-LARGE-NEXT:    ldx.w $a0, $a1, $a0
 ; LA64-LARGE-NEXT:    ori $a1, $zero, 1
-; LA64-LARGE-NEXT:    blt $a0, $a1, .LBB21_2
+; LA64-LARGE-NEXT:    blt $a0, $a1, .LBB25_2
 ; LA64-LARGE-NEXT:  # %bb.1: # %if.then
 ; LA64-LARGE-NEXT:    pcalau12i $a0, %pc_hi20(g_a32+4)
 ; LA64-LARGE-NEXT:    addi.d $a1, $zero, %pc_lo12(g_a32+4)
@@ -694,7 +869,7 @@ define dso_local void @control_flow_with_mem_access() nounwind {
 ; LA64-LARGE-NEXT:    lu52i.d $a1, $a1, %pc64_hi12(g_a32+4)
 ; LA64-LARGE-NEXT:    ori $a2, $zero, 10
 ; LA64-LARGE-NEXT:    stx.w $a2, $a1, $a0
-; LA64-LARGE-NEXT:  .LBB21_2: # %if.end
+; LA64-LARGE-NEXT:  .LBB25_2: # %if.end
 ; LA64-LARGE-NEXT:    ret
 entry:
   %0 = load i32, ptr getelementptr inbounds ([1 x i32], ptr @g_a32, i32 1), align 4

From 569698443d2b1dad04dc4daa4559d754deabe64e Mon Sep 17 00:00:00 2001
From: Andrea Faulds <andrea.faulds@amd.com>
Date: Thu, 15 Aug 2024 15:11:31 +0200
Subject: [PATCH 023/441] [mlir][gpu] Fix typo in test filename (#104053)

The word "redule" doesn't appear anywhere else in the MLIR codebase and
seems to be a typo of "reduce".
---
 ...ubgroup-redule-lowering.mlir => subgroup-reduce-lowering.mlir} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename mlir/test/Dialect/GPU/{subgroup-redule-lowering.mlir => subgroup-reduce-lowering.mlir} (100%)

diff --git a/mlir/test/Dialect/GPU/subgroup-redule-lowering.mlir b/mlir/test/Dialect/GPU/subgroup-reduce-lowering.mlir
similarity index 100%
rename from mlir/test/Dialect/GPU/subgroup-redule-lowering.mlir
rename to mlir/test/Dialect/GPU/subgroup-reduce-lowering.mlir

From d2a8351be2b0cc8572de3014f1bac1f03fa92617 Mon Sep 17 00:00:00 2001
From: Steven Wu <stevenwu@apple.com>
Date: Thu, 15 Aug 2024 06:16:27 -0700
Subject: [PATCH 024/441] [CompilerRT][Tests] Fix
 profile/darwin-proof-of-concept.c (#104237)

Fix profile/darwin-proof-of-concept.c on AppleSilicon Mac where there is
a different page alignment. The previous fix to drop alignment is
actually breaking the tests on Apple Silicon Mac. Revert to the original
section alignment and requires an ARM64 target for this test to run.
---
 .../ContinuousSyncMode/darwin-proof-of-concept.c     | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/compiler-rt/test/profile/ContinuousSyncMode/darwin-proof-of-concept.c b/compiler-rt/test/profile/ContinuousSyncMode/darwin-proof-of-concept.c
index 3ed7c1894b6d106..d163a28d0e90ba6 100644
--- a/compiler-rt/test/profile/ContinuousSyncMode/darwin-proof-of-concept.c
+++ b/compiler-rt/test/profile/ContinuousSyncMode/darwin-proof-of-concept.c
@@ -4,12 +4,12 @@
 // mode to a new platform, but is not in and of itself a test of the profiling
 // runtime.
 
-// REQUIRES: darwin
+// REQUIRES: darwin, target={{arm64.*}}
 
 // Align counters and data to the maximum expected page size (16K).
 // RUN: %clang -g -o %t %s \
-// RUN:   -Wl,-sectalign,__DATA,__pcnts,0x1000 \
-// RUN:   -Wl,-sectalign,__DATA,__pdata,0x1000
+// RUN:   -Wl,-sectalign,__DATA,__pcnts,0x4000 \
+// RUN:   -Wl,-sectalign,__DATA,__pdata,0x4000
 
 // Create a 'profile' using mmap() and validate it.
 // RUN: %run %t create %t.tmpfile
@@ -24,7 +24,7 @@
 
 __attribute__((section("__DATA,__pcnts"))) int counters[] = {0xbad};
 extern int cnts_start __asm("section$start$__DATA$__pcnts");
-const size_t cnts_len = 0x1000;
+const size_t cnts_len = 0x4000;
 
 __attribute__((section("__DATA,__pdata"))) int data[] = {1, 2, 3};
 extern int data_start __asm("section$start$__DATA$__pdata");
@@ -131,12 +131,12 @@ int main(int argc, char **argv) {
     fprintf(stderr, "__pcnts is not page-aligned: 0x%lx.\n", cnts_start_int);
     return EXIT_FAILURE;
   }
-  if (data_start_int % 0x1000 != 0) {
+  if (data_start_int % 0x4000 != 0) {
     fprintf(stderr, "__pdata is not correctly aligned: 0x%lx.\n",
             data_start_int);
     return EXIT_FAILURE;
   }
-  if (cnts_start_int + 0x1000 != data_start_int) {
+  if (cnts_start_int + 0x4000 != data_start_int) {
     fprintf(stderr, "__pdata not ordered after __pcnts.\n");
     return EXIT_FAILURE;
   }

From a4525fcc8f127139a493164c360725ae4c6c87b3 Mon Sep 17 00:00:00 2001
From: Jie Fu <jiefu@tencent.com>
Date: Thu, 15 Aug 2024 21:17:12 +0800
Subject: [PATCH 025/441] [CodeGen] Fix -Wcovered-switch-default in
 Combiner.cpp (NFC)

/llvm-project/llvm/lib/CodeGen/GlobalISel/Combiner.cpp:220:3:
error: default label in switch which covers all enumeration values [-Werror,-Wcovered-switch-default]
  default:
  ^
1 error generated.
---
 llvm/lib/CodeGen/GlobalISel/Combiner.cpp | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/llvm/lib/CodeGen/GlobalISel/Combiner.cpp b/llvm/lib/CodeGen/GlobalISel/Combiner.cpp
index 75b2525e368af68..c5ec73cd5c65d83 100644
--- a/llvm/lib/CodeGen/GlobalISel/Combiner.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/Combiner.cpp
@@ -217,9 +217,8 @@ Combiner::WorkListMaintainer::create(Level Lvl, WorkListTy &WorkList,
   case Level::SinglePass:
     return std::make_unique<WorkListMaintainerImpl<Level::SinglePass>>(WorkList,
                                                                        MRI);
-  default:
-    llvm_unreachable("Illegal ObserverLevel");
   }
+  llvm_unreachable("Illegal ObserverLevel");
 }
 
 Combiner::Combiner(MachineFunction &MF, CombinerInfo &CInfo,

From 026d963cb004689477d2b5798cbba5ad41c25a70 Mon Sep 17 00:00:00 2001
From: Daniel Grumberg <dgrumberg@apple.com>
Date: Thu, 15 Aug 2024 14:19:49 +0100
Subject: [PATCH 026/441] [clang][ExtractAPI] Compute inherited availability
 information (#103040)

Additionally this computes availability information for all platforms
ahead of possibly introducing a flag to enable this behavior.

rdar://123513706
---
 clang/include/clang/AST/Availability.h        |   4 +
 clang/lib/AST/Availability.cpp                | 103 +++++++++--
 .../Serialization/SymbolGraphSerializer.cpp   |  31 ++--
 .../test/ExtractAPI/inherited_availability.m  | 175 ++++++++++++++++++
 4 files changed, 283 insertions(+), 30 deletions(-)
 create mode 100644 clang/test/ExtractAPI/inherited_availability.m

diff --git a/clang/include/clang/AST/Availability.h b/clang/include/clang/AST/Availability.h
index 26ae622e5b4496f..60ca1383f0a44ea 100644
--- a/clang/include/clang/AST/Availability.h
+++ b/clang/include/clang/AST/Availability.h
@@ -97,6 +97,10 @@ struct AvailabilityInfo {
     return UnconditionallyUnavailable;
   }
 
+  /// Augments the existing information with additional constraints provided by
+  /// \c Other.
+  void mergeWith(AvailabilityInfo Other);
+
   AvailabilityInfo(StringRef Domain, VersionTuple I, VersionTuple D,
                    VersionTuple O, bool U, bool UD, bool UU)
       : Domain(Domain), Introduced(I), Deprecated(D), Obsoleted(O),
diff --git a/clang/lib/AST/Availability.cpp b/clang/lib/AST/Availability.cpp
index 238359a2dedfcf4..cf040fc727d11f4 100644
--- a/clang/lib/AST/Availability.cpp
+++ b/clang/lib/AST/Availability.cpp
@@ -16,33 +16,104 @@
 #include "clang/AST/Decl.h"
 #include "clang/Basic/TargetInfo.h"
 
-namespace clang {
+namespace {
+
+/// Represents the availability of a symbol across platforms.
+struct AvailabilitySet {
+  bool UnconditionallyDeprecated = false;
+  bool UnconditionallyUnavailable = false;
+
+  void insert(clang::AvailabilityInfo &&Availability) {
+    auto *Found = getForPlatform(Availability.Domain);
+    if (Found)
+      Found->mergeWith(std::move(Availability));
+    else
+      Availabilities.emplace_back(std::move(Availability));
+  }
+
+  clang::AvailabilityInfo *getForPlatform(llvm::StringRef Domain) {
+    auto *It = llvm::find_if(Availabilities,
+                             [Domain](const clang::AvailabilityInfo &Info) {
+                               return Domain.compare(Info.Domain) == 0;
+                             });
+    return It == Availabilities.end() ? nullptr : It;
+  }
 
-AvailabilityInfo AvailabilityInfo::createFromDecl(const Decl *Decl) {
-  ASTContext &Context = Decl->getASTContext();
-  StringRef PlatformName = Context.getTargetInfo().getPlatformName();
-  AvailabilityInfo Availability;
+private:
+  llvm::SmallVector<clang::AvailabilityInfo> Availabilities;
+};
 
+static void createInfoForDecl(const clang::Decl *Decl,
+                              AvailabilitySet &Availabilities) {
   // Collect availability attributes from all redeclarations.
   for (const auto *RD : Decl->redecls()) {
-    for (const auto *A : RD->specific_attrs<AvailabilityAttr>()) {
-      if (A->getPlatform()->getName() != PlatformName)
-        continue;
-      Availability = AvailabilityInfo(
+    for (const auto *A : RD->specific_attrs<clang::AvailabilityAttr>()) {
+      Availabilities.insert(clang::AvailabilityInfo(
           A->getPlatform()->getName(), A->getIntroduced(), A->getDeprecated(),
-          A->getObsoleted(), A->getUnavailable(), false, false);
-      break;
+          A->getObsoleted(), A->getUnavailable(), false, false));
     }
 
-    if (const auto *A = RD->getAttr<UnavailableAttr>())
+    if (const auto *A = RD->getAttr<clang::UnavailableAttr>())
       if (!A->isImplicit())
-        Availability.UnconditionallyUnavailable = true;
+        Availabilities.UnconditionallyUnavailable = true;
 
-    if (const auto *A = RD->getAttr<DeprecatedAttr>())
+    if (const auto *A = RD->getAttr<clang::DeprecatedAttr>())
       if (!A->isImplicit())
-        Availability.UnconditionallyDeprecated = true;
+        Availabilities.UnconditionallyDeprecated = true;
   }
-  return Availability;
+}
+
+} // namespace
+
+namespace clang {
+
+void AvailabilityInfo::mergeWith(AvailabilityInfo Other) {
+  if (isDefault() && Other.isDefault())
+    return;
+
+  if (Domain.empty())
+    Domain = Other.Domain;
+
+  UnconditionallyUnavailable |= Other.UnconditionallyUnavailable;
+  UnconditionallyDeprecated |= Other.UnconditionallyDeprecated;
+  Unavailable |= Other.Unavailable;
+
+  Introduced = std::max(Introduced, Other.Introduced);
+
+  // Default VersionTuple is 0.0.0 so if both are non default let's pick the
+  // smallest version number, otherwise select the one that is non-zero if there
+  // is one.
+  if (!Deprecated.empty() && !Other.Deprecated.empty())
+    Deprecated = std::min(Deprecated, Other.Deprecated);
+  else
+    Deprecated = std::max(Deprecated, Other.Deprecated);
+
+  if (!Obsoleted.empty() && !Other.Obsoleted.empty())
+    Obsoleted = std::min(Obsoleted, Other.Obsoleted);
+  else
+    Obsoleted = std::max(Obsoleted, Other.Obsoleted);
+}
+
+AvailabilityInfo AvailabilityInfo::createFromDecl(const Decl *D) {
+  AvailabilitySet Availabilities;
+  // Walk DeclContexts upwards starting from D to find the combined availability
+  // of the symbol.
+  for (const auto *Ctx = D; Ctx;
+       Ctx = llvm::cast_or_null<Decl>(Ctx->getDeclContext()))
+    createInfoForDecl(Ctx, Availabilities);
+
+  if (auto *Avail = Availabilities.getForPlatform(
+          D->getASTContext().getTargetInfo().getPlatformName())) {
+    Avail->UnconditionallyDeprecated = Availabilities.UnconditionallyDeprecated;
+    Avail->UnconditionallyUnavailable =
+        Availabilities.UnconditionallyUnavailable;
+    return std::move(*Avail);
+  }
+
+  AvailabilityInfo Avail;
+  Avail.UnconditionallyDeprecated = Availabilities.UnconditionallyDeprecated;
+  Avail.UnconditionallyUnavailable = Availabilities.UnconditionallyUnavailable;
+  return Avail;
 }
 
 } // namespace clang
diff --git a/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp b/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp
index 6e56ee5b573f66a..84ed5467dd2fb91 100644
--- a/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp
+++ b/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp
@@ -171,22 +171,25 @@ std::optional<Array> serializeAvailability(const AvailabilityInfo &Avail) {
     UnconditionallyDeprecated["isUnconditionallyDeprecated"] = true;
     AvailabilityArray.emplace_back(std::move(UnconditionallyDeprecated));
   }
-  Object Availability;
-
-  Availability["domain"] = Avail.Domain;
-
-  if (Avail.isUnavailable()) {
-    Availability["isUnconditionallyUnavailable"] = true;
-  } else {
-    serializeObject(Availability, "introduced",
-                    serializeSemanticVersion(Avail.Introduced));
-    serializeObject(Availability, "deprecated",
-                    serializeSemanticVersion(Avail.Deprecated));
-    serializeObject(Availability, "obsoleted",
-                    serializeSemanticVersion(Avail.Obsoleted));
+
+  if (Avail.Domain.str() != "") {
+    Object Availability;
+    Availability["domain"] = Avail.Domain;
+
+    if (Avail.isUnavailable()) {
+      Availability["isUnconditionallyUnavailable"] = true;
+    } else {
+      serializeObject(Availability, "introduced",
+                      serializeSemanticVersion(Avail.Introduced));
+      serializeObject(Availability, "deprecated",
+                      serializeSemanticVersion(Avail.Deprecated));
+      serializeObject(Availability, "obsoleted",
+                      serializeSemanticVersion(Avail.Obsoleted));
+    }
+
+    AvailabilityArray.emplace_back(std::move(Availability));
   }
 
-  AvailabilityArray.emplace_back(std::move(Availability));
   return AvailabilityArray;
 }
 
diff --git a/clang/test/ExtractAPI/inherited_availability.m b/clang/test/ExtractAPI/inherited_availability.m
new file mode 100644
index 000000000000000..c24e7fa8e208f03
--- /dev/null
+++ b/clang/test/ExtractAPI/inherited_availability.m
@@ -0,0 +1,175 @@
+// RUN: rm -rf %t
+// RUN: %clang_cc1 -extract-api --pretty-sgf --emit-sgf-symbol-labels-for-testing -triple arm64-apple-macosx \
+// RUN:   -x objective-c-header %s -o %t/output.symbols.json -verify
+
+
+// RUN: FileCheck %s --input-file %t/output.symbols.json --check-prefix A
+__attribute__((availability(macos, introduced=9.0, deprecated=12.0, obsoleted=20.0)))
+@interface A
+// A-LABEL: "!testLabel": "c:objc(cs)A"
+// A:      "availability": [
+// A-NEXT:   {
+// A-NEXT:     "deprecated": {
+// A-NEXT:       "major": 12,
+// A-NEXT:       "minor": 0,
+// A-NEXT:       "patch": 0
+// A-NEXT:     }
+// A-NEXT:     "domain": "macos"
+// A-NEXT:     "introduced": {
+// A-NEXT:       "major": 9,
+// A-NEXT:       "minor": 0,
+// A-NEXT:       "patch": 0
+// A-NEXT:     }
+// A-NEXT:     "obsoleted": {
+// A-NEXT:       "major": 20,
+// A-NEXT:       "minor": 0,
+// A-NEXT:       "patch": 0
+// A-NEXT:     }
+// A-NEXT:   }
+// A-NEXT: ]
+
+// RUN: FileCheck %s --input-file %t/output.symbols.json --check-prefix CP
+@property(class) int CP;
+// CP-LABEL: "!testLabel": "c:objc(cs)A(cpy)CP"
+// CP:      "availability": [
+// CP-NEXT:   {
+// CP-NEXT:     "deprecated": {
+// CP-NEXT:       "major": 12,
+// CP-NEXT:       "minor": 0,
+// CP-NEXT:       "patch": 0
+// CP-NEXT:     }
+// CP-NEXT:     "domain": "macos"
+// CP-NEXT:     "introduced": {
+// CP-NEXT:       "major": 9,
+// CP-NEXT:       "minor": 0,
+// CP-NEXT:       "patch": 0
+// CP-NEXT:     }
+// CP-NEXT:     "obsoleted": {
+// CP-NEXT:       "major": 20,
+// CP-NEXT:       "minor": 0,
+// CP-NEXT:       "patch": 0
+// CP-NEXT:     }
+// CP-NEXT:   }
+// CP-NEXT: ]
+
+// RUN: FileCheck %s --input-file %t/output.symbols.json --check-prefix IP
+@property int IP;
+// IP-LABEL: "!testLabel": "c:objc(cs)A(py)IP"
+// IP:      "availability": [
+// IP-NEXT:   {
+// IP-NEXT:     "deprecated": {
+// IP-NEXT:       "major": 12,
+// IP-NEXT:       "minor": 0,
+// IP-NEXT:       "patch": 0
+// IP-NEXT:     }
+// IP-NEXT:     "domain": "macos"
+// IP-NEXT:     "introduced": {
+// IP-NEXT:       "major": 9,
+// IP-NEXT:       "minor": 0,
+// IP-NEXT:       "patch": 0
+// IP-NEXT:     }
+// IP-NEXT:     "obsoleted": {
+// IP-NEXT:       "major": 20,
+// IP-NEXT:       "minor": 0,
+// IP-NEXT:       "patch": 0
+// IP-NEXT:     }
+// IP-NEXT:   }
+// IP-NEXT: ]
+
+// RUN: FileCheck %s --input-file %t/output.symbols.json --check-prefix MR
+@property int moreRestrictive __attribute__((availability(macos, introduced=10.0, deprecated=11.0, obsoleted=19.0)));
+// MR-LABEL: "!testLabel": "c:objc(cs)A(py)moreRestrictive"
+// MR:      "availability": [
+// MR-NEXT:   {
+// MR-NEXT:     "deprecated": {
+// MR-NEXT:       "major": 11,
+// MR-NEXT:       "minor": 0,
+// MR-NEXT:       "patch": 0
+// MR-NEXT:     }
+// MR-NEXT:     "domain": "macos"
+// MR-NEXT:     "introduced": {
+// MR-NEXT:       "major": 10,
+// MR-NEXT:       "minor": 0,
+// MR-NEXT:       "patch": 0
+// MR-NEXT:     }
+// MR-NEXT:     "obsoleted": {
+// MR-NEXT:       "major": 19,
+// MR-NEXT:       "minor": 0,
+// MR-NEXT:       "patch": 0
+// MR-NEXT:     }
+// MR-NEXT:   }
+// MR-NEXT: ]
+
+@end
+
+// RUN: FileCheck %s --input-file %t/output.symbols.json --check-prefix B
+__attribute__((deprecated("B is deprecated")))
+@interface B
+// B-LABEL: "!testLabel": "c:objc(cs)B"
+// B:      "availability": [
+// B-NEXT:   {
+// B-NEXT:     "domain": "*"
+// B-NEXT:     "isUnconditionallyDeprecated": true
+// B-NEXT:   }
+// B-NEXT: ]
+
+// RUN: FileCheck %s --input-file %t/output.symbols.json --check-prefix BIP
+@property int BIP;
+// BIP-LABEL: "!testLabel": "c:objc(cs)B(py)BIP"
+// BIP:      "availability": [
+// BIP-NEXT:   {
+// BIP-NEXT:     "domain": "*"
+// BIP-NEXT:     "isUnconditionallyDeprecated": true
+// BIP-NEXT:   }
+// BIP-NEXT: ]
+@end
+
+// RUN: FileCheck %s --input-file %t/output.symbols.json --check-prefix C
+__attribute__((availability(macos, unavailable)))
+@interface C
+// C-LABEL: "!testLabel": "c:objc(cs)C"
+// C:      "availability": [
+// C-NEXT:   {
+// C-NEXT:     "domain": "macos"
+// C-NEXT:     "isUnconditionallyUnavailable": true
+// C-NEXT:   }
+// C-NEXT: ]
+
+// RUN: FileCheck %s --input-file %t/output.symbols.json --check-prefix CIP
+@property int CIP;
+// CIP-LABEL: "!testLabel": "c:objc(cs)C(py)CIP"
+// CIP:      "availability": [
+// CIP-NEXT:   {
+// CIP-NEXT:     "domain": "macos"
+// CIP-NEXT:     "isUnconditionallyUnavailable": true
+// CIP-NEXT:   }
+// CIP-NEXT: ]
+@end
+
+@interface D
+// RUN: FileCheck %s --input-file %t/output.symbols.json --check-prefix DIP
+@property int DIP __attribute__((availability(macos, introduced=10.0, deprecated=11.0, obsoleted=19.0)));
+// DIP-LABEL: "!testLabel": "c:objc(cs)D(py)DIP"
+// DIP:      "availability": [
+// DIP-NEXT:   {
+// DIP-NEXT:     "deprecated": {
+// DIP-NEXT:       "major": 11,
+// DIP-NEXT:       "minor": 0,
+// DIP-NEXT:       "patch": 0
+// DIP-NEXT:     }
+// DIP-NEXT:     "domain": "macos"
+// DIP-NEXT:     "introduced": {
+// DIP-NEXT:       "major": 10,
+// DIP-NEXT:       "minor": 0,
+// DIP-NEXT:       "patch": 0
+// DIP-NEXT:     }
+// DIP-NEXT:     "obsoleted": {
+// DIP-NEXT:       "major": 19,
+// DIP-NEXT:       "minor": 0,
+// DIP-NEXT:       "patch": 0
+// DIP-NEXT:     }
+// DIP-NEXT:   }
+// DIP-NEXT: ]
+@end
+
+// expected-no-diagnostics

From 6543bd718e6a4e4d6a8473b478f8a46d3eb1562a Mon Sep 17 00:00:00 2001
From: Sergei Barannikov <barannikov88@gmail.com>
Date: Thu, 15 Aug 2024 16:41:52 +0300
Subject: [PATCH 027/441] [DataLayout] Extract loop body into a function to
 reduce nesting (NFC) (#104420)

Also, use `iterator_range` version of `split`.

Pull Request: https://github.com/llvm/llvm-project/pull/104420
---
 llvm/include/llvm/IR/DataLayout.h    |  10 +-
 llvm/lib/IR/DataLayout.cpp           | 495 ++++++++++++++-------------
 llvm/unittests/IR/DataLayoutTest.cpp |  10 +
 3 files changed, 267 insertions(+), 248 deletions(-)

diff --git a/llvm/include/llvm/IR/DataLayout.h b/llvm/include/llvm/IR/DataLayout.h
index 1185939cd9c75b7..228b723ee663ff7 100644
--- a/llvm/include/llvm/IR/DataLayout.h
+++ b/llvm/include/llvm/IR/DataLayout.h
@@ -165,9 +165,11 @@ class DataLayout {
   /// Internal helper method that returns requested alignment for type.
   Align getAlignment(Type *Ty, bool abi_or_pref) const;
 
-  /// Attempts to parse a target data specification string and reports an error
-  /// if the string is malformed.
-  Error parseSpecifier(StringRef Desc);
+  /// Attempts to parse a single specification.
+  Error parseSpecification(StringRef Specification);
+
+  /// Attempts to parse a data layout string.
+  Error parseLayoutString(StringRef LayoutString);
 
 public:
   /// Constructs a DataLayout with default values.
@@ -188,7 +190,7 @@ class DataLayout {
 
   /// Parse a data layout string and return the layout. Return an error
   /// description on failure.
-  static Expected<DataLayout> parse(StringRef LayoutDescription);
+  static Expected<DataLayout> parse(StringRef LayoutString);
 
   /// Layout endianness...
   bool isLittleEndian() const { return !BigEndian; }
diff --git a/llvm/lib/IR/DataLayout.cpp b/llvm/lib/IR/DataLayout.cpp
index 5f9a0fbc61905fc..24c29458abfa819 100644
--- a/llvm/lib/IR/DataLayout.cpp
+++ b/llvm/lib/IR/DataLayout.cpp
@@ -17,6 +17,7 @@
 
 #include "llvm/IR/DataLayout.h"
 #include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DerivedTypes.h"
@@ -215,7 +216,7 @@ DataLayout::DataLayout()
       PointerSpecs(ArrayRef(DefaultPointerSpecs)) {}
 
 DataLayout::DataLayout(StringRef LayoutString) : DataLayout() {
-  if (Error Err = parseSpecifier(LayoutString))
+  if (Error Err = parseLayoutString(LayoutString))
     report_fatal_error(std::move(Err));
 }
 
@@ -260,9 +261,9 @@ bool DataLayout::operator==(const DataLayout &Other) const {
          StructPrefAlignment == Other.StructPrefAlignment;
 }
 
-Expected<DataLayout> DataLayout::parse(StringRef LayoutDescription) {
+Expected<DataLayout> DataLayout::parse(StringRef LayoutString) {
   DataLayout Layout;
-  if (Error Err = Layout.parseSpecifier(LayoutDescription))
+  if (Error Err = Layout.parseLayoutString(LayoutString))
     return std::move(Err);
   return Layout;
 }
@@ -311,289 +312,295 @@ static Error getAddrSpace(StringRef R, unsigned &AddrSpace) {
   return Error::success();
 }
 
-Error DataLayout::parseSpecifier(StringRef Desc) {
-  StringRepresentation = std::string(Desc);
-  while (!Desc.empty()) {
-    // Split at '-'.
-    std::pair<StringRef, StringRef> Split;
-    if (Error Err = ::split(Desc, '-', Split))
-      return Err;
-    Desc = Split.second;
-
-    // Split at ':'.
-    if (Error Err = ::split(Split.first, ':', Split))
-      return Err;
+Error DataLayout::parseSpecification(StringRef Spec) {
+  // Split at ':'.
+  std::pair<StringRef, StringRef> Split;
+  if (Error Err = ::split(Spec, ':', Split))
+    return Err;
 
-    // Aliases used below.
-    StringRef &Tok  = Split.first;  // Current token.
-    StringRef &Rest = Split.second; // The rest of the string.
+  // Aliases used below.
+  StringRef &Tok = Split.first;   // Current token.
+  StringRef &Rest = Split.second; // The rest of the string.
 
-    if (Tok == "ni") {
-      do {
-        if (Error Err = ::split(Rest, ':', Split))
-          return Err;
-        Rest = Split.second;
-        unsigned AS;
-        if (Error Err = getInt(Split.first, AS))
-          return Err;
-        if (AS == 0)
-          return reportError("Address space 0 can never be non-integral");
-        NonIntegralAddressSpaces.push_back(AS);
-      } while (!Rest.empty());
+  if (Tok == "ni") {
+    do {
+      if (Error Err = ::split(Rest, ':', Split))
+        return Err;
+      Rest = Split.second;
+      unsigned AS;
+      if (Error Err = getInt(Split.first, AS))
+        return Err;
+      if (AS == 0)
+        return reportError("Address space 0 can never be non-integral");
+      NonIntegralAddressSpaces.push_back(AS);
+    } while (!Rest.empty());
 
-      continue;
-    }
+    return Error::success();
+  }
 
-    char SpecifierChar = Tok.front();
-    Tok = Tok.substr(1);
+  char SpecifierChar = Tok.front();
+  Tok = Tok.substr(1);
 
-    switch (SpecifierChar) {
-    case 's':
-      // Deprecated, but ignoring here to preserve loading older textual llvm
-      // ASM file
-      break;
-    case 'E':
-      BigEndian = true;
-      break;
-    case 'e':
-      BigEndian = false;
-      break;
-    case 'p': {
-      // Address space.
-      unsigned AddrSpace = 0;
-      if (!Tok.empty())
-        if (Error Err = getInt(Tok, AddrSpace))
-          return Err;
-      if (!isUInt<24>(AddrSpace))
-        return reportError("Invalid address space, must be a 24-bit integer");
-
-      // Size.
-      if (Rest.empty())
-        return reportError(
-            "Missing size specification for pointer in datalayout string");
-      if (Error Err = ::split(Rest, ':', Split))
-        return Err;
-      unsigned PointerMemSize;
-      if (Error Err = getInt(Tok, PointerMemSize))
+  switch (SpecifierChar) {
+  case 's':
+    // Deprecated, but ignoring here to preserve loading older textual llvm
+    // ASM file
+    break;
+  case 'E':
+    BigEndian = true;
+    break;
+  case 'e':
+    BigEndian = false;
+    break;
+  case 'p': {
+    // Address space.
+    unsigned AddrSpace = 0;
+    if (!Tok.empty())
+      if (Error Err = getInt(Tok, AddrSpace))
         return Err;
-      if (!PointerMemSize)
-        return reportError("Invalid pointer size of 0 bytes");
+    if (!isUInt<24>(AddrSpace))
+      return reportError("Invalid address space, must be a 24-bit integer");
+
+    // Size.
+    if (Rest.empty())
+      return reportError(
+          "Missing size specification for pointer in datalayout string");
+    if (Error Err = ::split(Rest, ':', Split))
+      return Err;
+    unsigned PointerMemSize;
+    if (Error Err = getInt(Tok, PointerMemSize))
+      return Err;
+    if (!PointerMemSize)
+      return reportError("Invalid pointer size of 0 bytes");
+
+    // ABI alignment.
+    if (Rest.empty())
+      return reportError(
+          "Missing alignment specification for pointer in datalayout string");
+    if (Error Err = ::split(Rest, ':', Split))
+      return Err;
+    unsigned PointerABIAlign;
+    if (Error Err = getIntInBytes(Tok, PointerABIAlign))
+      return Err;
+    if (!isPowerOf2_64(PointerABIAlign))
+      return reportError("Pointer ABI alignment must be a power of 2");
 
-      // ABI alignment.
-      if (Rest.empty())
-        return reportError(
-            "Missing alignment specification for pointer in datalayout string");
+    // Size of index used in GEP for address calculation.
+    // The parameter is optional. By default it is equal to size of pointer.
+    unsigned IndexSize = PointerMemSize;
+
+    // Preferred alignment.
+    unsigned PointerPrefAlign = PointerABIAlign;
+    if (!Rest.empty()) {
       if (Error Err = ::split(Rest, ':', Split))
         return Err;
-      unsigned PointerABIAlign;
-      if (Error Err = getIntInBytes(Tok, PointerABIAlign))
+      if (Error Err = getIntInBytes(Tok, PointerPrefAlign))
         return Err;
-      if (!isPowerOf2_64(PointerABIAlign))
-        return reportError("Pointer ABI alignment must be a power of 2");
+      if (!isPowerOf2_64(PointerPrefAlign))
+        return reportError("Pointer preferred alignment must be a power of 2");
 
-      // Size of index used in GEP for address calculation.
-      // The parameter is optional. By default it is equal to size of pointer.
-      unsigned IndexSize = PointerMemSize;
-
-      // Preferred alignment.
-      unsigned PointerPrefAlign = PointerABIAlign;
+      // Now read the index. It is the second optional parameter here.
       if (!Rest.empty()) {
         if (Error Err = ::split(Rest, ':', Split))
           return Err;
-        if (Error Err = getIntInBytes(Tok, PointerPrefAlign))
+        if (Error Err = getInt(Tok, IndexSize))
           return Err;
-        if (!isPowerOf2_64(PointerPrefAlign))
-          return reportError(
-              "Pointer preferred alignment must be a power of 2");
-
-        // Now read the index. It is the second optional parameter here.
-        if (!Rest.empty()) {
-          if (Error Err = ::split(Rest, ':', Split))
-            return Err;
-          if (Error Err = getInt(Tok, IndexSize))
-            return Err;
-          if (!IndexSize)
-            return reportError("Invalid index size of 0 bytes");
-        }
+        if (!IndexSize)
+          return reportError("Invalid index size of 0 bytes");
       }
-      if (Error Err = setPointerSpec(
-              AddrSpace, PointerMemSize, assumeAligned(PointerABIAlign),
-              assumeAligned(PointerPrefAlign), IndexSize))
-        return Err;
-      break;
     }
+    if (Error Err = setPointerSpec(AddrSpace, PointerMemSize,
+                                   assumeAligned(PointerABIAlign),
+                                   assumeAligned(PointerPrefAlign), IndexSize))
+      return Err;
+    break;
+  }
+  case 'i':
+  case 'v':
+  case 'f':
+  case 'a': {
+    TypeSpecifier Specifier;
+    switch (SpecifierChar) {
+    default:
+      llvm_unreachable("Unexpected specifier!");
     case 'i':
+      Specifier = TypeSpecifier::Integer;
+      break;
     case 'v':
+      Specifier = TypeSpecifier::Vector;
+      break;
     case 'f':
-    case 'a': {
-      TypeSpecifier Specifier;
-      switch (SpecifierChar) {
-      default:
-        llvm_unreachable("Unexpected specifier!");
-      case 'i':
-        Specifier = TypeSpecifier::Integer;
-        break;
-      case 'v':
-        Specifier = TypeSpecifier::Vector;
-        break;
-      case 'f':
-        Specifier = TypeSpecifier::Float;
-        break;
-      case 'a':
-        Specifier = TypeSpecifier::Aggregate;
-        break;
-      }
+      Specifier = TypeSpecifier::Float;
+      break;
+    case 'a':
+      Specifier = TypeSpecifier::Aggregate;
+      break;
+    }
 
-      // Bit size.
-      unsigned Size = 0;
-      if (!Tok.empty())
-        if (Error Err = getInt(Tok, Size))
-          return Err;
+    // Bit size.
+    unsigned Size = 0;
+    if (!Tok.empty())
+      if (Error Err = getInt(Tok, Size))
+        return Err;
 
-      if (Specifier == TypeSpecifier::Aggregate && Size != 0)
-        return reportError(
-            "Sized aggregate specification in datalayout string");
+    if (Specifier == TypeSpecifier::Aggregate && Size != 0)
+      return reportError("Sized aggregate specification in datalayout string");
 
-      // ABI alignment.
-      if (Rest.empty())
-        return reportError(
-            "Missing alignment specification in datalayout string");
+    // ABI alignment.
+    if (Rest.empty())
+      return reportError(
+          "Missing alignment specification in datalayout string");
+    if (Error Err = ::split(Rest, ':', Split))
+      return Err;
+    unsigned ABIAlign;
+    if (Error Err = getIntInBytes(Tok, ABIAlign))
+      return Err;
+    if (Specifier != TypeSpecifier::Aggregate && !ABIAlign)
+      return reportError(
+          "ABI alignment specification must be >0 for non-aggregate types");
+
+    if (!isUInt<16>(ABIAlign))
+      return reportError("Invalid ABI alignment, must be a 16bit integer");
+    if (ABIAlign != 0 && !isPowerOf2_64(ABIAlign))
+      return reportError("Invalid ABI alignment, must be a power of 2");
+    if (Specifier == TypeSpecifier::Integer && Size == 8 && ABIAlign != 1)
+      return reportError("Invalid ABI alignment, i8 must be naturally aligned");
+
+    // Preferred alignment.
+    unsigned PrefAlign = ABIAlign;
+    if (!Rest.empty()) {
       if (Error Err = ::split(Rest, ':', Split))
         return Err;
-      unsigned ABIAlign;
-      if (Error Err = getIntInBytes(Tok, ABIAlign))
+      if (Error Err = getIntInBytes(Tok, PrefAlign))
         return Err;
-      if (Specifier != TypeSpecifier::Aggregate && !ABIAlign)
-        return reportError(
-            "ABI alignment specification must be >0 for non-aggregate types");
+    }
 
-      if (!isUInt<16>(ABIAlign))
-        return reportError("Invalid ABI alignment, must be a 16bit integer");
-      if (ABIAlign != 0 && !isPowerOf2_64(ABIAlign))
-        return reportError("Invalid ABI alignment, must be a power of 2");
-      if (Specifier == TypeSpecifier::Integer && Size == 8 && ABIAlign != 1)
-        return reportError(
-            "Invalid ABI alignment, i8 must be naturally aligned");
+    if (!isUInt<16>(PrefAlign))
+      return reportError(
+          "Invalid preferred alignment, must be a 16bit integer");
+    if (PrefAlign != 0 && !isPowerOf2_64(PrefAlign))
+      return reportError("Invalid preferred alignment, must be a power of 2");
 
-      // Preferred alignment.
-      unsigned PrefAlign = ABIAlign;
-      if (!Rest.empty()) {
-        if (Error Err = ::split(Rest, ':', Split))
-          return Err;
-        if (Error Err = getIntInBytes(Tok, PrefAlign))
-          return Err;
-      }
+    if (Error Err = setPrimitiveSpec(Specifier, Size, assumeAligned(ABIAlign),
+                                     assumeAligned(PrefAlign)))
+      return Err;
 
-      if (!isUInt<16>(PrefAlign))
+    break;
+  }
+  case 'n': // Native integer types.
+    while (true) {
+      unsigned Width;
+      if (Error Err = getInt(Tok, Width))
+        return Err;
+      if (Width == 0)
         return reportError(
-            "Invalid preferred alignment, must be a 16bit integer");
-      if (PrefAlign != 0 && !isPowerOf2_64(PrefAlign))
-        return reportError("Invalid preferred alignment, must be a power of 2");
-
-      if (Error Err = setPrimitiveSpec(Specifier, Size, assumeAligned(ABIAlign),
-                                       assumeAligned(PrefAlign)))
+            "Zero width native integer type in datalayout string");
+      LegalIntWidths.push_back(Width);
+      if (Rest.empty())
+        break;
+      if (Error Err = ::split(Rest, ':', Split))
         return Err;
-
-      break;
     }
-    case 'n':  // Native integer types.
-      while (true) {
-        unsigned Width;
-        if (Error Err = getInt(Tok, Width))
-          return Err;
-        if (Width == 0)
-          return reportError(
-              "Zero width native integer type in datalayout string");
-        LegalIntWidths.push_back(Width);
-        if (Rest.empty())
-          break;
-        if (Error Err = ::split(Rest, ':', Split))
-          return Err;
-      }
-      break;
-    case 'S': { // Stack natural alignment.
-      uint64_t Alignment;
-      if (Error Err = getIntInBytes(Tok, Alignment))
-        return Err;
-      if (Alignment != 0 && !llvm::isPowerOf2_64(Alignment))
-        return reportError("Alignment is neither 0 nor a power of 2");
-      StackNaturalAlign = MaybeAlign(Alignment);
+    break;
+  case 'S': { // Stack natural alignment.
+    uint64_t Alignment;
+    if (Error Err = getIntInBytes(Tok, Alignment))
+      return Err;
+    if (Alignment != 0 && !llvm::isPowerOf2_64(Alignment))
+      return reportError("Alignment is neither 0 nor a power of 2");
+    StackNaturalAlign = MaybeAlign(Alignment);
+    break;
+  }
+  case 'F': {
+    switch (Tok.front()) {
+    case 'i':
+      TheFunctionPtrAlignType = FunctionPtrAlignType::Independent;
       break;
-    }
-    case 'F': {
-      switch (Tok.front()) {
-      case 'i':
-        TheFunctionPtrAlignType = FunctionPtrAlignType::Independent;
-        break;
-      case 'n':
-        TheFunctionPtrAlignType = FunctionPtrAlignType::MultipleOfFunctionAlign;
-        break;
-      default:
-        return reportError("Unknown function pointer alignment type in "
-                           "datalayout string");
-      }
-      Tok = Tok.substr(1);
-      uint64_t Alignment;
-      if (Error Err = getIntInBytes(Tok, Alignment))
-        return Err;
-      if (Alignment != 0 && !llvm::isPowerOf2_64(Alignment))
-        return reportError("Alignment is neither 0 nor a power of 2");
-      FunctionPtrAlign = MaybeAlign(Alignment);
+    case 'n':
+      TheFunctionPtrAlignType = FunctionPtrAlignType::MultipleOfFunctionAlign;
       break;
+    default:
+      return reportError("Unknown function pointer alignment type in "
+                         "datalayout string");
     }
-    case 'P': { // Function address space.
-      if (Error Err = getAddrSpace(Tok, ProgramAddrSpace))
-        return Err;
+    Tok = Tok.substr(1);
+    uint64_t Alignment;
+    if (Error Err = getIntInBytes(Tok, Alignment))
+      return Err;
+    if (Alignment != 0 && !llvm::isPowerOf2_64(Alignment))
+      return reportError("Alignment is neither 0 nor a power of 2");
+    FunctionPtrAlign = MaybeAlign(Alignment);
+    break;
+  }
+  case 'P': { // Function address space.
+    if (Error Err = getAddrSpace(Tok, ProgramAddrSpace))
+      return Err;
+    break;
+  }
+  case 'A': { // Default stack/alloca address space.
+    if (Error Err = getAddrSpace(Tok, AllocaAddrSpace))
+      return Err;
+    break;
+  }
+  case 'G': { // Default address space for global variables.
+    if (Error Err = getAddrSpace(Tok, DefaultGlobalsAddrSpace))
+      return Err;
+    break;
+  }
+  case 'm':
+    if (!Tok.empty())
+      return reportError("Unexpected trailing characters after mangling "
+                         "specifier in datalayout string");
+    if (Rest.empty())
+      return reportError("Expected mangling specifier in datalayout string");
+    if (Rest.size() > 1)
+      return reportError("Unknown mangling specifier in datalayout string");
+    switch (Rest[0]) {
+    default:
+      return reportError("Unknown mangling in datalayout string");
+    case 'e':
+      ManglingMode = MM_ELF;
       break;
-    }
-    case 'A': { // Default stack/alloca address space.
-      if (Error Err = getAddrSpace(Tok, AllocaAddrSpace))
-        return Err;
+    case 'l':
+      ManglingMode = MM_GOFF;
       break;
-    }
-    case 'G': { // Default address space for global variables.
-      if (Error Err = getAddrSpace(Tok, DefaultGlobalsAddrSpace))
-        return Err;
+    case 'o':
+      ManglingMode = MM_MachO;
       break;
-    }
     case 'm':
-      if (!Tok.empty())
-        return reportError("Unexpected trailing characters after mangling "
-                           "specifier in datalayout string");
-      if (Rest.empty())
-        return reportError("Expected mangling specifier in datalayout string");
-      if (Rest.size() > 1)
-        return reportError("Unknown mangling specifier in datalayout string");
-      switch(Rest[0]) {
-      default:
-        return reportError("Unknown mangling in datalayout string");
-      case 'e':
-        ManglingMode = MM_ELF;
-        break;
-      case 'l':
-        ManglingMode = MM_GOFF;
-        break;
-      case 'o':
-        ManglingMode = MM_MachO;
-        break;
-      case 'm':
-        ManglingMode = MM_Mips;
-        break;
-      case 'w':
-        ManglingMode = MM_WinCOFF;
-        break;
-      case 'x':
-        ManglingMode = MM_WinCOFFX86;
-        break;
-      case 'a':
-        ManglingMode = MM_XCOFF;
-        break;
-      }
+      ManglingMode = MM_Mips;
       break;
-    default:
-      return reportError("Unknown specifier in datalayout string");
+    case 'w':
+      ManglingMode = MM_WinCOFF;
+      break;
+    case 'x':
+      ManglingMode = MM_WinCOFFX86;
+      break;
+    case 'a':
+      ManglingMode = MM_XCOFF;
       break;
     }
+    break;
+  default:
+    return reportError("Unknown specifier in datalayout string");
+  }
+
+  return Error::success();
+}
+
+Error DataLayout::parseLayoutString(StringRef LayoutString) {
+  StringRepresentation = std::string(LayoutString);
+
+  if (LayoutString.empty())
+    return Error::success();
+
+  // Split the data layout string into specifications separated by '-' and
+  // parse each specification individually, updating internal data structures.
+  for (StringRef Spec : split(LayoutString, '-')) {
+    if (Spec.empty())
+      return createStringError("empty specification is not allowed");
+    if (Error Err = parseSpecification(Spec))
+      return Err;
   }
 
   return Error::success();
diff --git a/llvm/unittests/IR/DataLayoutTest.cpp b/llvm/unittests/IR/DataLayoutTest.cpp
index dcb2e614f4c40de..f5c930ebdbb9c46 100644
--- a/llvm/unittests/IR/DataLayoutTest.cpp
+++ b/llvm/unittests/IR/DataLayoutTest.cpp
@@ -124,6 +124,16 @@ TEST(DataLayoutTest, ParseErrors) {
       FailedWithMessage("Alignment is neither 0 nor a power of 2"));
 }
 
+TEST(DataLayout, LayoutStringFormat) {
+  for (StringRef Str : {"", "e", "m:e", "m:e-e"})
+    EXPECT_THAT_EXPECTED(DataLayout::parse(Str), Succeeded());
+
+  for (StringRef Str : {"-", "e-", "-m:e", "m:e--e"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(Str),
+        FailedWithMessage("empty specification is not allowed"));
+}
+
 TEST(DataLayoutTest, CopyAssignmentInvalidatesStructLayout) {
   DataLayout DL1 = cantFail(DataLayout::parse("p:32:32"));
   DataLayout DL2 = cantFail(DataLayout::parse("p:64:64"));

From aaab4fcf656df30a533848d06400544d01992393 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Thu, 15 Aug 2024 15:19:41 +0200
Subject: [PATCH 028/441] Revert "[SLP][NFC]Remove unused using declarations,
 reduce mem usage in containers, NFC"

This reverts commit e1b15504a831e63af6fb9a6e83faaa10ef425ae6.

This causes compile-time regressions, see:
http://llvm-compile-time-tracker.com/compare.php?from=e687a9f2dd389a54a10456e57693f93df0c64c02&to=e1b15504a831e63af6fb9a6e83faaa10ef425ae6&stat=instructions:u

Probably some of the new SmallVector sizes are sub-optimal.
---
 .../llvm/Transforms/Vectorize/SLPVectorizer.h |  4 +--
 .../Transforms/Vectorize/SLPVectorizer.cpp    | 34 ++++++++++---------
 2 files changed, 20 insertions(+), 18 deletions(-)

diff --git a/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h b/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
index 809beadb5f7df3c..95531544a1c8176 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
@@ -56,9 +56,9 @@ class BoUpSLP;
 } // end namespace slpvectorizer
 
 struct SLPVectorizerPass : public PassInfoMixin<SLPVectorizerPass> {
-  using StoreList = SmallVector<StoreInst *, 4>;
+  using StoreList = SmallVector<StoreInst *, 8>;
   using StoreListMap = MapVector<Value *, StoreList>;
-  using GEPList = SmallVector<GetElementPtrInst *, 4>;
+  using GEPList = SmallVector<GetElementPtrInst *, 8>;
   using GEPListMap = MapVector<Value *, GEPList>;
   using InstSetVector = SmallSetVector<Instruction *, 8>;
 
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 81c4193cfe1081e..81841a8f692870f 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -1243,11 +1243,13 @@ class BoUpSLP {
     StridedVectorize
   };
 
-  using ValueList = SmallVector<Value *, 4>;
-  using ValueSet = SmallPtrSet<Value *, 8>;
+  using ValueList = SmallVector<Value *, 8>;
+  using InstrList = SmallVector<Instruction *, 16>;
+  using ValueSet = SmallPtrSet<Value *, 16>;
+  using StoreList = SmallVector<StoreInst *, 8>;
   using ExtraValueToDebugLocsMap =
       MapVector<Value *, SmallVector<Instruction *, 2>>;
-  using OrdersType = SmallVector<unsigned, 0>;
+  using OrdersType = SmallVector<unsigned, 4>;
 
   BoUpSLP(Function *Func, ScalarEvolution *Se, TargetTransformInfo *Tti,
           TargetLibraryInfo *TLi, AAResults *Aa, LoopInfo *Li,
@@ -1469,7 +1471,7 @@ class BoUpSLP {
   /// \param TryRecursiveCheck used to check if long masked gather can be
   /// represented as a serie of loads/insert subvector, if profitable.
   LoadsState canVectorizeLoads(ArrayRef<Value *> VL, const Value *VL0,
-                               OrdersType &Order,
+                               SmallVectorImpl<unsigned> &Order,
                                SmallVectorImpl<Value *> &PointerOps,
                                bool TryRecursiveCheck = true) const;
 
@@ -2838,7 +2840,7 @@ class BoUpSLP {
   /// \param ResizeAllowed indicates whether it is allowed to handle subvector
   /// extract order.
   bool canReuseExtract(ArrayRef<Value *> VL, Value *OpValue,
-                       OrdersType &CurrentOrder,
+                       SmallVectorImpl<unsigned> &CurrentOrder,
                        bool ResizeAllowed = false) const;
 
   /// Vectorize a single entry in the tree.
@@ -3082,10 +3084,10 @@ class BoUpSLP {
     CombinedOpcode CombinedOp = NotCombinedOp;
 
     /// Does this sequence require some shuffling?
-    SmallVector<int, 0> ReuseShuffleIndices;
+    SmallVector<int, 4> ReuseShuffleIndices;
 
     /// Does this entry require reordering?
-    OrdersType ReorderIndices;
+    SmallVector<unsigned, 4> ReorderIndices;
 
     /// Points back to the VectorizableTree.
     ///
@@ -3106,7 +3108,7 @@ class BoUpSLP {
     /// The operands of each instruction in each lane Operands[op_index][lane].
     /// Note: This helps avoid the replication of the code that performs the
     /// reordering of operands during buildTree_rec() and vectorizeTree().
-    SmallVector<ValueList, 0> Operands;
+    SmallVector<ValueList, 2> Operands;
 
     /// The main/alternate instruction.
     Instruction *MainOp = nullptr;
@@ -3714,13 +3716,13 @@ class BoUpSLP {
 
     /// The dependent memory instructions.
     /// This list is derived on demand in calculateDependencies().
-    SmallVector<ScheduleData *, 0> MemoryDependencies;
+    SmallVector<ScheduleData *, 4> MemoryDependencies;
 
     /// List of instructions which this instruction could be control dependent
     /// on.  Allowing such nodes to be scheduled below this one could introduce
     /// a runtime fault which didn't exist in the original program.
     /// ex: this is a load or udiv following a readonly call which inf loops
-    SmallVector<ScheduleData *, 0> ControlDependencies;
+    SmallVector<ScheduleData *, 4> ControlDependencies;
 
     /// This ScheduleData is in the current scheduling region if this matches
     /// the current SchedulingRegionID of BlockScheduling.
@@ -4298,12 +4300,12 @@ static void reorderReuses(SmallVectorImpl<int> &Reuses, ArrayRef<int> Mask) {
 /// the original order of the scalars. Procedure transforms the provided order
 /// in accordance with the given \p Mask. If the resulting \p Order is just an
 /// identity order, \p Order is cleared.
-static void reorderOrder(BoUpSLP::OrdersType &Order, ArrayRef<int> Mask,
+static void reorderOrder(SmallVectorImpl<unsigned> &Order, ArrayRef<int> Mask,
                          bool BottomOrder = false) {
   assert(!Mask.empty() && "Expected non-empty mask.");
   unsigned Sz = Mask.size();
   if (BottomOrder) {
-    BoUpSLP::OrdersType PrevOrder;
+    SmallVector<unsigned> PrevOrder;
     if (Order.empty()) {
       PrevOrder.resize(Sz);
       std::iota(PrevOrder.begin(), PrevOrder.end(), 0);
@@ -4693,7 +4695,7 @@ getShuffleCost(const TargetTransformInfo &TTI, TTI::ShuffleKind Kind,
 }
 
 BoUpSLP::LoadsState BoUpSLP::canVectorizeLoads(
-    ArrayRef<Value *> VL, const Value *VL0, OrdersType &Order,
+    ArrayRef<Value *> VL, const Value *VL0, SmallVectorImpl<unsigned> &Order,
     SmallVectorImpl<Value *> &PointerOps, bool TryRecursiveCheck) const {
   // Check that a vectorized load would load the same memory as a scalar
   // load. For example, we don't want to vectorize loads that are smaller
@@ -4821,7 +4823,7 @@ BoUpSLP::LoadsState BoUpSLP::canVectorizeLoads(
         for (unsigned Cnt = 0, End = VL.size(); Cnt + VF <= End;
              Cnt += VF, ++VectorizedCnt) {
           ArrayRef<Value *> Slice = VL.slice(Cnt, VF);
-          OrdersType Order;
+          SmallVector<unsigned> Order;
           SmallVector<Value *> PointerOps;
           LoadsState LS =
               canVectorizeLoads(Slice, Slice.front(), Order, PointerOps,
@@ -5395,7 +5397,7 @@ void BoUpSLP::reorderNodeWithReuses(TreeEntry &TE, ArrayRef<int> Mask) const {
   TE.ReorderIndices.clear();
   // Try to improve gathered nodes with clustered reuses, if possible.
   ArrayRef<int> Slice = ArrayRef(NewMask).slice(0, Sz);
-  OrdersType NewOrder(Slice);
+  SmallVector<unsigned> NewOrder(Slice);
   inversePermutation(NewOrder, NewMask);
   reorderScalars(TE.Scalars, NewMask);
   // Fill the reuses mask with the identity submasks.
@@ -7715,7 +7717,7 @@ unsigned BoUpSLP::canMapToVector(Type *T) const {
 }
 
 bool BoUpSLP::canReuseExtract(ArrayRef<Value *> VL, Value *OpValue,
-                              OrdersType &CurrentOrder,
+                              SmallVectorImpl<unsigned> &CurrentOrder,
                               bool ResizeAllowed) const {
   const auto *It = find_if(VL, IsaPred<ExtractElementInst, ExtractValueInst>);
   assert(It != VL.end() && "Expected at least one extract instruction.");

From 75cb7de404ee236d6297c551740a2681583d7e5e Mon Sep 17 00:00:00 2001
From: earnol <earnol@users.noreply.github.com>
Date: Thu, 15 Aug 2024 10:15:27 -0400
Subject: [PATCH 029/441] [ubsan] Display correct runtime messages for negative
 _BitInt (#96240)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Without this patch compiler-rt ubsan library has a bug displaying
incorrect values for variables of the _BitInt (previously called
_ExtInt) type. This patch affects affects both: generation of metadata
inside code generator and runtime part. The runtime part provided only
for i386 and x86_64 runtimes. Other runtimes should be updated to take
full benefit of this patch.
The patch is constructed the way to be backward compatible and int and
float type runtime diagnostics should be unaffected for not yet updated
runtimes.

This patch fixes issue
https://github.com/llvm/llvm-project/issues/64100.

Co-authored-by: Eänolituri Lómitaurë <vladislav.aranov@ericsson.com>
Co-authored-by: Aaron Ballman <aaron@aaronballman.com>
Co-authored-by: Paul Kirth <paulkirth@google.com>
---
 clang/lib/CodeGen/CGExpr.cpp                  |  59 +++++-
 clang/test/CodeGen/bit-int-ubsan.c            |  96 ++++++++++
 compiler-rt/lib/ubsan/ubsan_value.cpp         |  17 +-
 compiler-rt/lib/ubsan/ubsan_value.h           |  33 +++-
 .../ubsan/TestCases/Integer/bit-int-pass.c    |  42 +++++
 .../test/ubsan/TestCases/Integer/bit-int.c    | 170 ++++++++++++++++++
 6 files changed, 404 insertions(+), 13 deletions(-)
 create mode 100644 clang/test/CodeGen/bit-int-ubsan.c
 create mode 100644 compiler-rt/test/ubsan/TestCases/Integer/bit-int-pass.c
 create mode 100644 compiler-rt/test/ubsan/TestCases/Integer/bit-int.c

diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index 0672861790633bb..48d9a3b8a5acb3b 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -42,6 +42,7 @@
 #include "llvm/IR/MatrixBuilder.h"
 #include "llvm/Passes/OptimizationLevel.h"
 #include "llvm/Support/ConvertUTF.h"
+#include "llvm/Support/Endian.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/SaveAndRestore.h"
@@ -65,6 +66,22 @@ static llvm::cl::opt<bool> ClSanitizeGuardChecks(
     "ubsan-guard-checks", llvm::cl::Optional,
     llvm::cl::desc("Guard UBSAN checks with `llvm.allow.ubsan.check()`."));
 
+//===--------------------------------------------------------------------===//
+//                        Defines for metadata
+//===--------------------------------------------------------------------===//
+
+// Those values are crucial to be the SAME as in ubsan runtime library.
+enum VariableTypeDescriptorKind : uint16_t {
+  /// An integer type.
+  TK_Integer = 0x0000,
+  /// A floating-point type.
+  TK_Float = 0x0001,
+  /// An _BitInt(N) type.
+  TK_BitInt = 0x0002,
+  /// Any other type. The value representation is unspecified.
+  TK_Unknown = 0xffff
+};
+
 //===--------------------------------------------------------------------===//
 //                        Miscellaneous Helper Methods
 //===--------------------------------------------------------------------===//
@@ -3288,22 +3305,40 @@ LValue CodeGenFunction::EmitPredefinedLValue(const PredefinedExpr *E) {
 ///   { i16 TypeKind, i16 TypeInfo }
 /// \endcode
 ///
-/// followed by an array of i8 containing the type name. TypeKind is 0 for an
-/// integer, 1 for a floating point value, and -1 for anything else.
+/// followed by an array of i8 containing the type name with extra information
+/// for BitInt. TypeKind is TK_Integer(0) for an integer, TK_Float(1) for a
+/// floating point value, TK_BitInt(2) for BitInt and TK_Unknown(0xFFFF) for
+/// anything else.
 llvm::Constant *CodeGenFunction::EmitCheckTypeDescriptor(QualType T) {
   // Only emit each type's descriptor once.
   if (llvm::Constant *C = CGM.getTypeDescriptorFromMap(T))
     return C;
 
-  uint16_t TypeKind = -1;
+  uint16_t TypeKind = TK_Unknown;
   uint16_t TypeInfo = 0;
+  bool IsBitInt = false;
 
   if (T->isIntegerType()) {
-    TypeKind = 0;
+    TypeKind = TK_Integer;
     TypeInfo = (llvm::Log2_32(getContext().getTypeSize(T)) << 1) |
                (T->isSignedIntegerType() ? 1 : 0);
+    // Follow suggestion from discussion of issue 64100.
+    // So we can write the exact amount of bits in TypeName after '\0'
+    // making it <diagnostic-like type name>.'\0'.<32-bit width>.
+    if (T->isSignedIntegerType() && T->getAs<BitIntType>()) {
+      // Do a sanity checks as we are using 32-bit type to store bit length.
+      assert(getContext().getTypeSize(T) > 0 &&
+             " non positive amount of bits in __BitInt type");
+      assert(getContext().getTypeSize(T) <= 0xFFFFFFFF &&
+             " too many bits in __BitInt type");
+
+      // Redefine TypeKind with the actual __BitInt type if we have signed
+      // BitInt.
+      TypeKind = TK_BitInt;
+      IsBitInt = true;
+    }
   } else if (T->isFloatingType()) {
-    TypeKind = 1;
+    TypeKind = TK_Float;
     TypeInfo = getContext().getTypeSize(T);
   }
 
@@ -3314,6 +3349,20 @@ llvm::Constant *CodeGenFunction::EmitCheckTypeDescriptor(QualType T) {
       DiagnosticsEngine::ak_qualtype, (intptr_t)T.getAsOpaquePtr(), StringRef(),
       StringRef(), std::nullopt, Buffer, std::nullopt);
 
+  if (IsBitInt) {
+    // The Structure is: 0 to end the string, 32 bit unsigned integer in target
+    // endianness, zero.
+    char S[6] = {'\0', '\0', '\0', '\0', '\0', '\0'};
+    const auto *EIT = T->castAs<BitIntType>();
+    uint32_t Bits = EIT->getNumBits();
+    llvm::support::endian::write32(S + 1, Bits,
+                                   getTarget().isBigEndian()
+                                       ? llvm::endianness::big
+                                       : llvm::endianness::little);
+    StringRef Str = StringRef(S, sizeof(S) / sizeof(decltype(S[0])));
+    Buffer.append(Str);
+  }
+
   llvm::Constant *Components[] = {
     Builder.getInt16(TypeKind), Builder.getInt16(TypeInfo),
     llvm::ConstantDataArray::getString(getLLVMContext(), Buffer)
diff --git a/clang/test/CodeGen/bit-int-ubsan.c b/clang/test/CodeGen/bit-int-ubsan.c
new file mode 100644
index 000000000000000..35f96963c181d15
--- /dev/null
+++ b/clang/test/CodeGen/bit-int-ubsan.c
@@ -0,0 +1,96 @@
+// REQUIRES: x86-registered-target
+// RUN: %clang -Wno-constant-conversion -Wno-array-bounds -Wno-division-by-zero -Wno-shift-negative-value -Wno-shift-count-negative -Wno-int-to-pointer-cast -fsanitize=array-bounds,enum,float-cast-overflow,integer-divide-by-zero,implicit-unsigned-integer-truncation,implicit-signed-integer-truncation,implicit-integer-sign-change,unsigned-integer-overflow,signed-integer-overflow,shift-base,shift-exponent -O0 -S -emit-llvm -o - %s | FileCheck %s
+
+// The runtime test checking the _BitInt ubsan feature is located in compiler-rt/test/ubsan/TestCases/Integer/bit-int.c
+
+#include <stdint.h>
+#include <stdio.h>
+
+uint32_t float_divide_by_zero() {
+  float f = 1.0f / 0.0f;
+  // CHECK: constant { i16, i16, [8 x i8] } { i16 1, i16 32, [8 x i8] c"'float'\00" }
+  _BitInt(37) r = (_BitInt(37))f;
+  // CHECK: constant { i16, i16, [20 x i8] } { i16 2, i16 13, [20 x i8] c"'_BitInt(37)'\00%\00\00\00\00\00" }
+  return r;
+}
+
+uint32_t integer_divide_by_zero() __attribute__((no_sanitize("memory"))) {
+  _BitInt(37) x = 1 / 0;
+  // CHECK: constant { i16, i16, [32 x i8] } { i16 0, i16 10, [32 x i8] c"'uint32_t' (aka 'unsigned int')\00" }
+  return x;
+}
+
+uint32_t implicit_unsigned_integer_truncation() {
+  unsigned _BitInt(37) x = 2U;
+  x += float_divide_by_zero();
+  x += integer_divide_by_zero();
+  x = x + 0xFFFFFFFFFFFFFFFFULL;
+  // CHECK: constant { i16, i16, [23 x i8] } { i16 0, i16 12, [23 x i8] c"'unsigned _BitInt(37)'\00" }
+  uint32_t r = x & 0xFFFFFFFF;
+  return r;
+}
+
+uint32_t array_bounds() {
+  _BitInt(37) x[4];
+  _BitInt(37) y = x[10];
+  // CHECK: constant { i16, i16, [17 x i8] } { i16 -1, i16 0, [17 x i8] c"'_BitInt(37)[4]'\00" }
+  return (uint32_t)y;
+}
+
+uint32_t float_cast_overflow() {
+  float a = 100000000.0f;
+  _BitInt(7) b = (_BitInt(7))a;
+  // CHECK: constant { i16, i16, [19 x i8] } { i16 2, i16 7, [19 x i8] c"'_BitInt(7)'\00\07\00\00\00\00\00" }
+  return b;
+}
+
+_BitInt(13) implicit_signed_integer_truncation() {
+  _BitInt(73) x = (_BitInt(73)) ~((~0UL) >> 1);
+  return x;
+  // CHECK: constant { i16, i16, [20 x i8] } { i16 2, i16 {{([[:xdigit:]]{2})}}, [20 x i8] c"'_BitInt(73)'\00I\00\00\00\00\00" }
+  // CHECK: constant { i16, i16, [20 x i8] } { i16 2, i16 9, [20 x i8] c"'_BitInt(13)'\00\0D\00\00\00\00\00" }
+}
+
+uint32_t negative_shift1(unsigned _BitInt(37) x)
+    __attribute__((no_sanitize("memory"))) {
+  _BitInt(9) c = -2;
+  return x >> c;
+  // CHECK: constant { i16, i16, [19 x i8] } { i16 2, i16 9, [19 x i8] c"'_BitInt(9)'\00\09\00\00\00\00\00" }
+}
+
+uint32_t negative_shift2(unsigned _BitInt(37) x)
+    __attribute__((no_sanitize("memory"))) {
+  _BitInt(17) c = -2;
+  return x >> c;
+  // CHECK: constant { i16, i16, [20 x i8] } { i16 2, i16 11, [20 x i8] c"'_BitInt(17)'\00\11\00\00\00\00\00" }
+}
+
+uint32_t negative_shift3(unsigned _BitInt(37) x)
+    __attribute__((no_sanitize("memory"))) {
+  _BitInt(34) c = -2;
+  return x >> c;
+  // CHECK: constant { i16, i16, [20 x i8] } { i16 2, i16 13, [20 x i8] c"'_BitInt(34)'\00\22\00\00\00\00\00" }
+}
+
+uint32_t negative_shift5(unsigned _BitInt(37) x)
+    __attribute__((no_sanitize("memory"))) {
+  _BitInt(68) c = -2;
+  return x >> c;
+  // CHECK: constant { i16, i16, [20 x i8] } { i16 2, i16 {{([[:xdigit:]]{2})}}, [20 x i8] c"'_BitInt(68)'\00D\00\00\00\00\00" }
+}
+
+int main(int argc, char **argv) {
+  // clang-format off
+  uint64_t result =
+      1ULL +
+      implicit_unsigned_integer_truncation() +
+      (uint32_t)array_bounds() +
+      float_cast_overflow() +
+      (uint64_t)implicit_signed_integer_truncation() +
+      negative_shift1(5) +
+      negative_shift2(5) +
+      negative_shift3(5) +
+      negative_shift5(5);
+  // clang-format on
+  printf("%u\n", (uint32_t)(result & 0xFFFFFFFF));
+}
diff --git a/compiler-rt/lib/ubsan/ubsan_value.cpp b/compiler-rt/lib/ubsan/ubsan_value.cpp
index dc61e5b939d952b..6e88ebaf34d4be3 100644
--- a/compiler-rt/lib/ubsan/ubsan_value.cpp
+++ b/compiler-rt/lib/ubsan/ubsan_value.cpp
@@ -67,18 +67,21 @@ const char *__ubsan::getObjCClassName(ValueHandle Pointer) {
 
 SIntMax Value::getSIntValue() const {
   CHECK(getType().isSignedIntegerTy());
+  // Val was zero-extended to ValueHandle. Sign-extend from original width
+  // to SIntMax.
+  const unsigned ExtraBits =
+      sizeof(SIntMax) * 8 - getType().getIntegerBitCount();
   if (isInlineInt()) {
-    // Val was zero-extended to ValueHandle. Sign-extend from original width
-    // to SIntMax.
-    const unsigned ExtraBits =
-      sizeof(SIntMax) * 8 - getType().getIntegerBitWidth();
     return SIntMax(UIntMax(Val) << ExtraBits) >> ExtraBits;
   }
-  if (getType().getIntegerBitWidth() == 64)
-    return *reinterpret_cast<s64*>(Val);
+  if (getType().getIntegerBitWidth() == 64) {
+    return SIntMax(UIntMax(*reinterpret_cast<s64 *>(Val)) << ExtraBits) >>
+           ExtraBits;
+  }
 #if HAVE_INT128_T
   if (getType().getIntegerBitWidth() == 128)
-    return *reinterpret_cast<s128*>(Val);
+    return SIntMax(UIntMax(*reinterpret_cast<s128 *>(Val)) << ExtraBits) >>
+           ExtraBits;
 #else
   if (getType().getIntegerBitWidth() == 128)
     UNREACHABLE("libclang_rt.ubsan was built without __int128 support");
diff --git a/compiler-rt/lib/ubsan/ubsan_value.h b/compiler-rt/lib/ubsan/ubsan_value.h
index e0957276dd24197..430c9ea0dc8d150 100644
--- a/compiler-rt/lib/ubsan/ubsan_value.h
+++ b/compiler-rt/lib/ubsan/ubsan_value.h
@@ -103,6 +103,13 @@ class TypeDescriptor {
     /// representation is that of bitcasting the floating-point value to an
     /// integer type.
     TK_Float = 0x0001,
+    /// An _BitInt(N) type. Lowest bit is 1 for a signed value, 0 for an
+    /// unsigned value. Remaining bits are log_2(bit_width). The value
+    /// representation is the integer itself if it fits into a ValueHandle, and
+    /// a pointer to the integer otherwise. TypeName contains the true width
+    /// of the type for the signed _BitInt(N) type stored after zero bit after
+    /// TypeName as 32-bit unsigned integer.
+    TK_BitInt = 0x0002,
     /// Any other type. The value representation is unspecified.
     TK_Unknown = 0xffff
   };
@@ -113,10 +120,15 @@ class TypeDescriptor {
     return static_cast<Kind>(TypeKind);
   }
 
-  bool isIntegerTy() const { return getKind() == TK_Integer; }
+  bool isIntegerTy() const {
+    return getKind() == TK_Integer || getKind() == TK_BitInt;
+  }
+  bool isBitIntTy() const { return getKind() == TK_BitInt; }
+
   bool isSignedIntegerTy() const {
     return isIntegerTy() && (TypeInfo & 1);
   }
+  bool isSignedBitIntTy() const { return isBitIntTy() && (TypeInfo & 1); }
   bool isUnsignedIntegerTy() const {
     return isIntegerTy() && !(TypeInfo & 1);
   }
@@ -125,6 +137,25 @@ class TypeDescriptor {
     return 1 << (TypeInfo >> 1);
   }
 
+  const char *getBitIntBitCountPointer() const {
+    DCHECK(isBitIntTy());
+    DCHECK(isSignedBitIntTy());
+    // Scan Name for zero and return the next address
+    const char *p = getTypeName();
+    while (*p != '\0')
+      ++p;
+    // Return the next address
+    return p + 1;
+  }
+
+  unsigned getIntegerBitCount() const {
+    DCHECK(isIntegerTy());
+    if (isSignedBitIntTy())
+      return *reinterpret_cast<const u32 *>(getBitIntBitCountPointer());
+    else
+      return getIntegerBitWidth();
+  }
+
   bool isFloatTy() const { return getKind() == TK_Float; }
   unsigned getFloatBitWidth() const {
     CHECK(isFloatTy());
diff --git a/compiler-rt/test/ubsan/TestCases/Integer/bit-int-pass.c b/compiler-rt/test/ubsan/TestCases/Integer/bit-int-pass.c
new file mode 100644
index 000000000000000..a25428f0eb872f9
--- /dev/null
+++ b/compiler-rt/test/ubsan/TestCases/Integer/bit-int-pass.c
@@ -0,0 +1,42 @@
+// RUN: %clang -Wno-constant-conversion -Wno-array-bounds -Wno-division-by-zero -Wno-shift-negative-value -Wno-shift-count-negative -Wno-int-to-pointer-cast -O0 -fsanitize=alignment,array-bounds,bool,float-cast-overflow,implicit-integer-sign-change,implicit-signed-integer-truncation,implicit-unsigned-integer-truncation,integer-divide-by-zero,nonnull-attribute,null,nullability-arg,nullability-assign,nullability-return,pointer-overflow,returns-nonnull-attribute,shift-base,shift-exponent,signed-integer-overflow,unreachable,unsigned-integer-overflow,unsigned-shift-base,vla-bound %s -o %t1 && %run %t1 2>&1 | FileCheck %s
+
+#include <stdint.h>
+#include <stdio.h>
+
+// In this test there is an expectation of assignment of _BitInt not producing any output.
+uint32_t nullability_arg(_BitInt(37) *_Nonnull x)
+    __attribute__((no_sanitize("address")))
+    __attribute__((no_sanitize("memory"))) {
+  _BitInt(37) y = *(_BitInt(37) *)&x;
+  return (y > 0) ? y : 0;
+}
+
+// In this test there is an expectation of ubsan not triggeting on returning random address which is inside address space of the process.
+_BitInt(37) nonnull_attribute(__attribute__((nonnull)) _BitInt(37) * x)
+    __attribute__((no_sanitize("address")))
+    __attribute__((no_sanitize("memory"))) {
+  return *(_BitInt(37) *)&x;
+}
+
+// In this test there is an expectation of assignment of uint32_t from "invalid" _BitInt is not producing any output.
+uint32_t nullability_assign(_BitInt(7) * x)
+    __attribute__((no_sanitize("address")))
+    __attribute__((no_sanitize("memory"))) {
+  _BitInt(7) *_Nonnull y = x;
+  int32_t r = *(_BitInt(7) *)&y;
+  return (r > 0) ? r : 0;
+}
+
+// In those examples the file is expected to compile & run with no diagnostics
+// CHECK-NOT: runtime error:
+
+int main(int argc, char **argv) {
+  // clang-format off
+  uint64_t result =
+      1ULL +
+      nullability_arg((_BitInt(37) *)argc) +
+      ((uint64_t)nonnull_attribute((_BitInt(37) *)argc) & 0xFFFFFFFF) +
+      nullability_assign((_BitInt(7) *)argc);
+  // clang-format on
+  printf("%u\n", (uint32_t)(result & 0xFFFFFFFF));
+}
diff --git a/compiler-rt/test/ubsan/TestCases/Integer/bit-int.c b/compiler-rt/test/ubsan/TestCases/Integer/bit-int.c
new file mode 100644
index 000000000000000..18dc10bf3a38899
--- /dev/null
+++ b/compiler-rt/test/ubsan/TestCases/Integer/bit-int.c
@@ -0,0 +1,170 @@
+// RUN: %clang -Wno-constant-conversion -Wno-array-bounds -Wno-division-by-zero -Wno-shift-negative-value -Wno-shift-count-negative -Wno-int-to-pointer-cast -O0 -fsanitize=array-bounds,float-cast-overflow,implicit-integer-sign-change,implicit-signed-integer-truncation,implicit-unsigned-integer-truncation,integer-divide-by-zero,pointer-overflow,shift-base,shift-exponent,signed-integer-overflow,unsigned-integer-overflow,unsigned-shift-base,vla-bound %s -o %t1 && %run %t1 2>&1 | FileCheck %s
+
+#include <stdint.h>
+#include <stdio.h>
+
+uint32_t float_divide_by_zero() {
+  float f = 1.0f / 0.0f;
+  _BitInt(37) r = (_BitInt(37))f;
+  // CHECK: {{.*}}bit-int.c:[[#@LINE-1]]:19: runtime error: inf is outside the range of representable values of type
+  return r;
+}
+
+uint32_t integer_divide_by_zero() __attribute__((no_sanitize("memory"))) {
+  _BitInt(37) x = 1 / 0;
+  // CHECK: {{.*}}bit-int.c:[[#@LINE-1]]:21: runtime error: division by zero
+  return x;
+}
+
+uint32_t implicit_unsigned_integer_truncation() {
+  unsigned _BitInt(37) x = 2U;
+  x += float_divide_by_zero();
+  x += integer_divide_by_zero();
+  x = x + 0xFFFFFFFFFFFFFFFFULL;
+  // CHECK: {{.*}}bit-int.c:[[#@LINE-1]]:9: runtime error: unsigned integer overflow:
+  uint32_t r = x & 0xFFFFFFFF;
+  return r;
+}
+
+uint32_t pointer_overflow() __attribute__((no_sanitize("address"))) {
+  _BitInt(37) *x = (_BitInt(37) *)1;
+  _BitInt(37) *y = x - 1;
+  // CHECK: {{.*}}bit-int.c:[[#@LINE-1]]:22: runtime error: pointer index expression with base
+  uint32_t r = *(_BitInt(37) *)&y;
+  // CHECK: {{.*}}bit-int.c:[[#@LINE-1]]:16: runtime error: implicit conversion from type
+  return r;
+}
+
+uint32_t vla_bound(_BitInt(37) x) {
+  _BitInt(37) a[x - 1];
+  // CHECK: {{.*}}bit-int.c:[[#@LINE-1]]:17: runtime error: variable length array bound evaluates to non-positive value
+  return 0;
+}
+
+uint32_t unsigned_shift_base() {
+  unsigned _BitInt(37) x = ~0U << 1;
+  // CHECK: {{.*}}bit-int.c:[[#@LINE-1]]:32: runtime error: left shift of 4294967295 by 1 places cannot be represented in type
+  return x;
+}
+
+uint32_t array_bounds() {
+  _BitInt(37) x[4];
+  _BitInt(37) y = x[10];
+  // CHECK: {{.*}}bit-int.c:[[#@LINE-1]]:19: runtime error: index 10 out of bounds for type
+  return (uint32_t)y;
+}
+
+uint32_t float_cast_overflow() {
+  float a = 100000000.0f;
+  _BitInt(7) b = (_BitInt(7))a;
+  // CHECK: {{.*}}bit-int.c:[[#@LINE-1]]:18: runtime error: 1e+08 is outside the range of representable values of type
+  return b;
+}
+
+uint32_t implicit_integer_sign_change(unsigned _BitInt(37) x) {
+  _BitInt(37) r = x;
+  // CHECK: {{.*}}bit-int.c:[[#@LINE-1]]:19: runtime error: implicit conversion from type '{{[^']+}}' of value
+  return r & 0xFFFFFFFF;
+}
+
+_BitInt(13) implicit_signed_integer_truncation() {
+#ifdef __SIZEOF_INT128__
+  _BitInt(73) x = (_BitInt(73)) ~((~0UL) >> 1);
+#else
+  uint32_t x = 0x7FFFFFFFUL;
+#endif
+  return x;
+  // CHECK: {{.*}}bit-int.c:[[#@LINE-1]]:10: runtime error: implicit conversion from type
+}
+
+_BitInt(37) shift_exponent() __attribute__((no_sanitize("memory"))) {
+  _BitInt(37) x = 1 << (-1);
+  // CHECK: {{.*}}bit-int.c:[[#@LINE-1]]:21: runtime error: shift exponent -1 is negative
+  return x;
+}
+
+_BitInt(37) shift_base() __attribute__((no_sanitize("memory"))) {
+  _BitInt(37) x = (-1) << 1;
+  // CHECK: {{.*}}bit-int.c:[[#@LINE-1]]:24: runtime error: left shift of negative value -1
+  return x;
+}
+
+uint32_t negative_shift1(unsigned _BitInt(37) x)
+    __attribute__((no_sanitize("memory"))) {
+  _BitInt(9) c = -2;
+  return x >> c;
+  // CHECK: {{.*}}bit-int.c:[[#@LINE-1]]:12: runtime error: shift exponent -2 is negative
+}
+
+uint32_t negative_shift2(unsigned _BitInt(37) x)
+    __attribute__((no_sanitize("memory"))) {
+  _BitInt(17) c = -2;
+  return x >> c;
+  // CHECK: {{.*}}bit-int.c:[[#@LINE-1]]:12: runtime error: shift exponent -2 is negative
+}
+
+uint32_t negative_shift3(unsigned _BitInt(37) x)
+    __attribute__((no_sanitize("memory"))) {
+  _BitInt(34) c = -2;
+  return x >> c;
+  // CHECK: {{.*}}bit-int.c:[[#@LINE-1]]:12: runtime error: shift exponent -2 is negative
+}
+
+uint32_t negative_shift4(unsigned _BitInt(37) x)
+    __attribute__((no_sanitize("memory"))) {
+  int64_t c = -2;
+  return x >> c;
+  // CHECK: {{.*}}bit-int.c:[[#@LINE-1]]:12: runtime error: shift exponent -2 is negative
+}
+
+uint32_t negative_shift5(unsigned _BitInt(37) x)
+    __attribute__((no_sanitize("memory"))) {
+#ifdef __SIZEOF_INT128__
+  _BitInt(68) c = -2;
+#else
+  // We cannot check BitInt values > 64 without int128_t support
+  _BitInt(48) c = -2;
+#endif
+  return x >> c;
+  // CHECK: {{.*}}bit-int.c:[[#@LINE-1]]:12: runtime error: shift exponent -2 is negative
+}
+
+uint32_t unsigned_integer_overflow() __attribute__((no_sanitize("memory"))) {
+  unsigned _BitInt(37) x = ~0U;
+  ++x;
+  return x;
+  // CHECK: {{.*}}bit-int.c:[[#@LINE-1]]:10: runtime error: implicit conversion from type
+}
+
+// In this test no run-time overflow expected, so no diagnostics here, but should be a conversion error from the negative number on return.
+uint32_t signed_integer_overflow() __attribute__((no_sanitize("memory"))) {
+  _BitInt(37) x = (_BitInt(37)) ~((0x8FFFFFFFFFFFFFFFULL) >> 1);
+  --x;
+  return x;
+  // CHECK: {{.*}}bit-int.c:[[#@LINE-1]]:10: runtime error: implicit conversion from type
+}
+
+int main(int argc, char **argv) {
+  // clang-format off
+  uint64_t result =
+      1ULL +
+      implicit_unsigned_integer_truncation() +
+      pointer_overflow() +
+      vla_bound(argc) +
+      unsigned_shift_base() +
+      (uint32_t)array_bounds() +
+      float_cast_overflow() +
+      implicit_integer_sign_change((unsigned _BitInt(37))(argc - 2)) +
+      (uint64_t)implicit_signed_integer_truncation() +
+      shift_exponent() +
+      (uint32_t)shift_base() +
+      negative_shift1(5) +
+      negative_shift2(5) +
+      negative_shift3(5) +
+      negative_shift4(5) +
+      negative_shift5(5) +
+      unsigned_integer_overflow() +
+      signed_integer_overflow();
+  // clang-format on
+  printf("%u\n", (uint32_t)(result & 0xFFFFFFFF));
+}

From 65ac12d3c9877ecf5b97552364e7eead887d94eb Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev@outlook.com>
Date: Thu, 15 Aug 2024 07:21:10 -0700
Subject: [PATCH 030/441] [SLP][NFC]Add a test with incorrect minbitwidth
 analysis for reduced operands

---
 .../X86/operand-is-reduced-val.ll             | 46 +++++++++++++++++++
 1 file changed, 46 insertions(+)
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/operand-is-reduced-val.ll

diff --git a/llvm/test/Transforms/SLPVectorizer/X86/operand-is-reduced-val.ll b/llvm/test/Transforms/SLPVectorizer/X86/operand-is-reduced-val.ll
new file mode 100644
index 000000000000000..5fb93e27539d8e7
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/operand-is-reduced-val.ll
@@ -0,0 +1,46 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux < %s -slp-threshold=-10 | FileCheck %s
+
+define i64 @src(i32 %a) {
+; CHECK-LABEL: define i64 @src(
+; CHECK-SAME: i32 [[A:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP17:%.*]] = sext i32 [[A]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i32> poison, i32 [[A]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = add <4 x i32> [[TMP2]], <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP4:%.*]] = sext <4 x i32> [[TMP3]] to <4 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = and <4 x i32> [[TMP3]], <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP6:%.*]] = zext <4 x i32> [[TMP5]] to <4 x i64>
+; CHECK-NEXT:    [[TMP18:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP6]])
+; CHECK-NEXT:    [[TMP16:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP4]])
+; CHECK-NEXT:    [[TMP19:%.*]] = add i64 [[TMP18]], [[TMP16]]
+; CHECK-NEXT:    [[OP_RDX1:%.*]] = add i64 [[TMP19]], 4294967297
+; CHECK-NEXT:    [[TMP21:%.*]] = add i64 [[OP_RDX1]], [[TMP17]]
+; CHECK-NEXT:    ret i64 [[TMP21]]
+;
+entry:
+  %0 = sext i32 %a to i64
+  %1 = add nsw i64 %0, 4294967297
+  %2 = sext i32 %a to i64
+  %3 = add nsw i64 %2, 4294967297
+  %4 = add i64 %3, %1
+  %5 = and i64 %3, 1
+  %6 = add i64 %4, %5
+  %7 = sext i32 %a to i64
+  %8 = add nsw i64 %7, 4294967297
+  %9 = add i64 %8, %6
+  %10 = and i64 %8, 1
+  %11 = add i64 %9, %10
+  %12 = sext i32 %a to i64
+  %13 = add nsw i64 %12, 4294967297
+  %14 = add i64 %13, %11
+  %15 = and i64 %13, 1
+  %16 = add i64 %14, %15
+  %17 = sext i32 %a to i64
+  %18 = add nsw i64 %17, 4294967297
+  %19 = add i64 %18, %16
+  %20 = and i64 %18, 1
+  %21 = add i64 %19, %20
+  ret i64 %21
+}

From 13a6a7975256ebdbce85f1174ae2eec735fa0d7a Mon Sep 17 00:00:00 2001
From: Tobias Stadler <mail@stadler-tobias.de>
Date: Thu, 15 Aug 2024 16:27:18 +0200
Subject: [PATCH 031/441] [GlobalISel] Combiner: Fix warning after #102163

Default case in covered switches is illegal.
https://llvm.org/docs/CodingStandards.html#don-t-use-default-labels-in-fully-covered-switches-over-enumerations

Pull Request: https://github.com/llvm/llvm-project/pull/104441

From 56140a8258a3498cfcd9f0f05c182457d43cbfd2 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev@outlook.com>
Date: Thu, 15 Aug 2024 07:57:37 -0700
Subject: [PATCH 032/441] [SLP]Fix PR104422: Wrong value truncation

The minbitwidth restrictions can be skipped only for immediate reduced
values, for other nodes still need to check if external users allow
bitwidth reduction.

Fixes https://github.com/llvm/llvm-project/issues/104422
---
 llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp |  3 ++-
 .../SLPVectorizer/X86/operand-is-reduced-val.ll | 17 ++++++++++-------
 2 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 81841a8f692870f..87b4ed599b5bb15 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -15663,7 +15663,8 @@ bool BoUpSLP::collectValuesToDemote(
   if (any_of(E.Scalars, [&](Value *V) {
         return !all_of(V->users(), [=](User *U) {
           return getTreeEntry(U) ||
-                 (UserIgnoreList && UserIgnoreList->contains(U)) ||
+                 (E.Idx == 0 && UserIgnoreList &&
+                  UserIgnoreList->contains(U)) ||
                  (!isa<CmpInst>(U) && U->getType()->isSized() &&
                   !U->getType()->isScalableTy() &&
                   DL->getTypeSizeInBits(U->getType()) <= BitWidth);
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/operand-is-reduced-val.ll b/llvm/test/Transforms/SLPVectorizer/X86/operand-is-reduced-val.ll
index 5fb93e27539d8e7..5fcac3fbf3bafea 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/operand-is-reduced-val.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/operand-is-reduced-val.ll
@@ -8,15 +8,18 @@ define i64 @src(i32 %a) {
 ; CHECK-NEXT:    [[TMP17:%.*]] = sext i32 [[A]] to i64
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i32> poison, i32 [[A]], i32 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP3:%.*]] = add <4 x i32> [[TMP2]], <i32 1, i32 1, i32 1, i32 1>
-; CHECK-NEXT:    [[TMP4:%.*]] = sext <4 x i32> [[TMP3]] to <4 x i64>
-; CHECK-NEXT:    [[TMP5:%.*]] = and <4 x i32> [[TMP3]], <i32 1, i32 1, i32 1, i32 1>
-; CHECK-NEXT:    [[TMP6:%.*]] = zext <4 x i32> [[TMP5]] to <4 x i64>
+; CHECK-NEXT:    [[TMP3:%.*]] = sext <4 x i32> [[TMP2]] to <4 x i64>
+; CHECK-NEXT:    [[TMP4:%.*]] = add nsw <4 x i64> [[TMP3]], <i64 4294967297, i64 4294967297, i64 4294967297, i64 4294967297>
+; CHECK-NEXT:    [[TMP6:%.*]] = and <4 x i64> [[TMP4]], <i64 1, i64 1, i64 1, i64 1>
 ; CHECK-NEXT:    [[TMP18:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP6]])
 ; CHECK-NEXT:    [[TMP16:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP4]])
-; CHECK-NEXT:    [[TMP19:%.*]] = add i64 [[TMP18]], [[TMP16]]
-; CHECK-NEXT:    [[OP_RDX1:%.*]] = add i64 [[TMP19]], 4294967297
-; CHECK-NEXT:    [[TMP21:%.*]] = add i64 [[OP_RDX1]], [[TMP17]]
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <2 x i64> poison, i64 [[TMP16]], i32 0
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <2 x i64> [[TMP8]], i64 [[TMP18]], i32 1
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <2 x i64> <i64 poison, i64 4294967297>, i64 [[TMP17]], i32 0
+; CHECK-NEXT:    [[TMP11:%.*]] = add <2 x i64> [[TMP9]], [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <2 x i64> [[TMP11]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <2 x i64> [[TMP11]], i32 1
+; CHECK-NEXT:    [[TMP21:%.*]] = add i64 [[TMP12]], [[TMP13]]
 ; CHECK-NEXT:    ret i64 [[TMP21]]
 ;
 entry:

From 95daf1aedfe521704c601a26ad8011c6e237c38a Mon Sep 17 00:00:00 2001
From: Snehasish Kumar <snehasishk@google.com>
Date: Thu, 15 Aug 2024 08:06:41 -0700
Subject: [PATCH 033/441] Allow optimization of __size_returning_new variants.
 (#102258)

https://github.com/llvm/llvm-project/pull/101564 added support to TLI to
detect variants of operator new which provide feedback on the actual
size of memory allocated (http://wg21.link/P0901R5). This patch extends
SimplifyLibCalls to handle hot cold hinting of these variants.
---
 .../llvm/Transforms/Utils/BuildLibCalls.h     |   7 ++
 .../InstCombine/InstCombineCalls.cpp          |   2 +-
 .../Instrumentation/MemProfiler.cpp           |  14 ++-
 llvm/lib/Transforms/Utils/BuildLibCalls.cpp   |  51 +++++++++
 .../lib/Transforms/Utils/SimplifyLibCalls.cpp |  32 +++++-
 .../InstCombine/simplify-libcalls-new.ll      | 100 ++++++++++++++++++
 6 files changed, 199 insertions(+), 7 deletions(-)

diff --git a/llvm/include/llvm/Transforms/Utils/BuildLibCalls.h b/llvm/include/llvm/Transforms/Utils/BuildLibCalls.h
index 429d6a2e05236fd..1979c4af770b02b 100644
--- a/llvm/include/llvm/Transforms/Utils/BuildLibCalls.h
+++ b/llvm/include/llvm/Transforms/Utils/BuildLibCalls.h
@@ -267,6 +267,13 @@ namespace llvm {
                                       IRBuilderBase &B,
                                       const TargetLibraryInfo *TLI,
                                       LibFunc NewFunc, uint8_t HotCold);
+  Value *emitHotColdSizeReturningNew(Value *Num, IRBuilderBase &B,
+                                     const TargetLibraryInfo *TLI,
+                                     LibFunc NewFunc, uint8_t HotCold);
+  Value *emitHotColdSizeReturningNewAligned(Value *Num, Value *Align,
+                                            IRBuilderBase &B,
+                                            const TargetLibraryInfo *TLI,
+                                            LibFunc NewFunc, uint8_t HotCold);
 }
 
 #endif
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index 4f9a5bd2c17f037..2c2e1bc4686a4ed 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -3692,7 +3692,7 @@ Instruction *InstCombinerImpl::tryOptimizeCall(CallInst *CI) {
 
   // Skip optimizing notail and musttail calls so
   // LibCallSimplifier::optimizeCall doesn't have to preserve those invariants.
-  // LibCallSimplifier::optimizeCall should try to preseve tail calls though.
+  // LibCallSimplifier::optimizeCall should try to preserve tail calls though.
   if (CI->isMustTailCall() || CI->isNoTailCall())
     return nullptr;
 
diff --git a/llvm/lib/Transforms/Instrumentation/MemProfiler.cpp b/llvm/lib/Transforms/Instrumentation/MemProfiler.cpp
index 445bf0bb26cc497..4a43120c9a9e7fa 100644
--- a/llvm/lib/Transforms/Instrumentation/MemProfiler.cpp
+++ b/llvm/lib/Transforms/Instrumentation/MemProfiler.cpp
@@ -20,6 +20,7 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Analysis/MemoryBuiltins.h"
 #include "llvm/Analysis/MemoryProfileInfo.h"
+#include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/IR/Constant.h"
 #include "llvm/IR/DataLayout.h"
@@ -753,8 +754,8 @@ stackFrameIncludesInlinedCallStack(ArrayRef<Frame> ProfileCallStack,
   return InlCallStackIter == InlinedCallStack.end();
 }
 
-static bool isNewWithHotColdVariant(Function *Callee,
-                                    const TargetLibraryInfo &TLI) {
+static bool isAllocationWithHotColdVariant(Function *Callee,
+                                           const TargetLibraryInfo &TLI) {
   if (!Callee)
     return false;
   LibFunc Func;
@@ -769,6 +770,8 @@ static bool isNewWithHotColdVariant(Function *Callee,
   case LibFunc_ZnamRKSt9nothrow_t:
   case LibFunc_ZnamSt11align_val_t:
   case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t:
+  case LibFunc_size_returning_new:
+  case LibFunc_size_returning_new_aligned:
     return true;
   case LibFunc_Znwm12__hot_cold_t:
   case LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t:
@@ -778,6 +781,8 @@ static bool isNewWithHotColdVariant(Function *Callee,
   case LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t:
   case LibFunc_ZnamSt11align_val_t12__hot_cold_t:
   case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t:
+  case LibFunc_size_returning_new_hot_cold:
+  case LibFunc_size_returning_new_aligned_hot_cold:
     return ClMemProfMatchHotColdNew;
   default:
     return false;
@@ -945,9 +950,8 @@ readMemprof(Module &M, Function &F, IndexedInstrProfReader *MemProfReader,
       // instruction's locations match the prefix Frame locations on an
       // allocation context with the same leaf.
       if (AllocInfoIter != LocHashToAllocInfo.end()) {
-        // Only consider allocations via new, to reduce unnecessary metadata,
-        // since those are the only allocations that will be targeted initially.
-        if (!isNewWithHotColdVariant(CI->getCalledFunction(), TLI))
+        // Only consider allocations which support hinting.
+        if (!isAllocationWithHotColdVariant(CI->getCalledFunction(), TLI))
           continue;
         // We may match this instruction's location list to multiple MIB
         // contexts. Add them to a Trie specialized for trimming the contexts to
diff --git a/llvm/lib/Transforms/Utils/BuildLibCalls.cpp b/llvm/lib/Transforms/Utils/BuildLibCalls.cpp
index 30a343b2c564e86..c23d53f7111d214 100644
--- a/llvm/lib/Transforms/Utils/BuildLibCalls.cpp
+++ b/llvm/lib/Transforms/Utils/BuildLibCalls.cpp
@@ -19,6 +19,7 @@
 #include "llvm/IR/CallingConv.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DataLayout.h"
+#include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/Module.h"
@@ -1963,6 +1964,56 @@ Value *llvm::emitCalloc(Value *Num, Value *Size, IRBuilderBase &B,
   return CI;
 }
 
+Value *llvm::emitHotColdSizeReturningNew(Value *Num, IRBuilderBase &B,
+                                         const TargetLibraryInfo *TLI,
+                                         LibFunc SizeFeedbackNewFunc,
+                                         uint8_t HotCold) {
+  Module *M = B.GetInsertBlock()->getModule();
+  if (!isLibFuncEmittable(M, TLI, SizeFeedbackNewFunc))
+    return nullptr;
+
+  StringRef Name = TLI->getName(SizeFeedbackNewFunc);
+
+  // __sized_ptr_t struct return type { void*, size_t }
+  StructType *SizedPtrT =
+      StructType::get(M->getContext(), {B.getPtrTy(), Num->getType()});
+  FunctionCallee Func =
+      M->getOrInsertFunction(Name, SizedPtrT, Num->getType(), B.getInt8Ty());
+  inferNonMandatoryLibFuncAttrs(M, Name, *TLI);
+  CallInst *CI = B.CreateCall(Func, {Num, B.getInt8(HotCold)}, "sized_ptr");
+
+  if (const Function *F = dyn_cast<Function>(Func.getCallee()))
+    CI->setCallingConv(F->getCallingConv());
+
+  return CI;
+}
+
+Value *llvm::emitHotColdSizeReturningNewAligned(Value *Num, Value *Align,
+                                                IRBuilderBase &B,
+                                                const TargetLibraryInfo *TLI,
+                                                LibFunc SizeFeedbackNewFunc,
+                                                uint8_t HotCold) {
+  Module *M = B.GetInsertBlock()->getModule();
+  if (!isLibFuncEmittable(M, TLI, SizeFeedbackNewFunc))
+    return nullptr;
+
+  StringRef Name = TLI->getName(SizeFeedbackNewFunc);
+
+  // __sized_ptr_t struct return type { void*, size_t }
+  StructType *SizedPtrT =
+      StructType::get(M->getContext(), {B.getPtrTy(), Num->getType()});
+  FunctionCallee Func = M->getOrInsertFunction(Name, SizedPtrT, Num->getType(),
+                                               Align->getType(), B.getInt8Ty());
+  inferNonMandatoryLibFuncAttrs(M, Name, *TLI);
+  CallInst *CI =
+      B.CreateCall(Func, {Num, Align, B.getInt8(HotCold)}, "sized_ptr");
+
+  if (const Function *F = dyn_cast<Function>(Func.getCallee()))
+    CI->setCallingConv(F->getCallingConv());
+
+  return CI;
+}
+
 Value *llvm::emitHotColdNew(Value *Num, IRBuilderBase &B,
                             const TargetLibraryInfo *TLI, LibFunc NewFunc,
                             uint8_t HotCold) {
diff --git a/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp b/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
index 673cc1a6c08f76f..be4d4590cfd839d 100644
--- a/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
@@ -18,6 +18,7 @@
 #include "llvm/Analysis/ConstantFolding.h"
 #include "llvm/Analysis/Loads.h"
 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
+#include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/IR/AttributeMask.h"
 #include "llvm/IR/DataLayout.h"
@@ -1745,7 +1746,7 @@ Value *LibCallSimplifier::optimizeNew(CallInst *CI, IRBuilderBase &B,
   // if cold or hot, and leave as-is for default handling if "notcold" aka warm.
   // Note that in cases where we decide it is "notcold", it might be slightly
   // better to replace the hinted call with a non hinted call, to avoid the
-  // extra paramter and the if condition check of the hint value in the
+  // extra parameter and the if condition check of the hint value in the
   // allocator. This can be considered in the future.
   switch (Func) {
   case LibFunc_Znwm12__hot_cold_t:
@@ -1844,6 +1845,30 @@ Value *LibCallSimplifier::optimizeNew(CallInst *CI, IRBuilderBase &B,
           TLI, LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t,
           HotCold);
     break;
+  case LibFunc_size_returning_new:
+    if (HotCold != NotColdNewHintValue)
+      return emitHotColdSizeReturningNew(CI->getArgOperand(0), B, TLI,
+                                         LibFunc_size_returning_new_hot_cold,
+                                         HotCold);
+    break;
+  case LibFunc_size_returning_new_hot_cold:
+    if (OptimizeExistingHotColdNew)
+      return emitHotColdSizeReturningNew(CI->getArgOperand(0), B, TLI,
+                                         LibFunc_size_returning_new_hot_cold,
+                                         HotCold);
+    break;
+  case LibFunc_size_returning_new_aligned:
+    if (HotCold != NotColdNewHintValue)
+      return emitHotColdSizeReturningNewAligned(
+          CI->getArgOperand(0), CI->getArgOperand(1), B, TLI,
+          LibFunc_size_returning_new_aligned_hot_cold, HotCold);
+    break;
+  case LibFunc_size_returning_new_aligned_hot_cold:
+    if (OptimizeExistingHotColdNew)
+      return emitHotColdSizeReturningNewAligned(
+          CI->getArgOperand(0), CI->getArgOperand(1), B, TLI,
+          LibFunc_size_returning_new_aligned_hot_cold, HotCold);
+    break;
   default:
     return nullptr;
   }
@@ -3759,6 +3784,7 @@ Value *LibCallSimplifier::optimizeStringMemoryLibCall(CallInst *CI,
   Module *M = CI->getModule();
   LibFunc Func;
   Function *Callee = CI->getCalledFunction();
+
   // Check for string/memory library functions.
   if (TLI->getLibFunc(*Callee, Func) && isLibFuncEmittable(M, TLI, Func)) {
     // Make sure we never change the calling convention.
@@ -3851,6 +3877,10 @@ Value *LibCallSimplifier::optimizeStringMemoryLibCall(CallInst *CI,
     case LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t:
     case LibFunc_ZnamSt11align_val_t12__hot_cold_t:
     case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t:
+    case LibFunc_size_returning_new:
+    case LibFunc_size_returning_new_hot_cold:
+    case LibFunc_size_returning_new_aligned:
+    case LibFunc_size_returning_new_aligned_hot_cold:
       return optimizeNew(CI, Builder, Func);
     default:
       break;
diff --git a/llvm/test/Transforms/InstCombine/simplify-libcalls-new.ll b/llvm/test/Transforms/InstCombine/simplify-libcalls-new.ll
index ecfafbc69797b15..83471f1d0f613fc 100644
--- a/llvm/test/Transforms/InstCombine/simplify-libcalls-new.ll
+++ b/llvm/test/Transforms/InstCombine/simplify-libcalls-new.ll
@@ -340,6 +340,94 @@ define void @array_new_align_nothrow_hot_cold() {
   ret void
 }
 
+;; Check that operator __size_returning_new(unsigned long) converted to
+;; __size_returning_new(unsigned long, __hot_cold_t) with a hot or cold attribute.
+; HOTCOLD-LABEL: @size_returning_test()
+define void @size_returning_test() {
+  ;; Attribute cold converted to __hot_cold_t cold value.
+  ; HOTCOLD: @__size_returning_new_hot_cold(i64 10, i8 [[COLD]])
+  %call = call {ptr, i64} @__size_returning_new(i64 10) #3
+  %p  = extractvalue {ptr, i64} %call, 0
+  call void @dummy(ptr %p)
+  ;; Attribute notcold has no effect.
+  ; HOTCOLD: @__size_returning_new(i64 10)
+  %call1 = call {ptr, i64} @__size_returning_new(i64 10) #4
+  %p1  = extractvalue {ptr, i64} %call1, 0
+  call void @dummy(ptr %p1)
+  ;; Attribute hot converted to __hot_cold_t hot value.
+  ; HOTCOLD: @__size_returning_new_hot_cold(i64 10, i8 [[HOT]])
+  %call2 = call {ptr, i64} @__size_returning_new(i64 10) #5
+  %p2  = extractvalue {ptr, i64} %call2, 0
+  call void @dummy(ptr %p2)
+  ret void
+}
+
+;; Check that operator __size_returning_new_aligned(unsigned long, std::align_val_t) converted to
+;; __size_returning_new_aligned(unsigned long, std::align_val_t, __hot_cold_t) with a hot or cold attribute.
+; HOTCOLD-LABEL: @size_returning_aligned_test()
+define void @size_returning_aligned_test() {
+  ;; Attribute cold converted to __hot_cold_t cold value.
+  ; HOTCOLD: @__size_returning_new_aligned_hot_cold(i64 10, i64 8, i8 [[COLD]])
+  %call = call {ptr, i64} @__size_returning_new_aligned(i64 10, i64 8) #3
+  %p  = extractvalue {ptr, i64} %call, 0
+  call void @dummy(ptr %p)
+  ;; Attribute notcold has no effect.
+  ; HOTCOLD: @__size_returning_new_aligned(i64 10, i64 8)
+  %call1 = call {ptr, i64} @__size_returning_new_aligned(i64 10, i64 8) #4
+  %p1  = extractvalue {ptr, i64} %call1, 0
+  call void @dummy(ptr %p1)
+  ;; Attribute hot converted to __hot_cold_t hot value.
+  ; HOTCOLD: @__size_returning_new_aligned_hot_cold(i64 10, i64 8, i8 [[HOT]])
+  %call2 = call {ptr, i64} @__size_returning_new_aligned(i64 10, i64 8) #5
+  %p2  = extractvalue {ptr, i64} %call2, 0
+  call void @dummy(ptr %p2)
+  ret void
+}
+
+;; Check that __size_returning_new_hot_cold(unsigned long, __hot_cold_t)
+;; optionally has its hint updated.
+; HOTCOLD-LABEL: @size_returning_update_test()
+define void @size_returning_update_test() {
+  ;; Attribute cold converted to __hot_cold_t cold value.
+  ; HOTCOLD: @__size_returning_new_hot_cold(i64 10, i8 [[PREVHINTCOLD]])
+  %call = call {ptr, i64} @__size_returning_new_hot_cold(i64 10, i8 7) #3
+  %p  = extractvalue {ptr, i64} %call, 0
+  call void @dummy(ptr %p)
+  ;; Attribute notcold converted to __hot_cold_t notcold value.
+  ; HOTCOLD: @__size_returning_new_hot_cold(i64 10, i8 [[PREVHINTNOTCOLD]])
+  %call1 = call {ptr, i64} @__size_returning_new_hot_cold(i64 10, i8 7) #4
+  %p1 = extractvalue {ptr, i64} %call1, 0
+  call void @dummy(ptr %p1)
+  ;; Attribute hot converted to __hot_cold_t hot value.
+  ; HOTCOLD: @__size_returning_new_hot_cold(i64 10, i8 [[PREVHINTHOT]])
+  %call2 = call {ptr, i64} @__size_returning_new_hot_cold(i64 10, i8 7) #5
+  %p2 = extractvalue {ptr, i64} %call2, 0
+  call void @dummy(ptr %p2)
+  ret void
+}
+
+;; Check that __size_returning_new_aligned_hot_cold(unsigned long, __hot_cold_t)
+;; optionally has its hint updated.
+; HOTCOLD-LABEL: @size_returning_aligned_update_test()
+define void @size_returning_aligned_update_test() {
+  ;; Attribute cold converted to __hot_cold_t cold value.
+  ; HOTCOLD: @__size_returning_new_aligned_hot_cold(i64 10, i64 8, i8 [[PREVHINTCOLD]])
+  %call = call {ptr, i64} @__size_returning_new_aligned_hot_cold(i64 10, i64 8, i8 7) #3
+  %p  = extractvalue {ptr, i64} %call, 0
+  call void @dummy(ptr %p)
+  ;; Attribute notcold converted to __hot_cold_t notcold value.
+  ; HOTCOLD: @__size_returning_new_aligned_hot_cold(i64 10, i64 8, i8 [[PREVHINTNOTCOLD]])
+  %call1 = call {ptr, i64} @__size_returning_new_aligned_hot_cold(i64 10, i64 8, i8 7) #4
+  %p1 = extractvalue {ptr, i64} %call1, 0
+  call void @dummy(ptr %p1)
+  ;; Attribute hot converted to __hot_cold_t hot value.
+  ; HOTCOLD: @__size_returning_new_aligned_hot_cold(i64 10, i64 8, i8 [[PREVHINTHOT]])
+  %call2 = call {ptr, i64} @__size_returning_new_aligned_hot_cold(i64 10, i64 8, i8 7) #5
+  %p2 = extractvalue {ptr, i64} %call2, 0
+  call void @dummy(ptr %p2)
+  ret void
+}
+
 ;; So that instcombine doesn't optimize out the call.
 declare void @dummy(ptr)
 
@@ -360,6 +448,18 @@ declare ptr @_ZnamSt11align_val_t12__hot_cold_t(i64, i64, i8)
 declare ptr @_ZnamRKSt9nothrow_t12__hot_cold_t(i64, ptr, i8)
 declare ptr @_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t(i64, i64, ptr, i8)
 
+
+declare {ptr, i64} @__size_returning_new(i64)
+declare {ptr, i64} @__size_returning_new_hot_cold(i64, i8)
+declare {ptr, i64} @__size_returning_new_aligned(i64, i64)
+declare {ptr, i64} @__size_returning_new_aligned_hot_cold(i64, i64, i8)
+
 attributes #0 = { builtin allocsize(0) "memprof"="cold" }
 attributes #1 = { builtin allocsize(0) "memprof"="notcold" }
 attributes #2 = { builtin allocsize(0) "memprof"="hot" }
+
+;; Use separate attributes for __size_returning_new variants since they are not
+;; treated as builtins.
+attributes #3 = { "memprof" = "cold" }
+attributes #4 = { "memprof" = "notcold" }
+attributes #5 = { "memprof" = "hot" }

From fcefe957ddfdc5a2fe9463757b597635e3436e01 Mon Sep 17 00:00:00 2001
From: Jeffrey Byrnes <jeffrey.byrnes@amd.com>
Date: Thu, 15 Aug 2024 08:07:05 -0700
Subject: [PATCH 034/441] [LegalizeTypes][AMDGPU]: Allow for scalarization of
 insert_subvector (#104236)

Legalization for when the inserted subvector is to be scalarized.

https://godbolt.org/z/vx3joWqoh
---
 llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h |  1 +
 .../SelectionDAG/LegalizeVectorTypes.cpp      | 16 +++++++++++++
 .../AMDGPU/scalarize-insert-subvector.ll      | 24 +++++++++++++++++++
 3 files changed, 41 insertions(+)
 create mode 100644 llvm/test/CodeGen/AMDGPU/scalarize-insert-subvector.ll

diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
index 6de1e3eca7feda8..27dd4ae241bd100 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
@@ -865,6 +865,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
   SDValue ScalarizeVecOp_UnaryOp(SDNode *N);
   SDValue ScalarizeVecOp_UnaryOp_StrictFP(SDNode *N);
   SDValue ScalarizeVecOp_CONCAT_VECTORS(SDNode *N);
+  SDValue ScalarizeVecOp_INSERT_SUBVECTOR(SDNode *N, unsigned OpNo);
   SDValue ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N);
   SDValue ScalarizeVecOp_VSELECT(SDNode *N);
   SDValue ScalarizeVecOp_VSETCC(SDNode *N);
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index ddb7c8c54bbfe46..8c0c2616250c60f 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -765,6 +765,9 @@ bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) {
   case ISD::CONCAT_VECTORS:
     Res = ScalarizeVecOp_CONCAT_VECTORS(N);
     break;
+  case ISD::INSERT_SUBVECTOR:
+    Res = ScalarizeVecOp_INSERT_SUBVECTOR(N, OpNo);
+    break;
   case ISD::EXTRACT_VECTOR_ELT:
     Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N);
     break;
@@ -882,6 +885,19 @@ SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) {
   return DAG.getBuildVector(N->getValueType(0), SDLoc(N), Ops);
 }
 
+/// The inserted subvector is to be scalarized - use insert vector element
+/// instead.
+SDValue DAGTypeLegalizer::ScalarizeVecOp_INSERT_SUBVECTOR(SDNode *N,
+                                                          unsigned OpNo) {
+  // We should not be attempting to scalarize the containing vector
+  assert(OpNo == 1);
+  SDValue Elt = GetScalarizedVector(N->getOperand(1));
+  SDValue ContainingVec = N->getOperand(0);
+  return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(N),
+                     ContainingVec.getValueType(), ContainingVec, Elt,
+                     N->getOperand(2));
+}
+
 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>,
 /// so just return the element, ignoring the index.
 SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
diff --git a/llvm/test/CodeGen/AMDGPU/scalarize-insert-subvector.ll b/llvm/test/CodeGen/AMDGPU/scalarize-insert-subvector.ll
new file mode 100644
index 000000000000000..f6f367cf85f7293
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/scalarize-insert-subvector.ll
@@ -0,0 +1,24 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 < %s | FileCheck -check-prefix=GCN %s
+
+define void @scalarize_insert_subvector(ptr addrspace(3) %inptr, ptr addrspace(3) %inptr1, ptr addrspace(3) %outptr) {
+; GCN-LABEL: scalarize_insert_subvector:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    ds_read_b64 v[4:5], v0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    ds_read_b32 v5, v1 offset:4
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    ds_write_b64 v2, v[4:5]
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %load0 = load <2 x i32>, ptr addrspace(3) %inptr, align 8
+  %load1= load <2 x i32>, ptr addrspace(3) %inptr1, align 8
+  %shuffle0 = shufflevector <2 x i32> %load1, <2 x i32> poison, <1 x i32> <i32 1>
+  %bitcast0 = bitcast <1 x i32> %shuffle0 to <2 x half>
+  %bitcast1 = bitcast <2 x i32> %load0 to <4 x half>
+  %shuffle1 = shufflevector <2 x half> %bitcast0, <2 x half> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+  %shuffle2 = shufflevector <4 x half> %bitcast1, <4 x half> %shuffle1, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+  store <4 x half> %shuffle2, ptr addrspace(3) %outptr
+  ret void
+}

From c7df775440717ec5a3f47b6d485617d802cbd036 Mon Sep 17 00:00:00 2001
From: Mital Ashok <mital@mitalashok.co.uk>
Date: Thu, 15 Aug 2024 16:12:11 +0100
Subject: [PATCH 035/441] [Clang] Check explicit object parameter for defaulted
 operators properly (#100419)

Previously, the type of explicit object parameters was not considered
for relational operators. This was defined by CWG2586,
<https://cplusplus.github.io/CWG/issues/2586.html>. This fix also means
CWG2547 <https://cplusplus.github.io/CWG/issues/2547.html> is now fully
implemented. Fixes #100329, fixes #104413.

Now start rejecting invalid rvalue reference parameters, which weren't
checked for, and start accepting non-reference explicit object
parameters (like `bool operator==(this C, C) = default;`) which were
previously rejected for the object param not being a reference.

Also start rejecting non-reference explicit object parameters for
defaulted copy/move assign operators (`A& operator=(this A, const A&) =
default;` is invalid but was previously accepted). Fixes #104414.
---
 clang/docs/ReleaseNotes.rst                   |  3 +
 .../clang/Basic/DiagnosticSemaKinds.td        |  2 +-
 clang/lib/Sema/SemaDeclCXX.cpp                | 51 ++++++++--------
 .../class.compare.default/p1.cpp              |  2 +
 clang/test/CXX/drs/cwg25xx.cpp                | 59 +++++++++++++++++++
 clang/test/SemaCXX/cxx2b-deducing-this.cpp    | 43 +++++++++++++-
 clang/www/cxx_dr_status.html                  |  4 +-
 7 files changed, 136 insertions(+), 28 deletions(-)

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index f5696d6ce15dc7c..b6b7dd5705637a3 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -254,6 +254,9 @@ Bug Fixes to C++ Support
   specialization of a conversion function template.
 - Correctly diagnose attempts to use a concept name in its own definition;
   A concept name is introduced to its scope sooner to match the C++ standard. (#GH55875)
+- Properly reject defaulted relational operators with invalid types for explicit object parameters,
+  e.g., ``bool operator==(this int, const Foo&)`` (#GH100329), and rvalue reference parameters.
+- Properly reject defaulted copy/move assignment operators that have a non-reference explicit object parameter.
 
 Bug Fixes to AST Handling
 ^^^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index da2f939067bfab8..461eeb19f65e4a3 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -9752,7 +9752,7 @@ def err_defaulted_special_member_quals : Error<
   "have 'const'%select{, 'constexpr'|}1 or 'volatile' qualifiers">;
 def err_defaulted_special_member_explicit_object_mismatch : Error<
   "the type of the explicit object parameter of an explicitly-defaulted "
-  "%select{copy|move}0 assignment operator should match the type of the class %1">;
+  "%select{copy|move}0 assignment operator should be reference to %1">;
 def err_defaulted_special_member_volatile_param : Error<
   "the parameter for an explicitly-defaulted %sub{select_special_member_kind}0 "
   "may not be volatile">;
diff --git a/clang/lib/Sema/SemaDeclCXX.cpp b/clang/lib/Sema/SemaDeclCXX.cpp
index c87edf41f35cee8..9ca91a2def39f5c 100644
--- a/clang/lib/Sema/SemaDeclCXX.cpp
+++ b/clang/lib/Sema/SemaDeclCXX.cpp
@@ -7644,9 +7644,13 @@ bool Sema::CheckExplicitlyDefaultedSpecialMember(CXXMethodDecl *MD,
     // parameter is of (possibly different) type “reference to C”,
     // in which case the type of F1 would differ from the type of F2
     // in that the type of F1 has an additional parameter;
-    if (!Context.hasSameType(
-            ThisType.getNonReferenceType().getUnqualifiedType(),
-            Context.getRecordType(RD))) {
+    QualType ExplicitObjectParameter = MD->isExplicitObjectMemberFunction()
+                                           ? MD->getParamDecl(0)->getType()
+                                           : QualType();
+    if (!ExplicitObjectParameter.isNull() &&
+        (!ExplicitObjectParameter->isReferenceType() ||
+         !Context.hasSameType(ExplicitObjectParameter.getNonReferenceType(),
+                              Context.getRecordType(RD)))) {
       if (DeleteOnTypeMismatch)
         ShouldDeleteForTypeMismatch = true;
       else {
@@ -8730,8 +8734,9 @@ bool Sema::CheckExplicitlyDefaultedComparison(Scope *S, FunctionDecl *FD,
     // If we're out-of-class, this is the class we're comparing.
     if (!RD)
       RD = MD->getParent();
-    QualType T = MD->getFunctionObjectParameterType();
-    if (!T.isConstQualified()) {
+    QualType T = MD->getFunctionObjectParameterReferenceType();
+    if (!T.getNonReferenceType().isConstQualified() &&
+        (MD->isImplicitObjectMemberFunction() || T->isLValueReferenceType())) {
       SourceLocation Loc, InsertLoc;
       if (MD->isExplicitObjectMemberFunction()) {
         Loc = MD->getParamDecl(0)->getBeginLoc();
@@ -8750,11 +8755,17 @@ bool Sema::CheckExplicitlyDefaultedComparison(Scope *S, FunctionDecl *FD,
       }
 
       // Add the 'const' to the type to recover.
-      const auto *FPT = MD->getType()->castAs<FunctionProtoType>();
-      FunctionProtoType::ExtProtoInfo EPI = FPT->getExtProtoInfo();
-      EPI.TypeQuals.addConst();
-      MD->setType(Context.getFunctionType(FPT->getReturnType(),
-                                          FPT->getParamTypes(), EPI));
+      if (MD->isExplicitObjectMemberFunction()) {
+        assert(T->isLValueReferenceType());
+        MD->getParamDecl(0)->setType(Context.getLValueReferenceType(
+            T.getNonReferenceType().withConst()));
+      } else {
+        const auto *FPT = MD->getType()->castAs<FunctionProtoType>();
+        FunctionProtoType::ExtProtoInfo EPI = FPT->getExtProtoInfo();
+        EPI.TypeQuals.addConst();
+        MD->setType(Context.getFunctionType(FPT->getReturnType(),
+                                            FPT->getParamTypes(), EPI));
+      }
     }
 
     if (MD->isVolatile()) {
@@ -8781,18 +8792,15 @@ bool Sema::CheckExplicitlyDefaultedComparison(Scope *S, FunctionDecl *FD,
 
   const ParmVarDecl *KnownParm = nullptr;
   for (const ParmVarDecl *Param : FD->parameters()) {
-    if (Param->isExplicitObjectParameter())
-      continue;
     QualType ParmTy = Param->getType();
-
     if (!KnownParm) {
       auto CTy = ParmTy;
       // Is it `T const &`?
-      bool Ok = !IsMethod;
+      bool Ok = !IsMethod || FD->hasCXXExplicitFunctionObjectParameter();
       QualType ExpectedTy;
       if (RD)
         ExpectedTy = Context.getRecordType(RD);
-      if (auto *Ref = CTy->getAs<ReferenceType>()) {
+      if (auto *Ref = CTy->getAs<LValueReferenceType>()) {
         CTy = Ref->getPointeeType();
         if (RD)
           ExpectedTy.addConst();
@@ -8800,14 +8808,11 @@ bool Sema::CheckExplicitlyDefaultedComparison(Scope *S, FunctionDecl *FD,
       }
 
       // Is T a class?
-      if (!Ok) {
-      } else if (RD) {
-        if (!RD->isDependentType() && !Context.hasSameType(CTy, ExpectedTy))
-          Ok = false;
-      } else if (auto *CRD = CTy->getAsRecordDecl()) {
-        RD = cast<CXXRecordDecl>(CRD);
+      if (RD) {
+        Ok &= RD->isDependentType() || Context.hasSameType(CTy, ExpectedTy);
       } else {
-        Ok = false;
+        RD = CTy->getAsCXXRecordDecl();
+        Ok &= RD != nullptr;
       }
 
       if (Ok) {
@@ -8847,7 +8852,7 @@ bool Sema::CheckExplicitlyDefaultedComparison(Scope *S, FunctionDecl *FD,
     assert(FD->getFriendObjectKind() && "expected a friend declaration");
   } else {
     // Out of class, require the defaulted comparison to be a friend (of a
-    // complete type).
+    // complete type, per CWG2547).
     if (RequireCompleteType(FD->getLocation(), Context.getRecordType(RD),
                             diag::err_defaulted_comparison_not_friend, int(DCK),
                             int(1)))
diff --git a/clang/test/CXX/class/class.compare/class.compare.default/p1.cpp b/clang/test/CXX/class/class.compare/class.compare.default/p1.cpp
index ddf82f432c2eab0..a195e0548152d6c 100644
--- a/clang/test/CXX/class/class.compare/class.compare.default/p1.cpp
+++ b/clang/test/CXX/class/class.compare/class.compare.default/p1.cpp
@@ -16,6 +16,8 @@ struct A {
   bool operator<(const A&) const;
   bool operator<=(const A&) const = default;
   bool operator==(const A&) const && = default; // expected-error {{ref-qualifier '&&' is not allowed on a defaulted comparison operator}}
+  bool operator<=(const A&&) const = default; // expected-error {{invalid parameter type for defaulted relational comparison operator; found 'const A &&', expected 'const A &'}}
+  bool operator<=(const int&) const = default; // expected-error {{invalid parameter type for defaulted relational comparison operator; found 'const int &', expected 'const A &'}}
   bool operator>=(const A&) const volatile = default; // expected-error {{defaulted comparison function must not be volatile}}
   bool operator<=>(const A&) = default; // expected-error {{defaulted member three-way comparison operator must be const-qualified}}
   bool operator>=(const B&) const = default; // expected-error-re {{invalid parameter type for defaulted relational comparison operator; found 'const B &', expected 'const A &'{{$}}}}
diff --git a/clang/test/CXX/drs/cwg25xx.cpp b/clang/test/CXX/drs/cwg25xx.cpp
index 0d9f5eac23866a2..1924008f15ba580 100644
--- a/clang/test/CXX/drs/cwg25xx.cpp
+++ b/clang/test/CXX/drs/cwg25xx.cpp
@@ -92,6 +92,30 @@ using ::cwg2521::operator""_div;
 #endif
 } // namespace cwg2521
 
+namespace cwg2547 { // cwg2547: 20
+#if __cplusplus >= 202302L
+struct S;
+// since-cxx23-note@-1 {{forward declaration of 'cwg2547::S'}}
+// since-cxx23-note@-2 {{forward declaration of 'cwg2547::S'}}
+// since-cxx23-note@-3 {{forward declaration of 'cwg2547::S'}}
+bool operator==(S, S) = default;  // error: S is not complete
+// since-cxx23-error@-1 {{variable has incomplete type 'S'}}
+// since-cxx23-error@-2 {{variable has incomplete type 'S'}}
+// since-cxx23-error@-3 {{equality comparison operator is not a friend of incomplete class 'cwg2547::S'}}
+struct S {
+  friend bool operator==(S, const S&) = default; // error: parameters of different types
+  // since-cxx23-error@-1 {{parameters for defaulted equality comparison operator must have the same type (found 'S' vs 'const S &')}}
+};
+enum E { };
+bool operator==(E, E) = default;  // error: not a member or friend of a class
+// since-cxx23-error@-1 {{invalid parameter type for non-member defaulted equality comparison operator; found 'E', expected class or reference to a constant class}}
+
+struct S2 {
+  bool operator==(this int, S2) = default;
+  // since-cxx23-error@-1 {{invalid parameter type for defaulted equality comparison operator; found 'int', expected 'const cwg2547::S2 &'}}
+};
+#endif
+} // namespace cwg2547
 
 #if __cplusplus >= 202302L
 namespace cwg2553 { // cwg2553: 18 review 2023-07-14
@@ -253,6 +277,41 @@ static_assert(__is_layout_compatible(U, V), "");
 #endif
 } // namespace cwg2583
 
+namespace cwg2586 { // cwg2586: 20
+#if __cplusplus >= 202302L
+struct X {
+  X& operator=(this X&, const X&) = default;
+  X& operator=(this X&, X&) = default;
+  X& operator=(this X&&, X&&) = default;
+  // FIXME: The notes could be clearer on *how* the type differs
+  // e.g., "if an explicit object parameter is used it must be of type reference to 'X'"
+  X& operator=(this int, const X&) = default;
+  // since-cxx23-warning@-1 {{explicitly defaulted copy assignment operator is implicitly deleted}}
+  // since-cxx23-note@-2 {{function is implicitly deleted because its declared type does not match the type of an implicit copy assignment operator}}
+  X& operator=(this X, const X&) = default;
+  // since-cxx23-warning@-1 {{explicitly defaulted copy assignment operator is implicitly deleted}}
+  // since-cxx23-note@-2 {{function is implicitly deleted because its declared type does not match the type of an implicit copy assignment operator}}
+};
+struct Y {
+  void operator=(this int, const Y&); // This is copy constructor, suppresses implicit declaration
+};
+static_assert([]<typename T = Y>{
+  return !requires(T t, const T& ct) { t = ct; };
+}());
+
+struct Z {
+  bool operator==(this const Z&, const Z&) = default;
+  bool operator==(this Z, Z) = default;
+  bool operator==(this Z, const Z&) = default;
+  // since-cxx23-error@-1 {{parameters for defaulted equality comparison operator must have the same type (found 'Z' vs 'const Z &')}}
+  bool operator==(this const Z&, Z) = default;
+  // since-cxx23-error@-1 {{parameters for defaulted equality comparison operator must have the same type (found 'const Z &' vs 'Z')}}
+  bool operator==(this int, Z) = default;
+  // since-cxx23-error@-1 {{invalid parameter type for defaulted equality comparison operator; found 'int', expected 'const cwg2586::Z &'}}
+};
+#endif
+} // namespace cwg2586
+
 namespace cwg2598 { // cwg2598: 18
 #if __cplusplus >= 201103L
 struct NonLiteral {
diff --git a/clang/test/SemaCXX/cxx2b-deducing-this.cpp b/clang/test/SemaCXX/cxx2b-deducing-this.cpp
index 45fee6514c12bc6..23fb383fb73cbbe 100644
--- a/clang/test/SemaCXX/cxx2b-deducing-this.cpp
+++ b/clang/test/SemaCXX/cxx2b-deducing-this.cpp
@@ -729,10 +729,10 @@ struct S2 {
 };
 
 S2& S2::operator=(this int&& self, const S2&) = default;
-// expected-error@-1 {{the type of the explicit object parameter of an explicitly-defaulted copy assignment operator should match the type of the class 'S2'}}
+// expected-error@-1 {{the type of the explicit object parameter of an explicitly-defaulted copy assignment operator should be reference to 'S2'}}
 
 S2& S2::operator=(this int&& self, S2&&) = default;
-// expected-error@-1 {{the type of the explicit object parameter of an explicitly-defaulted move assignment operator should match the type of the class 'S2'}}
+// expected-error@-1 {{the type of the explicit object parameter of an explicitly-defaulted move assignment operator should be reference to 'S2'}}
 
 struct Move {
     Move& operator=(this int&, Move&&) = default;
@@ -972,3 +972,42 @@ struct R {
 	f(r_value_ref); // expected-error {{no matching member function for call to 'f'}}
   }
 };
+
+namespace GH100329 {
+struct A {
+    bool operator == (this const int&, const A&);
+};
+bool A::operator == (this const int&, const A&) = default;
+// expected-error@-1 {{invalid parameter type for defaulted equality comparison operator; found 'const int &', expected 'const GH100329::A &'}}
+} // namespace GH100329
+
+namespace defaulted_assign {
+struct A {
+  A& operator=(this A, const A&) = default;
+  // expected-warning@-1 {{explicitly defaulted copy assignment operator is implicitly deleted}}
+  // expected-note@-2 {{function is implicitly deleted because its declared type does not match the type of an implicit copy assignment operator}}
+  A& operator=(this int, const A&) = default;
+  // expected-warning@-1 {{explicitly defaulted copy assignment operator is implicitly deleted}}
+  // expected-note@-2 {{function is implicitly deleted because its declared type does not match the type of an implicit copy assignment operator}}
+};
+} // namespace defaulted_assign
+
+namespace defaulted_compare {
+struct A {
+  bool operator==(this A&, const A&) = default;
+  // expected-error@-1 {{defaulted member equality comparison operator must be const-qualified}}
+  bool operator==(this const A, const A&) = default;
+  // expected-error@-1 {{invalid parameter type for defaulted equality comparison operator; found 'const A', expected 'const defaulted_compare::A &'}}
+  bool operator==(this A, A) = default;
+};
+struct B {
+  int a;
+  bool operator==(this B, B) = default;
+};
+static_assert(B{0} == B{0});
+static_assert(B{0} != B{1});
+template<B b>
+struct X;
+static_assert(__is_same(X<B{0}>, X<B{0}>));
+static_assert(!__is_same(X<B{0}>, X<B{1}>));
+} // namespace defaulted_compare
diff --git a/clang/www/cxx_dr_status.html b/clang/www/cxx_dr_status.html
index 6c283b68aa9656f..9b0de55483d2755 100755
--- a/clang/www/cxx_dr_status.html
+++ b/clang/www/cxx_dr_status.html
@@ -15097,7 +15097,7 @@ <h2 id="cxxdr">C++ defect report implementation status</h2>
     <td><a href="https://cplusplus.github.io/CWG/issues/2547.html">2547</a></td>
     <td>DRWP</td>
     <td>Defaulted comparison operator function for non-classes</td>
-    <td class="unknown" align="center">Unknown</td>
+    <td class="unreleased" align="center">Clang 20</td>
   </tr>
   <tr id="2548">
     <td><a href="https://cplusplus.github.io/CWG/issues/2548.html">2548</a></td>
@@ -15331,7 +15331,7 @@ <h2 id="cxxdr">C++ defect report implementation status</h2>
     <td><a href="https://cplusplus.github.io/CWG/issues/2586.html">2586</a></td>
     <td>CD6</td>
     <td>Explicit object parameter for assignment and comparison</td>
-    <td class="unknown" align="center">Unknown</td>
+    <td class="unreleased" align="center">Clang 20</td>
   </tr>
   <tr class="open" id="2587">
     <td><a href="https://cplusplus.github.io/CWG/issues/2587.html">2587</a></td>

From 2d52eb6a434fe47e67086f5ec1c3789bf6e7a604 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev@outlook.com>
Date: Wed, 14 Aug 2024 12:17:19 -0700
Subject: [PATCH 036/441] [SLP][NFC]Remove unused using declarations, reduce
 mem usage in containers, NFC

---
 .../Transforms/Vectorize/SLPVectorizer.cpp    | 24 +++++++++----------
 1 file changed, 11 insertions(+), 13 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 87b4ed599b5bb15..15cff2f10030109 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -1244,12 +1244,10 @@ class BoUpSLP {
   };
 
   using ValueList = SmallVector<Value *, 8>;
-  using InstrList = SmallVector<Instruction *, 16>;
   using ValueSet = SmallPtrSet<Value *, 16>;
-  using StoreList = SmallVector<StoreInst *, 8>;
   using ExtraValueToDebugLocsMap =
       MapVector<Value *, SmallVector<Instruction *, 2>>;
-  using OrdersType = SmallVector<unsigned, 4>;
+  using OrdersType = SmallVector<unsigned, 0>;
 
   BoUpSLP(Function *Func, ScalarEvolution *Se, TargetTransformInfo *Tti,
           TargetLibraryInfo *TLi, AAResults *Aa, LoopInfo *Li,
@@ -1471,7 +1469,7 @@ class BoUpSLP {
   /// \param TryRecursiveCheck used to check if long masked gather can be
   /// represented as a serie of loads/insert subvector, if profitable.
   LoadsState canVectorizeLoads(ArrayRef<Value *> VL, const Value *VL0,
-                               SmallVectorImpl<unsigned> &Order,
+                               OrdersType &Order,
                                SmallVectorImpl<Value *> &PointerOps,
                                bool TryRecursiveCheck = true) const;
 
@@ -2840,7 +2838,7 @@ class BoUpSLP {
   /// \param ResizeAllowed indicates whether it is allowed to handle subvector
   /// extract order.
   bool canReuseExtract(ArrayRef<Value *> VL, Value *OpValue,
-                       SmallVectorImpl<unsigned> &CurrentOrder,
+                       OrdersType &CurrentOrder,
                        bool ResizeAllowed = false) const;
 
   /// Vectorize a single entry in the tree.
@@ -3084,10 +3082,10 @@ class BoUpSLP {
     CombinedOpcode CombinedOp = NotCombinedOp;
 
     /// Does this sequence require some shuffling?
-    SmallVector<int, 4> ReuseShuffleIndices;
+    SmallVector<int, 0> ReuseShuffleIndices;
 
     /// Does this entry require reordering?
-    SmallVector<unsigned, 4> ReorderIndices;
+    OrdersType ReorderIndices;
 
     /// Points back to the VectorizableTree.
     ///
@@ -4300,12 +4298,12 @@ static void reorderReuses(SmallVectorImpl<int> &Reuses, ArrayRef<int> Mask) {
 /// the original order of the scalars. Procedure transforms the provided order
 /// in accordance with the given \p Mask. If the resulting \p Order is just an
 /// identity order, \p Order is cleared.
-static void reorderOrder(SmallVectorImpl<unsigned> &Order, ArrayRef<int> Mask,
+static void reorderOrder(BoUpSLP::OrdersType &Order, ArrayRef<int> Mask,
                          bool BottomOrder = false) {
   assert(!Mask.empty() && "Expected non-empty mask.");
   unsigned Sz = Mask.size();
   if (BottomOrder) {
-    SmallVector<unsigned> PrevOrder;
+    BoUpSLP::OrdersType PrevOrder;
     if (Order.empty()) {
       PrevOrder.resize(Sz);
       std::iota(PrevOrder.begin(), PrevOrder.end(), 0);
@@ -4695,7 +4693,7 @@ getShuffleCost(const TargetTransformInfo &TTI, TTI::ShuffleKind Kind,
 }
 
 BoUpSLP::LoadsState BoUpSLP::canVectorizeLoads(
-    ArrayRef<Value *> VL, const Value *VL0, SmallVectorImpl<unsigned> &Order,
+    ArrayRef<Value *> VL, const Value *VL0, OrdersType &Order,
     SmallVectorImpl<Value *> &PointerOps, bool TryRecursiveCheck) const {
   // Check that a vectorized load would load the same memory as a scalar
   // load. For example, we don't want to vectorize loads that are smaller
@@ -4823,7 +4821,7 @@ BoUpSLP::LoadsState BoUpSLP::canVectorizeLoads(
         for (unsigned Cnt = 0, End = VL.size(); Cnt + VF <= End;
              Cnt += VF, ++VectorizedCnt) {
           ArrayRef<Value *> Slice = VL.slice(Cnt, VF);
-          SmallVector<unsigned> Order;
+          OrdersType Order;
           SmallVector<Value *> PointerOps;
           LoadsState LS =
               canVectorizeLoads(Slice, Slice.front(), Order, PointerOps,
@@ -5397,7 +5395,7 @@ void BoUpSLP::reorderNodeWithReuses(TreeEntry &TE, ArrayRef<int> Mask) const {
   TE.ReorderIndices.clear();
   // Try to improve gathered nodes with clustered reuses, if possible.
   ArrayRef<int> Slice = ArrayRef(NewMask).slice(0, Sz);
-  SmallVector<unsigned> NewOrder(Slice);
+  OrdersType NewOrder(Slice);
   inversePermutation(NewOrder, NewMask);
   reorderScalars(TE.Scalars, NewMask);
   // Fill the reuses mask with the identity submasks.
@@ -7717,7 +7715,7 @@ unsigned BoUpSLP::canMapToVector(Type *T) const {
 }
 
 bool BoUpSLP::canReuseExtract(ArrayRef<Value *> VL, Value *OpValue,
-                              SmallVectorImpl<unsigned> &CurrentOrder,
+                              OrdersType &CurrentOrder,
                               bool ResizeAllowed) const {
   const auto *It = find_if(VL, IsaPred<ExtractElementInst, ExtractValueInst>);
   assert(It != VL.end() && "Expected at least one extract instruction.");

From db8ef6188cbbe2125e6d60bdef77a535105772df Mon Sep 17 00:00:00 2001
From: Rahul Joshi <rjoshi@nvidia.com>
Date: Thu, 15 Aug 2024 08:15:30 -0700
Subject: [PATCH 037/441] [NFC] Fix code line exceeding 80 columns (#104428)

---
 llvm/tools/llc/llc.cpp | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/llvm/tools/llc/llc.cpp b/llvm/tools/llc/llc.cpp
index d3f7c2b7f819b68..80c84a977c26c64 100644
--- a/llvm/tools/llc/llc.cpp
+++ b/llvm/tools/llc/llc.cpp
@@ -130,8 +130,9 @@ static cl::opt<std::string> SplitDwarfFile(
 static cl::opt<bool> NoVerify("disable-verify", cl::Hidden,
                               cl::desc("Do not verify input module"));
 
-static cl::opt<bool> DisableSimplifyLibCalls("disable-simplify-libcalls",
-                                             cl::desc("Disable simplify-libcalls"));
+static cl::opt<bool>
+    DisableSimplifyLibCalls("disable-simplify-libcalls",
+                            cl::desc("Disable simplify-libcalls"));
 
 static cl::opt<bool> ShowMCEncoding("show-mc-encoding", cl::Hidden,
                                     cl::desc("Show encoding in .s output"));

From 11c2da8fb7dc4d5dede094fa61077827004a3997 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke@igalia.com>
Date: Thu, 15 Aug 2024 23:18:03 +0800
Subject: [PATCH 038/441] [RISCV] Narrow indices to e16 for LMUL > 1 when
 lowering vector_reverse (#104427)

The vector_shuffle lowering already does this to reduce register
pressure, so also do it here.
---
 llvm/lib/Target/RISCV/RISCVISelLowering.cpp   | 11 ++-
 .../RISCV/rvv/named-vector-shuffle-reverse.ll | 84 +++++++++++--------
 2 files changed, 59 insertions(+), 36 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 911fa45d7173e88..e16dd132ed71bb0 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -10321,8 +10321,6 @@ SDValue RISCVTargetLowering::lowerVECTOR_REVERSE(SDValue Op,
 
   // If this is SEW=8 and VLMAX is potentially more than 256, we need
   // to use vrgatherei16.vv.
-  // TODO: It's also possible to use vrgatherei16.vv for other types to
-  // decrease register width for the index calculation.
   if (MaxVLMAX > 256 && EltSize == 8) {
     // If this is LMUL=8, we have to split before can use vrgatherei16.vv.
     // Reverse each half, then reassemble them in reverse order.
@@ -10348,6 +10346,15 @@ SDValue RISCVTargetLowering::lowerVECTOR_REVERSE(SDValue Op,
     GatherOpc = RISCVISD::VRGATHEREI16_VV_VL;
   }
 
+  // At LMUL > 1, do the index computation in 16 bits to reduce register
+  // pressure.
+  if (IntVT.getScalarType().bitsGT(MVT::i16) &&
+      IntVT.bitsGT(getLMUL1VT(IntVT))) {
+    assert(isUInt<16>(MaxVLMAX - 1)); // Largest VLMAX is 65536 @ zvl65536b
+    GatherOpc = RISCVISD::VRGATHEREI16_VV_VL;
+    IntVT = IntVT.changeVectorElementType(MVT::i16);
+  }
+
   MVT XLenVT = Subtarget.getXLenVT();
   auto [Mask, VL] = getDefaultScalableVLOps(VecVT, DL, DAG, Subtarget);
 
diff --git a/llvm/test/CodeGen/RISCV/rvv/named-vector-shuffle-reverse.ll b/llvm/test/CodeGen/RISCV/rvv/named-vector-shuffle-reverse.ll
index 96094eea631ba50..60a03f1d97e5380 100644
--- a/llvm/test/CodeGen/RISCV/rvv/named-vector-shuffle-reverse.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/named-vector-shuffle-reverse.ll
@@ -1285,10 +1285,11 @@ define <vscale x 4 x i32> @reverse_nxv4i32(<vscale x 4 x i32> %a) {
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    srli a0, a0, 1
 ; CHECK-NEXT:    addi a0, a0, -1
-; CHECK-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vid.v v10
 ; CHECK-NEXT:    vrsub.vx v12, v10, a0
-; CHECK-NEXT:    vrgather.vv v10, v8, v12
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vrgatherei16.vv v10, v8, v12
 ; CHECK-NEXT:    vmv.v.v v8, v10
 ; CHECK-NEXT:    ret
   %res = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> %a)
@@ -1300,10 +1301,11 @@ define <vscale x 8 x i32> @reverse_nxv8i32(<vscale x 8 x i32> %a) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    addi a0, a0, -1
-; CHECK-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vid.v v12
 ; CHECK-NEXT:    vrsub.vx v16, v12, a0
-; CHECK-NEXT:    vrgather.vv v12, v8, v16
+; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vrgatherei16.vv v12, v8, v16
 ; CHECK-NEXT:    vmv.v.v v8, v12
 ; CHECK-NEXT:    ret
   %res = call <vscale x 8 x i32> @llvm.vector.reverse.nxv8i32(<vscale x 8 x i32> %a)
@@ -1316,10 +1318,11 @@ define <vscale x 16 x i32> @reverse_nxv16i32(<vscale x 16 x i32> %a) {
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    slli a0, a0, 1
 ; CHECK-NEXT:    addi a0, a0, -1
-; CHECK-NEXT:    vsetvli a1, zero, e32, m8, ta, ma
+; CHECK-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; CHECK-NEXT:    vid.v v16
 ; CHECK-NEXT:    vrsub.vx v24, v16, a0
-; CHECK-NEXT:    vrgather.vv v16, v8, v24
+; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
+; CHECK-NEXT:    vrgatherei16.vv v16, v8, v24
 ; CHECK-NEXT:    vmv.v.v v8, v16
 ; CHECK-NEXT:    ret
   %res = call <vscale x 16 x i32> @llvm.vector.reverse.nxv16i32(<vscale x 16 x i32> %a)
@@ -1348,10 +1351,11 @@ define <vscale x 2 x i64> @reverse_nxv2i64(<vscale x 2 x i64> %a) {
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    srli a0, a0, 2
 ; CHECK-NEXT:    addi a0, a0, -1
-; CHECK-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
+; CHECK-NEXT:    vsetvli a1, zero, e16, mf2, ta, ma
 ; CHECK-NEXT:    vid.v v10
 ; CHECK-NEXT:    vrsub.vx v12, v10, a0
-; CHECK-NEXT:    vrgather.vv v10, v8, v12
+; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
+; CHECK-NEXT:    vrgatherei16.vv v10, v8, v12
 ; CHECK-NEXT:    vmv.v.v v8, v10
 ; CHECK-NEXT:    ret
   %res = call <vscale x 2 x i64> @llvm.vector.reverse.nxv2i64(<vscale x 2 x i64> %a)
@@ -1364,10 +1368,11 @@ define <vscale x 4 x i64> @reverse_nxv4i64(<vscale x 4 x i64> %a) {
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    srli a0, a0, 1
 ; CHECK-NEXT:    addi a0, a0, -1
-; CHECK-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
+; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vid.v v12
 ; CHECK-NEXT:    vrsub.vx v16, v12, a0
-; CHECK-NEXT:    vrgather.vv v12, v8, v16
+; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; CHECK-NEXT:    vrgatherei16.vv v12, v8, v16
 ; CHECK-NEXT:    vmv.v.v v8, v12
 ; CHECK-NEXT:    ret
   %res = call <vscale x 4 x i64> @llvm.vector.reverse.nxv4i64(<vscale x 4 x i64> %a)
@@ -1379,10 +1384,11 @@ define <vscale x 8 x i64> @reverse_nxv8i64(<vscale x 8 x i64> %a) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    addi a0, a0, -1
-; CHECK-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
+; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vid.v v16
 ; CHECK-NEXT:    vrsub.vx v24, v16, a0
-; CHECK-NEXT:    vrgather.vv v16, v8, v24
+; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
+; CHECK-NEXT:    vrgatherei16.vv v16, v8, v24
 ; CHECK-NEXT:    vmv.v.v v8, v16
 ; CHECK-NEXT:    ret
   %res = call <vscale x 8 x i64> @llvm.vector.reverse.nxv8i64(<vscale x 8 x i64> %a)
@@ -1526,10 +1532,11 @@ define <vscale x 4 x float> @reverse_nxv4f32(<vscale x 4 x float> %a) {
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    srli a0, a0, 1
 ; CHECK-NEXT:    addi a0, a0, -1
-; CHECK-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vid.v v10
 ; CHECK-NEXT:    vrsub.vx v12, v10, a0
-; CHECK-NEXT:    vrgather.vv v10, v8, v12
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vrgatherei16.vv v10, v8, v12
 ; CHECK-NEXT:    vmv.v.v v8, v10
 ; CHECK-NEXT:    ret
   %res = call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> %a)
@@ -1541,10 +1548,11 @@ define <vscale x 8 x float> @reverse_nxv8f32(<vscale x 8 x float> %a) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    addi a0, a0, -1
-; CHECK-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vid.v v12
 ; CHECK-NEXT:    vrsub.vx v16, v12, a0
-; CHECK-NEXT:    vrgather.vv v12, v8, v16
+; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vrgatherei16.vv v12, v8, v16
 ; CHECK-NEXT:    vmv.v.v v8, v12
 ; CHECK-NEXT:    ret
   %res = call <vscale x 8 x float> @llvm.vector.reverse.nxv8f32(<vscale x 8 x float> %a)
@@ -1557,10 +1565,11 @@ define <vscale x 16 x float> @reverse_nxv16f32(<vscale x 16 x float> %a) {
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    slli a0, a0, 1
 ; CHECK-NEXT:    addi a0, a0, -1
-; CHECK-NEXT:    vsetvli a1, zero, e32, m8, ta, ma
+; CHECK-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
 ; CHECK-NEXT:    vid.v v16
 ; CHECK-NEXT:    vrsub.vx v24, v16, a0
-; CHECK-NEXT:    vrgather.vv v16, v8, v24
+; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
+; CHECK-NEXT:    vrgatherei16.vv v16, v8, v24
 ; CHECK-NEXT:    vmv.v.v v8, v16
 ; CHECK-NEXT:    ret
   %res = call <vscale x 16 x float> @llvm.vector.reverse.nxv16f32(<vscale x 16 x float> %a)
@@ -1589,10 +1598,11 @@ define <vscale x 2 x double> @reverse_nxv2f64(<vscale x 2 x double> %a) {
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    srli a0, a0, 2
 ; CHECK-NEXT:    addi a0, a0, -1
-; CHECK-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
+; CHECK-NEXT:    vsetvli a1, zero, e16, mf2, ta, ma
 ; CHECK-NEXT:    vid.v v10
 ; CHECK-NEXT:    vrsub.vx v12, v10, a0
-; CHECK-NEXT:    vrgather.vv v10, v8, v12
+; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
+; CHECK-NEXT:    vrgatherei16.vv v10, v8, v12
 ; CHECK-NEXT:    vmv.v.v v8, v10
 ; CHECK-NEXT:    ret
   %res = call <vscale x 2 x double> @llvm.vector.reverse.nxv2f64(<vscale x 2 x double> %a)
@@ -1605,10 +1615,11 @@ define <vscale x 4 x double> @reverse_nxv4f64(<vscale x 4 x double> %a) {
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    srli a0, a0, 1
 ; CHECK-NEXT:    addi a0, a0, -1
-; CHECK-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
+; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vid.v v12
 ; CHECK-NEXT:    vrsub.vx v16, v12, a0
-; CHECK-NEXT:    vrgather.vv v12, v8, v16
+; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; CHECK-NEXT:    vrgatherei16.vv v12, v8, v16
 ; CHECK-NEXT:    vmv.v.v v8, v12
 ; CHECK-NEXT:    ret
   %res = call <vscale x 4 x double> @llvm.vector.reverse.nxv4f64(<vscale x 4 x double> %a)
@@ -1620,10 +1631,11 @@ define <vscale x 8 x double> @reverse_nxv8f64(<vscale x 8 x double> %a) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    addi a0, a0, -1
-; CHECK-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
+; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vid.v v16
 ; CHECK-NEXT:    vrsub.vx v24, v16, a0
-; CHECK-NEXT:    vrgather.vv v16, v8, v24
+; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
+; CHECK-NEXT:    vrgatherei16.vv v16, v8, v24
 ; CHECK-NEXT:    vmv.v.v v8, v16
 ; CHECK-NEXT:    ret
   %res = call <vscale x 8 x double> @llvm.vector.reverse.nxv8f64(<vscale x 8 x double> %a)
@@ -1638,10 +1650,11 @@ define <vscale x 3 x i64> @reverse_nxv3i64(<vscale x 3 x i64> %a) {
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    srli a0, a0, 1
 ; CHECK-NEXT:    addi a0, a0, -1
-; CHECK-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
+; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vid.v v12
 ; CHECK-NEXT:    vrsub.vx v12, v12, a0
-; CHECK-NEXT:    vrgather.vv v16, v8, v12
+; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; CHECK-NEXT:    vrgatherei16.vv v16, v8, v12
 ; CHECK-NEXT:    vmv1r.v v8, v17
 ; CHECK-NEXT:    vmv1r.v v9, v18
 ; CHECK-NEXT:    vmv1r.v v10, v19
@@ -1655,10 +1668,11 @@ define <vscale x 6 x i64> @reverse_nxv6i64(<vscale x 6 x i64> %a) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    addi a0, a0, -1
-; CHECK-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
+; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
 ; CHECK-NEXT:    vid.v v16
 ; CHECK-NEXT:    vrsub.vx v16, v16, a0
-; CHECK-NEXT:    vrgather.vv v24, v8, v16
+; CHECK-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
+; CHECK-NEXT:    vrgatherei16.vv v24, v8, v16
 ; CHECK-NEXT:    vmv2r.v v8, v26
 ; CHECK-NEXT:    vmv2r.v v10, v28
 ; CHECK-NEXT:    vmv2r.v v12, v30
@@ -1684,12 +1698,13 @@ define <vscale x 12 x i64> @reverse_nxv12i64(<vscale x 12 x i64> %a) {
 ; RV32-NEXT:    andi sp, sp, -64
 ; RV32-NEXT:    csrr a0, vlenb
 ; RV32-NEXT:    addi a1, a0, -1
-; RV32-NEXT:    vsetvli a2, zero, e64, m8, ta, ma
+; RV32-NEXT:    vsetvli a2, zero, e16, m2, ta, ma
 ; RV32-NEXT:    vid.v v24
 ; RV32-NEXT:    vrsub.vx v24, v24, a1
-; RV32-NEXT:    vrgather.vv v0, v16, v24
+; RV32-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
+; RV32-NEXT:    vrgatherei16.vv v0, v16, v24
 ; RV32-NEXT:    vmv4r.v v16, v4
-; RV32-NEXT:    vrgather.vv v0, v8, v24
+; RV32-NEXT:    vrgatherei16.vv v0, v8, v24
 ; RV32-NEXT:    vmv4r.v v20, v0
 ; RV32-NEXT:    slli a0, a0, 3
 ; RV32-NEXT:    addi a1, sp, 64
@@ -1720,12 +1735,13 @@ define <vscale x 12 x i64> @reverse_nxv12i64(<vscale x 12 x i64> %a) {
 ; RV64-NEXT:    andi sp, sp, -64
 ; RV64-NEXT:    csrr a0, vlenb
 ; RV64-NEXT:    addi a1, a0, -1
-; RV64-NEXT:    vsetvli a2, zero, e64, m8, ta, ma
+; RV64-NEXT:    vsetvli a2, zero, e16, m2, ta, ma
 ; RV64-NEXT:    vid.v v24
 ; RV64-NEXT:    vrsub.vx v24, v24, a1
-; RV64-NEXT:    vrgather.vv v0, v16, v24
+; RV64-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
+; RV64-NEXT:    vrgatherei16.vv v0, v16, v24
 ; RV64-NEXT:    vmv4r.v v16, v4
-; RV64-NEXT:    vrgather.vv v0, v8, v24
+; RV64-NEXT:    vrgatherei16.vv v0, v8, v24
 ; RV64-NEXT:    vmv4r.v v20, v0
 ; RV64-NEXT:    slli a0, a0, 3
 ; RV64-NEXT:    addi a1, sp, 64

From 57abd4e4abb705a453134051743542de5fd396bc Mon Sep 17 00:00:00 2001
From: Daniel Grumberg <dgrumberg@apple.com>
Date: Thu, 15 Aug 2024 16:23:31 +0100
Subject: [PATCH 039/441] [clang][ExtractAPI] Emit environment component of
 target triple in SGF (#103273)

rdar://133533830
---
 .../Serialization/SymbolGraphSerializer.cpp   |  4 ++++
 .../test/ExtractAPI/platform-serialization.c  | 20 +++++++++++++++++++
 2 files changed, 24 insertions(+)
 create mode 100644 clang/test/ExtractAPI/platform-serialization.c

diff --git a/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp b/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp
index 84ed5467dd2fb91..1f8029cbd39ad25 100644
--- a/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp
+++ b/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp
@@ -104,6 +104,10 @@ Object serializePlatform(const Triple &T) {
   Object Platform;
   Platform["architecture"] = T.getArchName();
   Platform["vendor"] = T.getVendorName();
+
+  if (!T.getEnvironmentName().empty())
+    Platform["environment"] = T.getEnvironmentName();
+
   Platform["operatingSystem"] = serializeOperatingSystem(T);
   return Platform;
 }
diff --git a/clang/test/ExtractAPI/platform-serialization.c b/clang/test/ExtractAPI/platform-serialization.c
new file mode 100644
index 000000000000000..6d6a13f085cd9d9
--- /dev/null
+++ b/clang/test/ExtractAPI/platform-serialization.c
@@ -0,0 +1,20 @@
+// RUN: %clang_cc1 -extract-api --pretty-sgf -triple arm64-apple-ios17.1-macabi \
+// RUN:   -x c-header %s -verify -o - | FileCheck %s
+
+int a;
+
+// CHECK:      "platform": {
+// CHECK-NEXT:   "architecture": "arm64",
+// CHECK-NEXT:   "environment": "macabi",
+// CHECK-NEXT:   "operatingSystem": {
+// CHECK-NEXT:     "minimumVersion": {
+// CHECK-NEXT:       "major": 14,
+// CHECK-NEXT:       "minor": 0,
+// CHECK-NEXT:       "patch": 0
+// CHECK-NEXT:     },
+// CHECK-NEXT:     "name": "ios"
+// CHECK-NEXT:   },
+// CHECK-NEXT:   "vendor": "apple"
+// CHECK-NEXT: }
+
+// expected-no-diagnostics

From 598970904736f3535939f6a5525022219e4ae517 Mon Sep 17 00:00:00 2001
From: Hansang Bae <hansang.bae@intel.com>
Date: Thu, 15 Aug 2024 10:42:22 -0500
Subject: [PATCH 040/441] [OpenMP] Miscellaneous small code improvements
 (#95603)

Removes a few uninitialized variables, possible resource leaks, and
redundant code.
---
 openmp/runtime/src/kmp.h              |  2 ++
 openmp/runtime/src/kmp_affinity.cpp   |  3 +--
 openmp/runtime/src/kmp_affinity.h     |  8 +++++---
 openmp/runtime/src/kmp_barrier.cpp    |  3 ++-
 openmp/runtime/src/kmp_csupport.cpp   |  6 +++---
 openmp/runtime/src/kmp_runtime.cpp    | 12 ++++++------
 openmp/runtime/src/kmp_tasking.cpp    |  2 +-
 openmp/runtime/src/kmp_wait_release.h | 10 ++++++----
 openmp/runtime/src/ompt-general.cpp   |  6 ++++++
 9 files changed, 32 insertions(+), 20 deletions(-)

diff --git a/openmp/runtime/src/kmp.h b/openmp/runtime/src/kmp.h
index 916c1dc25700875..4f4fb3292d63eec 100644
--- a/openmp/runtime/src/kmp.h
+++ b/openmp/runtime/src/kmp.h
@@ -4731,6 +4731,8 @@ class kmp_safe_raii_file_t {
       : f(nullptr) {
     open(filename, mode, env_var);
   }
+  kmp_safe_raii_file_t(const kmp_safe_raii_file_t &other) = delete;
+  kmp_safe_raii_file_t &operator=(const kmp_safe_raii_file_t &other) = delete;
   ~kmp_safe_raii_file_t() { close(); }
 
   /// Open filename using mode. This is automatically closed in the destructor.
diff --git a/openmp/runtime/src/kmp_affinity.cpp b/openmp/runtime/src/kmp_affinity.cpp
index dd48b1ea5c13f9f..cf5cad04eb57d56 100644
--- a/openmp/runtime/src/kmp_affinity.cpp
+++ b/openmp/runtime/src/kmp_affinity.cpp
@@ -1970,7 +1970,6 @@ static bool __kmp_affinity_create_hwloc_map(kmp_i18n_id_t *const msg_id) {
             hw_thread.ids[index + 1] = sub_id;
             index--;
           }
-          prev = memory;
         }
         prev = obj;
       }
@@ -4989,7 +4988,7 @@ static void __kmp_aux_affinity_initialize(kmp_affinity_t &affinity) {
   int depth = __kmp_topology->get_depth();
 
   // Create the table of masks, indexed by thread Id.
-  unsigned numUnique;
+  unsigned numUnique = 0;
   int numAddrs = __kmp_topology->get_num_hw_threads();
   // If OMP_PLACES=cores:<attribute> specified, then attempt
   // to make OS Id mask table using those attributes
diff --git a/openmp/runtime/src/kmp_affinity.h b/openmp/runtime/src/kmp_affinity.h
index ed24b6faf2f7e8f..9ab2c0cc70d8c6c 100644
--- a/openmp/runtime/src/kmp_affinity.h
+++ b/openmp/runtime/src/kmp_affinity.h
@@ -29,6 +29,8 @@ class KMPHwlocAffinity : public KMPAffinity {
       mask = hwloc_bitmap_alloc();
       this->zero();
     }
+    Mask(const Mask &other) = delete;
+    Mask &operator=(const Mask &other) = delete;
     ~Mask() { hwloc_bitmap_free(mask); }
     void set(int i) override { hwloc_bitmap_set(mask, i); }
     bool is_set(int i) const override { return hwloc_bitmap_isset(mask, i); }
@@ -1271,7 +1273,7 @@ class hierarchy_info {
       leaf. It corresponds to the number of entries in numPerLevel if we exclude
       all but one trailing 1. */
   kmp_uint32 depth;
-  kmp_uint32 base_num_threads;
+  kmp_uint32 base_num_threads = 0;
   enum init_status { initialized = 0, not_initialized = 1, initializing = 2 };
   volatile kmp_int8 uninitialized; // 0=initialized, 1=not initialized,
   // 2=initialization in progress
@@ -1281,8 +1283,8 @@ class hierarchy_info {
       the parent of a node at level i has. For example, if we have a machine
       with 4 packages, 4 cores/package and 2 HT per core, then numPerLevel =
       {2, 4, 4, 1, 1}. All empty levels are set to 1. */
-  kmp_uint32 *numPerLevel;
-  kmp_uint32 *skipPerLevel;
+  kmp_uint32 *numPerLevel = nullptr;
+  kmp_uint32 *skipPerLevel = nullptr;
 
   void deriveLevels() {
     int hier_depth = __kmp_topology->get_depth();
diff --git a/openmp/runtime/src/kmp_barrier.cpp b/openmp/runtime/src/kmp_barrier.cpp
index 658cee594e48d52..d7ef57c608149e7 100644
--- a/openmp/runtime/src/kmp_barrier.cpp
+++ b/openmp/runtime/src/kmp_barrier.cpp
@@ -444,7 +444,8 @@ static void __kmp_dist_barrier_release(
       next_go = my_current_iter + distributedBarrier::MAX_ITERS;
       my_go_index = tid / b->threads_per_go;
       if (this_thr->th.th_used_in_team.load() == 3) {
-        KMP_COMPARE_AND_STORE_ACQ32(&(this_thr->th.th_used_in_team), 3, 1);
+        (void)KMP_COMPARE_AND_STORE_ACQ32(&(this_thr->th.th_used_in_team), 3,
+                                          1);
       }
       // Check if go flag is set
       if (b->go[my_go_index].go.load() != next_go) {
diff --git a/openmp/runtime/src/kmp_csupport.cpp b/openmp/runtime/src/kmp_csupport.cpp
index b33c16fa79a658f..fdbf9ff45e35495 100644
--- a/openmp/runtime/src/kmp_csupport.cpp
+++ b/openmp/runtime/src/kmp_csupport.cpp
@@ -1589,7 +1589,7 @@ void __kmpc_critical_with_hint(ident_t *loc, kmp_int32 global_tid,
   kmp_dyna_lockseq_t lockseq = __kmp_map_hint_to_lock(hint);
   if (*lk == 0) {
     if (KMP_IS_D_LOCK(lockseq)) {
-      KMP_COMPARE_AND_STORE_ACQ32(
+      (void)KMP_COMPARE_AND_STORE_ACQ32(
           (volatile kmp_int32 *)&((kmp_base_tas_lock_t *)crit)->poll, 0,
           KMP_GET_D_TAG(lockseq));
     } else {
@@ -3486,8 +3486,8 @@ __kmp_enter_critical_section_reduce_block(ident_t *loc, kmp_int32 global_tid,
   // Check if it is initialized.
   if (*lk == 0) {
     if (KMP_IS_D_LOCK(__kmp_user_lock_seq)) {
-      KMP_COMPARE_AND_STORE_ACQ32((volatile kmp_int32 *)crit, 0,
-                                  KMP_GET_D_TAG(__kmp_user_lock_seq));
+      (void)KMP_COMPARE_AND_STORE_ACQ32((volatile kmp_int32 *)crit, 0,
+                                        KMP_GET_D_TAG(__kmp_user_lock_seq));
     } else {
       __kmp_init_indirect_csptr(crit, loc, global_tid,
                                 KMP_GET_I_TAG(__kmp_user_lock_seq));
diff --git a/openmp/runtime/src/kmp_runtime.cpp b/openmp/runtime/src/kmp_runtime.cpp
index 5b4391aa125d417..06bc4939359e264 100644
--- a/openmp/runtime/src/kmp_runtime.cpp
+++ b/openmp/runtime/src/kmp_runtime.cpp
@@ -1983,8 +1983,8 @@ int __kmp_fork_call(ident_t *loc, int gtid,
 
 #if OMPT_SUPPORT
     ompt_data_t ompt_parallel_data = ompt_data_none;
-    ompt_data_t *parent_task_data;
-    ompt_frame_t *ompt_frame;
+    ompt_data_t *parent_task_data = NULL;
+    ompt_frame_t *ompt_frame = NULL;
     void *return_address = NULL;
 
     if (ompt_enabled.enabled) {
@@ -5765,8 +5765,8 @@ void __kmp_free_team(kmp_root_t *root,
     for (f = 1; f < team->t.t_nproc; ++f) {
       KMP_DEBUG_ASSERT(team->t.t_threads[f]);
       if (__kmp_barrier_gather_pattern[bs_forkjoin_barrier] == bp_dist_bar) {
-        KMP_COMPARE_AND_STORE_ACQ32(&(team->t.t_threads[f]->th.th_used_in_team),
-                                    1, 2);
+        (void)KMP_COMPARE_AND_STORE_ACQ32(
+            &(team->t.t_threads[f]->th.th_used_in_team), 1, 2);
       }
       __kmp_free_thread(team->t.t_threads[f]);
     }
@@ -9220,8 +9220,8 @@ void __kmp_add_threads_to_team(kmp_team_t *team, int new_nthreads) {
   // to wake it up.
   for (int f = 1; f < new_nthreads; ++f) {
     KMP_DEBUG_ASSERT(team->t.t_threads[f]);
-    KMP_COMPARE_AND_STORE_ACQ32(&(team->t.t_threads[f]->th.th_used_in_team), 0,
-                                3);
+    (void)KMP_COMPARE_AND_STORE_ACQ32(
+        &(team->t.t_threads[f]->th.th_used_in_team), 0, 3);
     if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) { // Wake up sleeping threads
       __kmp_resume_32(team->t.t_threads[f]->th.th_info.ds.ds_gtid,
                       (kmp_flag_32<false, false> *)NULL);
diff --git a/openmp/runtime/src/kmp_tasking.cpp b/openmp/runtime/src/kmp_tasking.cpp
index 03ce0dd752af156..7edaa8e127e52c0 100644
--- a/openmp/runtime/src/kmp_tasking.cpp
+++ b/openmp/runtime/src/kmp_tasking.cpp
@@ -5276,7 +5276,7 @@ static void __kmp_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int if_val,
   switch (sched) {
   case 0: // no schedule clause specified, we can choose the default
     // let's try to schedule (team_size*10) tasks
-    grainsize = thread->th.th_team_nproc * 10;
+    grainsize = thread->th.th_team_nproc * static_cast<kmp_uint64>(10);
     KMP_FALLTHROUGH();
   case 2: // num_tasks provided
     if (grainsize > tc) {
diff --git a/openmp/runtime/src/kmp_wait_release.h b/openmp/runtime/src/kmp_wait_release.h
index 97db68943da7027..9baf280228ee52f 100644
--- a/openmp/runtime/src/kmp_wait_release.h
+++ b/openmp/runtime/src/kmp_wait_release.h
@@ -104,7 +104,8 @@ template <> struct flag_traits<flag_oncore> {
 template <flag_type FlagType> class kmp_flag {
 protected:
   flag_properties t; /**< "Type" of the flag in loc */
-  kmp_info_t *waiting_threads[1]; /**< Threads sleeping on this thread. */
+  /**< Threads sleeping on this thread. */
+  kmp_info_t *waiting_threads[1] = {nullptr};
   kmp_uint32 num_waiting_threads; /**< Num threads sleeping on this thread. */
   std::atomic<bool> *sleepLoc;
 
@@ -140,7 +141,7 @@ template <typename PtrType, flag_type FlagType, bool Sleepable>
 class kmp_flag_native : public kmp_flag<FlagType> {
 protected:
   volatile PtrType *loc;
-  PtrType checker; /**< When flag==checker, it has been released. */
+  PtrType checker = (PtrType)0; /**< When flag==checker, it has been released */
   typedef flag_traits<FlagType> traits_type;
 
 public:
@@ -234,7 +235,7 @@ template <typename PtrType, flag_type FlagType, bool Sleepable>
 class kmp_flag_atomic : public kmp_flag<FlagType> {
 protected:
   std::atomic<PtrType> *loc; /**< Pointer to flag location to wait on */
-  PtrType checker; /**< Flag == checker means it has been released. */
+  PtrType checker = (PtrType)0; /**< Flag==checker means it has been released */
 public:
   typedef flag_traits<FlagType> traits_type;
   typedef PtrType flag_t;
@@ -935,7 +936,8 @@ class kmp_flag_oncore : public kmp_flag_native<kmp_uint64, flag_oncore, false> {
   kmp_uint32 offset; /**< Portion of flag of interest for an operation. */
   bool flag_switch; /**< Indicates a switch in flag location. */
   enum barrier_type bt; /**< Barrier type. */
-  kmp_info_t *this_thr; /**< Thread to redirect to different flag location. */
+  /**< Thread to redirect to different flag location. */
+  kmp_info_t *this_thr = nullptr;
 #if USE_ITT_BUILD
   void *itt_sync_obj; /**< ITT object to pass to new flag location. */
 #endif
diff --git a/openmp/runtime/src/ompt-general.cpp b/openmp/runtime/src/ompt-general.cpp
index e07c5ff4fcc47d8..923eea2a563a917 100644
--- a/openmp/runtime/src/ompt-general.cpp
+++ b/openmp/runtime/src/ompt-general.cpp
@@ -104,9 +104,11 @@ static ompt_start_tool_result_t *ompt_start_tool_result = NULL;
 
 #if KMP_OS_WINDOWS
 static HMODULE ompt_tool_module = NULL;
+static HMODULE ompt_archer_module = NULL;
 #define OMPT_DLCLOSE(Lib) FreeLibrary(Lib)
 #else
 static void *ompt_tool_module = NULL;
+static void *ompt_archer_module = NULL;
 #define OMPT_DLCLOSE(Lib) dlclose(Lib)
 #endif
 
@@ -374,6 +376,7 @@ ompt_try_start_tool(unsigned int omp_version, const char *runtime_version) {
               "Tool was started and is using the OMPT interface.\n");
           OMPT_VERBOSE_INIT_PRINT(
               "----- END LOGGING OF TOOL REGISTRATION -----\n");
+          ompt_archer_module = h;
           return ret;
         }
         OMPT_VERBOSE_INIT_CONTINUED_PRINT(
@@ -381,6 +384,7 @@ ompt_try_start_tool(unsigned int omp_version, const char *runtime_version) {
       } else {
         OMPT_VERBOSE_INIT_CONTINUED_PRINT("Failed: %s\n", dlerror());
       }
+      OMPT_DLCLOSE(h);
     }
   }
 #endif
@@ -521,6 +525,8 @@ void ompt_fini() {
     }
   }
 
+  if (ompt_archer_module)
+    OMPT_DLCLOSE(ompt_archer_module);
   if (ompt_tool_module)
     OMPT_DLCLOSE(ompt_tool_module);
   memset(&ompt_enabled, 0, sizeof(ompt_enabled));

From e63b7ba0eb497d10d643aa5e27461c6c1bf8e221 Mon Sep 17 00:00:00 2001
From: Kevin McAfee <kmcafee@nvidia.com>
Date: Thu, 15 Aug 2024 08:49:55 -0700
Subject: [PATCH 041/441] [InstCombine][NFC] Add tests for shifts of constants
 by common factor (#103471)

---
 .../test/Transforms/InstCombine/shl-factor.ll | 24 +++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/llvm/test/Transforms/InstCombine/shl-factor.ll b/llvm/test/Transforms/InstCombine/shl-factor.ll
index 38eecaeff8e41e3..996b15f27f6d345 100644
--- a/llvm/test/Transforms/InstCombine/shl-factor.ll
+++ b/llvm/test/Transforms/InstCombine/shl-factor.ll
@@ -265,3 +265,27 @@ define i6 @sub_shl_same_amount_partial_nuw2(i6 %x, i6 %y, i6 %z) {
   ret i6 %diff
 }
 
+define i8 @add_shl_same_amount_constants(i8 %z) {
+; CHECK-LABEL: @add_shl_same_amount_constants(
+; CHECK-NEXT:    [[SUM:%.*]] = shl i8 7, [[Z:%.*]]
+; CHECK-NEXT:    ret i8 [[SUM]]
+;
+  %s1 = shl i8 4, %z
+  %s2 = shl i8 3, %z
+  %sum = add i8 %s1, %s2
+  ret i8 %sum
+}
+
+define i8 @add_shl_same_amount_constants_extra_use(i8 %z) {
+; CHECK-LABEL: @add_shl_same_amount_constants_extra_use(
+; CHECK-NEXT:    [[S1:%.*]] = shl i8 4, [[Z:%.*]]
+; CHECK-NEXT:    [[SUM:%.*]] = shl i8 7, [[Z]]
+; CHECK-NEXT:    call void @use8(i8 [[S1]])
+; CHECK-NEXT:    ret i8 [[SUM]]
+;
+  %s1 = shl i8 4, %z
+  %s2 = shl i8 3, %z
+  %sum = add i8 %s1, %s2
+  call void @use8(i8 %s1)
+  ret i8 %sum
+}

From 51328b78dc2b0be20e8d67f57f64445cec25162c Mon Sep 17 00:00:00 2001
From: sp <james@rooted.gg>
Date: Thu, 15 Aug 2024 17:00:03 +0100
Subject: [PATCH 042/441] [NFC] Fix spelling of "definitely". (#104455)

This corrects the misspelling of "definitely" as "definately". No
functional changes.
---
 clang/lib/StaticAnalyzer/Checkers/StringChecker.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/lib/StaticAnalyzer/Checkers/StringChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/StringChecker.cpp
index 8f1c31763e212c3..f50b5cd3c9e497e 100644
--- a/clang/lib/StaticAnalyzer/Checkers/StringChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/StringChecker.cpp
@@ -48,7 +48,7 @@ bool StringChecker::isCharToStringCtor(const CallEvent &Call,
 
   // Verify that the parameters have the expected types:
   // - arg 1: `const CharT *`
-  // - arg 2: some allocator - which is definately not `size_t`.
+  // - arg 2: some allocator - which is definitely not `size_t`.
   const QualType Arg1Ty = Call.getArgExpr(0)->getType().getCanonicalType();
   const QualType Arg2Ty = Call.getArgExpr(1)->getType().getCanonicalType();
 

From 64c856055aed97603510410fd2feab273e1d5b8a Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Thu, 15 Aug 2024 09:01:06 -0700
Subject: [PATCH 043/441] Revert "[Object][x86-64] Add support for
 `R_X86_64_GLOB_DAT` relocations. (#103029)" (#103497)

This reverts commit 5ae9faa538d100ab38f6f4f99c924de0e4270272.

RelocationResolver is only supposed to handle static relocation types.
Introducing GLOB_DAT could negatively impact other RelocationResolver
users who solely handle static relocations and want to report errors for
dynamic relocations.

If GLOB_DAT is the sole required relocation, explicitly checking for it
in the caller would be more reliable. Additionally, the caller should
handle GLOB_DAT on other architectures.
---
 llvm/lib/Object/RelocationResolver.cpp | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/llvm/lib/Object/RelocationResolver.cpp b/llvm/lib/Object/RelocationResolver.cpp
index 7f90834c3d21997..d9bb8f175ef83ac 100644
--- a/llvm/lib/Object/RelocationResolver.cpp
+++ b/llvm/lib/Object/RelocationResolver.cpp
@@ -47,7 +47,6 @@ static bool supportsX86_64(uint64_t Type) {
   case ELF::R_X86_64_PC64:
   case ELF::R_X86_64_32:
   case ELF::R_X86_64_32S:
-  case ELF::R_X86_64_GLOB_DAT:
     return true;
   default:
     return false;
@@ -69,8 +68,6 @@ static uint64_t resolveX86_64(uint64_t Type, uint64_t Offset, uint64_t S,
   case ELF::R_X86_64_32:
   case ELF::R_X86_64_32S:
     return (S + Addend) & 0xFFFFFFFF;
-  case ELF::R_X86_64_GLOB_DAT:
-    return S;
   default:
     llvm_unreachable("Invalid relocation type");
   }

From 46fb225f3ac602970ebb8973a5376cd9216ba38f Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Thu, 15 Aug 2024 18:07:18 +0200
Subject: [PATCH 044/441] [InstSimplify] Add tests for f16 to i128 range (NFC)

---
 .../Transforms/InstSimplify/fptoi-range.ll    | 22 +++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/llvm/test/Transforms/InstSimplify/fptoi-range.ll b/llvm/test/Transforms/InstSimplify/fptoi-range.ll
index cc9605259aa811e..e0689c26c719031 100644
--- a/llvm/test/Transforms/InstSimplify/fptoi-range.ll
+++ b/llvm/test/Transforms/InstSimplify/fptoi-range.ll
@@ -32,6 +32,17 @@ define i1 @f16_si16_max2(half %f) {
   ret i1 %c
 }
 
+define i1 @f16_si128_max2(half %f) {
+; CHECK-LABEL: @f16_si128_max2(
+; CHECK-NEXT:    [[I:%.*]] = fptosi half [[F:%.*]] to i128
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i128 [[I]], 65504
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %i = fptosi half %f to i128
+  %c = icmp sgt i128 %i, 65504
+  ret i1 %c
+}
+
 define i1 @f16_si_min1(half %f) {
 ; CHECK-LABEL: @f16_si_min1(
 ; CHECK-NEXT:    ret i1 true
@@ -41,6 +52,17 @@ define i1 @f16_si_min1(half %f) {
   ret i1 %c
 }
 
+define i1 @f16_si128_min1(half %f) {
+; CHECK-LABEL: @f16_si128_min1(
+; CHECK-NEXT:    [[I:%.*]] = fptosi half [[F:%.*]] to i128
+; CHECK-NEXT:    [[C:%.*]] = icmp sge i128 [[I]], -65504
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %i = fptosi half %f to i128
+  %c = icmp sge i128 %i, -65504
+  ret i1 %c
+}
+
 define i1 @f16_si16_min1(half %f) {
 ; CHECK-LABEL: @f16_si16_min1(
 ; CHECK-NEXT:    [[I:%.*]] = fptosi half [[F:%.*]] to i16

From afa0f53f96b5563a80fbdf8c41c8153bf8cd2685 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Thu, 15 Aug 2024 18:08:27 +0200
Subject: [PATCH 045/441] [ValueTracking] Fix f16 fptosi range for large
 integers

We were missing the signed flag on the negative value, so the
range was incorrectly interpreted for integers larger than 64-bit.

Split out from https://github.com/llvm/llvm-project/pull/80309.
---
 llvm/lib/Analysis/ValueTracking.cpp           |   2 +-
 .../CodeGen/Thumb2/mve-fpclamptosat_vec.ll    | 194 ++++--------------
 .../Transforms/InstSimplify/fptoi-range.ll    |   8 +-
 3 files changed, 43 insertions(+), 161 deletions(-)

diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index b6414a3e7990b0a..014a703a0005240 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -9671,7 +9671,7 @@ static void setLimitForFPToI(const Instruction *I, APInt &Lower, APInt &Upper) {
   if (!I->getOperand(0)->getType()->getScalarType()->isHalfTy())
     return;
   if (isa<FPToSIInst>(I) && BitWidth >= 17) {
-    Lower = APInt(BitWidth, -65504);
+    Lower = APInt(BitWidth, -65504, true);
     Upper = APInt(BitWidth, 65505);
   }
 
diff --git a/llvm/test/CodeGen/Thumb2/mve-fpclamptosat_vec.ll b/llvm/test/CodeGen/Thumb2/mve-fpclamptosat_vec.ll
index 94210d795867a0f..f2ac5268921800c 100644
--- a/llvm/test/CodeGen/Thumb2/mve-fpclamptosat_vec.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-fpclamptosat_vec.ll
@@ -1012,60 +1012,21 @@ entry:
 define arm_aapcs_vfpcc <2 x i64> @stest_f16i64(<2 x half> %x) {
 ; CHECK-LABEL: stest_f16i64:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, lr}
-; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, r10, lr}
+; CHECK-NEXT:    .save {r4, r5, r7, lr}
+; CHECK-NEXT:    push {r4, r5, r7, lr}
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
 ; CHECK-NEXT:    vmov.u16 r0, q0[1]
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    bl __fixhfti
-; CHECK-NEXT:    subs.w r7, r0, #-1
-; CHECK-NEXT:    mvn r9, #-2147483648
-; CHECK-NEXT:    sbcs.w r7, r1, r9
-; CHECK-NEXT:    mov.w r10, #-2147483648
-; CHECK-NEXT:    sbcs r7, r2, #0
-; CHECK-NEXT:    sbcs r7, r3, #0
-; CHECK-NEXT:    cset r7, lt
-; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    csel r3, r3, r7, ne
-; CHECK-NEXT:    csel r2, r2, r7, ne
-; CHECK-NEXT:    mov.w r7, #-1
-; CHECK-NEXT:    csel r1, r1, r9, ne
-; CHECK-NEXT:    csel r4, r0, r7, ne
-; CHECK-NEXT:    rsbs r0, r4, #0
-; CHECK-NEXT:    sbcs.w r0, r10, r1
-; CHECK-NEXT:    sbcs.w r0, r7, r2
-; CHECK-NEXT:    sbcs.w r0, r7, r3
-; CHECK-NEXT:    cset r5, lt
+; CHECK-NEXT:    mov r4, r0
 ; CHECK-NEXT:    vmov.u16 r0, q4[0]
-; CHECK-NEXT:    cmp r5, #0
-; CHECK-NEXT:    csel r8, r1, r10, ne
+; CHECK-NEXT:    mov r5, r1
 ; CHECK-NEXT:    bl __fixhfti
-; CHECK-NEXT:    subs.w r6, r0, #-1
-; CHECK-NEXT:    sbcs.w r6, r1, r9
-; CHECK-NEXT:    sbcs r6, r2, #0
-; CHECK-NEXT:    sbcs r6, r3, #0
-; CHECK-NEXT:    cset r6, lt
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    csel r0, r0, r7, ne
-; CHECK-NEXT:    csel r1, r1, r9, ne
-; CHECK-NEXT:    csel r3, r3, r6, ne
-; CHECK-NEXT:    csel r2, r2, r6, ne
-; CHECK-NEXT:    rsbs r6, r0, #0
-; CHECK-NEXT:    sbcs.w r6, r10, r1
-; CHECK-NEXT:    sbcs.w r2, r7, r2
-; CHECK-NEXT:    sbcs.w r2, r7, r3
-; CHECK-NEXT:    cset r2, lt
-; CHECK-NEXT:    cmp r2, #0
-; CHECK-NEXT:    csel r1, r1, r10, ne
-; CHECK-NEXT:    cmp r5, #0
-; CHECK-NEXT:    csel r3, r4, r5, ne
-; CHECK-NEXT:    cmp r2, #0
-; CHECK-NEXT:    csel r0, r0, r2, ne
-; CHECK-NEXT:    vmov q0[2], q0[0], r0, r3
-; CHECK-NEXT:    vmov q0[3], q0[1], r1, r8
+; CHECK-NEXT:    vmov q0[2], q0[0], r0, r4
+; CHECK-NEXT:    vmov q0[3], q0[1], r1, r5
 ; CHECK-NEXT:    vpop {d8, d9}
-; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, pc}
+; CHECK-NEXT:    pop {r4, r5, r7, pc}
 entry:
   %conv = fptosi <2 x half> %x to <2 x i128>
   %0 = icmp slt <2 x i128> %conv, <i128 9223372036854775807, i128 9223372036854775807>
@@ -1105,46 +1066,28 @@ entry:
 define arm_aapcs_vfpcc <2 x i64> @ustest_f16i64(<2 x half> %x) {
 ; CHECK-LABEL: ustest_f16i64:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, r9, lr}
-; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, lr}
-; CHECK-NEXT:    .pad #4
-; CHECK-NEXT:    sub sp, #4
+; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, lr}
+; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, lr}
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
 ; CHECK-NEXT:    vmov.u16 r0, q0[1]
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    bl __fixhfti
-; CHECK-NEXT:    subs r5, r2, #1
-; CHECK-NEXT:    mov.w r8, #1
-; CHECK-NEXT:    sbcs r5, r3, #0
-; CHECK-NEXT:    mov.w r7, #0
-; CHECK-NEXT:    cset r5, lt
-; CHECK-NEXT:    cmp r5, #0
-; CHECK-NEXT:    csel r0, r0, r5, ne
-; CHECK-NEXT:    csel r3, r3, r5, ne
-; CHECK-NEXT:    csel r2, r2, r8, ne
-; CHECK-NEXT:    csel r4, r1, r5, ne
+; CHECK-NEXT:    mov r4, r1
 ; CHECK-NEXT:    rsbs r1, r0, #0
-; CHECK-NEXT:    sbcs.w r1, r7, r4
-; CHECK-NEXT:    sbcs.w r1, r7, r2
-; CHECK-NEXT:    sbcs.w r1, r7, r3
+; CHECK-NEXT:    mov.w r5, #0
+; CHECK-NEXT:    sbcs.w r1, r5, r4
+; CHECK-NEXT:    sbcs.w r1, r5, r2
+; CHECK-NEXT:    sbcs.w r1, r5, r3
 ; CHECK-NEXT:    cset r6, lt
 ; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    csel r9, r0, r6, ne
+; CHECK-NEXT:    csel r8, r0, r6, ne
 ; CHECK-NEXT:    vmov.u16 r0, q4[0]
 ; CHECK-NEXT:    bl __fixhfti
-; CHECK-NEXT:    subs r5, r2, #1
-; CHECK-NEXT:    sbcs r5, r3, #0
-; CHECK-NEXT:    cset r5, lt
-; CHECK-NEXT:    cmp r5, #0
-; CHECK-NEXT:    csel r0, r0, r5, ne
-; CHECK-NEXT:    csel r2, r2, r8, ne
-; CHECK-NEXT:    csel r3, r3, r5, ne
-; CHECK-NEXT:    csel r1, r1, r5, ne
-; CHECK-NEXT:    rsbs r5, r0, #0
-; CHECK-NEXT:    sbcs.w r5, r7, r1
-; CHECK-NEXT:    sbcs.w r2, r7, r2
-; CHECK-NEXT:    sbcs.w r2, r7, r3
+; CHECK-NEXT:    rsbs r7, r0, #0
+; CHECK-NEXT:    sbcs.w r7, r5, r1
+; CHECK-NEXT:    sbcs.w r2, r5, r2
+; CHECK-NEXT:    sbcs.w r2, r5, r3
 ; CHECK-NEXT:    cset r2, lt
 ; CHECK-NEXT:    cmp r2, #0
 ; CHECK-NEXT:    csel r0, r0, r2, ne
@@ -1152,11 +1095,10 @@ define arm_aapcs_vfpcc <2 x i64> @ustest_f16i64(<2 x half> %x) {
 ; CHECK-NEXT:    csel r3, r4, r6, ne
 ; CHECK-NEXT:    cmp r2, #0
 ; CHECK-NEXT:    csel r1, r1, r2, ne
-; CHECK-NEXT:    vmov q0[2], q0[0], r0, r9
+; CHECK-NEXT:    vmov q0[2], q0[0], r0, r8
 ; CHECK-NEXT:    vmov q0[3], q0[1], r1, r3
 ; CHECK-NEXT:    vpop {d8, d9}
-; CHECK-NEXT:    add sp, #4
-; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, pc}
+; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, pc}
 entry:
   %conv = fptosi <2 x half> %x to <2 x i128>
   %0 = icmp slt <2 x i128> %conv, <i128 18446744073709551616, i128 18446744073709551616>
@@ -2119,60 +2061,21 @@ entry:
 define arm_aapcs_vfpcc <2 x i64> @stest_f16i64_mm(<2 x half> %x) {
 ; CHECK-LABEL: stest_f16i64_mm:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, lr}
-; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, r10, lr}
+; CHECK-NEXT:    .save {r4, r5, r7, lr}
+; CHECK-NEXT:    push {r4, r5, r7, lr}
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
 ; CHECK-NEXT:    vmov.u16 r0, q0[1]
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    bl __fixhfti
-; CHECK-NEXT:    subs.w r7, r0, #-1
-; CHECK-NEXT:    mvn r9, #-2147483648
-; CHECK-NEXT:    sbcs.w r7, r1, r9
-; CHECK-NEXT:    mov.w r10, #-2147483648
-; CHECK-NEXT:    sbcs r7, r2, #0
-; CHECK-NEXT:    sbcs r7, r3, #0
-; CHECK-NEXT:    cset r7, lt
-; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    csel r3, r3, r7, ne
-; CHECK-NEXT:    csel r2, r2, r7, ne
-; CHECK-NEXT:    mov.w r7, #-1
-; CHECK-NEXT:    csel r1, r1, r9, ne
-; CHECK-NEXT:    csel r4, r0, r7, ne
-; CHECK-NEXT:    rsbs r0, r4, #0
-; CHECK-NEXT:    sbcs.w r0, r10, r1
-; CHECK-NEXT:    sbcs.w r0, r7, r2
-; CHECK-NEXT:    sbcs.w r0, r7, r3
-; CHECK-NEXT:    cset r5, lt
+; CHECK-NEXT:    mov r4, r0
 ; CHECK-NEXT:    vmov.u16 r0, q4[0]
-; CHECK-NEXT:    cmp r5, #0
-; CHECK-NEXT:    csel r8, r1, r10, ne
+; CHECK-NEXT:    mov r5, r1
 ; CHECK-NEXT:    bl __fixhfti
-; CHECK-NEXT:    subs.w r6, r0, #-1
-; CHECK-NEXT:    sbcs.w r6, r1, r9
-; CHECK-NEXT:    sbcs r6, r2, #0
-; CHECK-NEXT:    sbcs r6, r3, #0
-; CHECK-NEXT:    cset r6, lt
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    csel r0, r0, r7, ne
-; CHECK-NEXT:    csel r1, r1, r9, ne
-; CHECK-NEXT:    csel r3, r3, r6, ne
-; CHECK-NEXT:    csel r2, r2, r6, ne
-; CHECK-NEXT:    rsbs r6, r0, #0
-; CHECK-NEXT:    sbcs.w r6, r10, r1
-; CHECK-NEXT:    sbcs.w r2, r7, r2
-; CHECK-NEXT:    sbcs.w r2, r7, r3
-; CHECK-NEXT:    cset r2, lt
-; CHECK-NEXT:    cmp r2, #0
-; CHECK-NEXT:    csel r1, r1, r10, ne
-; CHECK-NEXT:    cmp r5, #0
-; CHECK-NEXT:    csel r3, r4, r5, ne
-; CHECK-NEXT:    cmp r2, #0
-; CHECK-NEXT:    csel r0, r0, r2, ne
-; CHECK-NEXT:    vmov q0[2], q0[0], r0, r3
-; CHECK-NEXT:    vmov q0[3], q0[1], r1, r8
+; CHECK-NEXT:    vmov q0[2], q0[0], r0, r4
+; CHECK-NEXT:    vmov q0[3], q0[1], r1, r5
 ; CHECK-NEXT:    vpop {d8, d9}
-; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, pc}
+; CHECK-NEXT:    pop {r4, r5, r7, pc}
 entry:
   %conv = fptosi <2 x half> %x to <2 x i128>
   %spec.store.select = call <2 x i128> @llvm.smin.v2i128(<2 x i128> %conv, <2 x i128> <i128 9223372036854775807, i128 9223372036854775807>)
@@ -2209,51 +2112,34 @@ entry:
 define arm_aapcs_vfpcc <2 x i64> @ustest_f16i64_mm(<2 x half> %x) {
 ; CHECK-LABEL: ustest_f16i64_mm:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, r5, r6, r7, lr}
-; CHECK-NEXT:    push {r4, r5, r6, r7, lr}
-; CHECK-NEXT:    .pad #4
-; CHECK-NEXT:    sub sp, #4
+; CHECK-NEXT:    .save {r4, r5, r6, lr}
+; CHECK-NEXT:    push {r4, r5, r6, lr}
 ; CHECK-NEXT:    .vsave {d8, d9}
 ; CHECK-NEXT:    vpush {d8, d9}
 ; CHECK-NEXT:    vmov.u16 r0, q0[1]
 ; CHECK-NEXT:    vmov q4, q0
 ; CHECK-NEXT:    bl __fixhfti
-; CHECK-NEXT:    mov r4, r1
-; CHECK-NEXT:    subs r1, r2, #1
-; CHECK-NEXT:    sbcs r1, r3, #0
-; CHECK-NEXT:    cset r6, lt
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    csel r5, r0, r6, ne
-; CHECK-NEXT:    csel r7, r3, r6, ne
+; CHECK-NEXT:    mov r4, r0
 ; CHECK-NEXT:    vmov.u16 r0, q4[0]
-; CHECK-NEXT:    cmp r7, #0
-; CHECK-NEXT:    it mi
-; CHECK-NEXT:    movmi r5, #0
+; CHECK-NEXT:    mov r5, r1
+; CHECK-NEXT:    mov r6, r3
 ; CHECK-NEXT:    bl __fixhfti
-; CHECK-NEXT:    subs r2, #1
-; CHECK-NEXT:    sbcs r2, r3, #0
-; CHECK-NEXT:    cset r2, lt
-; CHECK-NEXT:    cmp r2, #0
-; CHECK-NEXT:    csel r3, r3, r2, ne
-; CHECK-NEXT:    csel r0, r0, r2, ne
+; CHECK-NEXT:    cmp r6, #0
+; CHECK-NEXT:    it mi
+; CHECK-NEXT:    movmi r4, #0
 ; CHECK-NEXT:    cmp r3, #0
 ; CHECK-NEXT:    it mi
 ; CHECK-NEXT:    movmi r0, #0
 ; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    vmov q0[2], q0[0], r0, r5
-; CHECK-NEXT:    csel r6, r4, r6, ne
-; CHECK-NEXT:    cmp r7, #0
+; CHECK-NEXT:    vmov q0[2], q0[0], r0, r4
 ; CHECK-NEXT:    it mi
-; CHECK-NEXT:    movmi r6, #0
-; CHECK-NEXT:    cmp r2, #0
-; CHECK-NEXT:    csel r1, r1, r2, ne
+; CHECK-NEXT:    movmi r5, #0
 ; CHECK-NEXT:    cmp r3, #0
 ; CHECK-NEXT:    it mi
 ; CHECK-NEXT:    movmi r1, #0
-; CHECK-NEXT:    vmov q0[3], q0[1], r1, r6
+; CHECK-NEXT:    vmov q0[3], q0[1], r1, r5
 ; CHECK-NEXT:    vpop {d8, d9}
-; CHECK-NEXT:    add sp, #4
-; CHECK-NEXT:    pop {r4, r5, r6, r7, pc}
+; CHECK-NEXT:    pop {r4, r5, r6, pc}
 entry:
   %conv = fptosi <2 x half> %x to <2 x i128>
   %spec.store.select = call <2 x i128> @llvm.smin.v2i128(<2 x i128> %conv, <2 x i128> <i128 18446744073709551616, i128 18446744073709551616>)
diff --git a/llvm/test/Transforms/InstSimplify/fptoi-range.ll b/llvm/test/Transforms/InstSimplify/fptoi-range.ll
index e0689c26c719031..95f2a9d50793c64 100644
--- a/llvm/test/Transforms/InstSimplify/fptoi-range.ll
+++ b/llvm/test/Transforms/InstSimplify/fptoi-range.ll
@@ -34,9 +34,7 @@ define i1 @f16_si16_max2(half %f) {
 
 define i1 @f16_si128_max2(half %f) {
 ; CHECK-LABEL: @f16_si128_max2(
-; CHECK-NEXT:    [[I:%.*]] = fptosi half [[F:%.*]] to i128
-; CHECK-NEXT:    [[C:%.*]] = icmp sgt i128 [[I]], 65504
-; CHECK-NEXT:    ret i1 [[C]]
+; CHECK-NEXT:    ret i1 false
 ;
   %i = fptosi half %f to i128
   %c = icmp sgt i128 %i, 65504
@@ -54,9 +52,7 @@ define i1 @f16_si_min1(half %f) {
 
 define i1 @f16_si128_min1(half %f) {
 ; CHECK-LABEL: @f16_si128_min1(
-; CHECK-NEXT:    [[I:%.*]] = fptosi half [[F:%.*]] to i128
-; CHECK-NEXT:    [[C:%.*]] = icmp sge i128 [[I]], -65504
-; CHECK-NEXT:    ret i1 [[C]]
+; CHECK-NEXT:    ret i1 true
 ;
   %i = fptosi half %f to i128
   %c = icmp sge i128 %i, -65504

From b6bb208662b980b3c29194f63f22e3af8f772a57 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev@outlook.com>
Date: Thu, 15 Aug 2024 09:18:33 -0700
Subject: [PATCH 046/441] Revert "[SLP][NFC]Remove unused using declarations,
 reduce mem usage in containers, NFC"

This reverts commit 2d52eb6a434fe47e67086f5ec1c3789bf6e7a604 to fix
compile time regression found in https://llvm-compile-time-tracker.com/compare.php?from=fcefe957ddfdc5a2fe9463757b597635e3436e01&to=2d52eb6a434fe47e67086f5ec1c3789bf6e7a604&stat=instructions%3Au.
---
 .../Transforms/Vectorize/SLPVectorizer.cpp    | 24 ++++++++++---------
 1 file changed, 13 insertions(+), 11 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 15cff2f10030109..87b4ed599b5bb15 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -1244,10 +1244,12 @@ class BoUpSLP {
   };
 
   using ValueList = SmallVector<Value *, 8>;
+  using InstrList = SmallVector<Instruction *, 16>;
   using ValueSet = SmallPtrSet<Value *, 16>;
+  using StoreList = SmallVector<StoreInst *, 8>;
   using ExtraValueToDebugLocsMap =
       MapVector<Value *, SmallVector<Instruction *, 2>>;
-  using OrdersType = SmallVector<unsigned, 0>;
+  using OrdersType = SmallVector<unsigned, 4>;
 
   BoUpSLP(Function *Func, ScalarEvolution *Se, TargetTransformInfo *Tti,
           TargetLibraryInfo *TLi, AAResults *Aa, LoopInfo *Li,
@@ -1469,7 +1471,7 @@ class BoUpSLP {
   /// \param TryRecursiveCheck used to check if long masked gather can be
   /// represented as a serie of loads/insert subvector, if profitable.
   LoadsState canVectorizeLoads(ArrayRef<Value *> VL, const Value *VL0,
-                               OrdersType &Order,
+                               SmallVectorImpl<unsigned> &Order,
                                SmallVectorImpl<Value *> &PointerOps,
                                bool TryRecursiveCheck = true) const;
 
@@ -2838,7 +2840,7 @@ class BoUpSLP {
   /// \param ResizeAllowed indicates whether it is allowed to handle subvector
   /// extract order.
   bool canReuseExtract(ArrayRef<Value *> VL, Value *OpValue,
-                       OrdersType &CurrentOrder,
+                       SmallVectorImpl<unsigned> &CurrentOrder,
                        bool ResizeAllowed = false) const;
 
   /// Vectorize a single entry in the tree.
@@ -3082,10 +3084,10 @@ class BoUpSLP {
     CombinedOpcode CombinedOp = NotCombinedOp;
 
     /// Does this sequence require some shuffling?
-    SmallVector<int, 0> ReuseShuffleIndices;
+    SmallVector<int, 4> ReuseShuffleIndices;
 
     /// Does this entry require reordering?
-    OrdersType ReorderIndices;
+    SmallVector<unsigned, 4> ReorderIndices;
 
     /// Points back to the VectorizableTree.
     ///
@@ -4298,12 +4300,12 @@ static void reorderReuses(SmallVectorImpl<int> &Reuses, ArrayRef<int> Mask) {
 /// the original order of the scalars. Procedure transforms the provided order
 /// in accordance with the given \p Mask. If the resulting \p Order is just an
 /// identity order, \p Order is cleared.
-static void reorderOrder(BoUpSLP::OrdersType &Order, ArrayRef<int> Mask,
+static void reorderOrder(SmallVectorImpl<unsigned> &Order, ArrayRef<int> Mask,
                          bool BottomOrder = false) {
   assert(!Mask.empty() && "Expected non-empty mask.");
   unsigned Sz = Mask.size();
   if (BottomOrder) {
-    BoUpSLP::OrdersType PrevOrder;
+    SmallVector<unsigned> PrevOrder;
     if (Order.empty()) {
       PrevOrder.resize(Sz);
       std::iota(PrevOrder.begin(), PrevOrder.end(), 0);
@@ -4693,7 +4695,7 @@ getShuffleCost(const TargetTransformInfo &TTI, TTI::ShuffleKind Kind,
 }
 
 BoUpSLP::LoadsState BoUpSLP::canVectorizeLoads(
-    ArrayRef<Value *> VL, const Value *VL0, OrdersType &Order,
+    ArrayRef<Value *> VL, const Value *VL0, SmallVectorImpl<unsigned> &Order,
     SmallVectorImpl<Value *> &PointerOps, bool TryRecursiveCheck) const {
   // Check that a vectorized load would load the same memory as a scalar
   // load. For example, we don't want to vectorize loads that are smaller
@@ -4821,7 +4823,7 @@ BoUpSLP::LoadsState BoUpSLP::canVectorizeLoads(
         for (unsigned Cnt = 0, End = VL.size(); Cnt + VF <= End;
              Cnt += VF, ++VectorizedCnt) {
           ArrayRef<Value *> Slice = VL.slice(Cnt, VF);
-          OrdersType Order;
+          SmallVector<unsigned> Order;
           SmallVector<Value *> PointerOps;
           LoadsState LS =
               canVectorizeLoads(Slice, Slice.front(), Order, PointerOps,
@@ -5395,7 +5397,7 @@ void BoUpSLP::reorderNodeWithReuses(TreeEntry &TE, ArrayRef<int> Mask) const {
   TE.ReorderIndices.clear();
   // Try to improve gathered nodes with clustered reuses, if possible.
   ArrayRef<int> Slice = ArrayRef(NewMask).slice(0, Sz);
-  OrdersType NewOrder(Slice);
+  SmallVector<unsigned> NewOrder(Slice);
   inversePermutation(NewOrder, NewMask);
   reorderScalars(TE.Scalars, NewMask);
   // Fill the reuses mask with the identity submasks.
@@ -7715,7 +7717,7 @@ unsigned BoUpSLP::canMapToVector(Type *T) const {
 }
 
 bool BoUpSLP::canReuseExtract(ArrayRef<Value *> VL, Value *OpValue,
-                              OrdersType &CurrentOrder,
+                              SmallVectorImpl<unsigned> &CurrentOrder,
                               bool ResizeAllowed) const {
   const auto *It = find_if(VL, IsaPred<ExtractElementInst, ExtractValueInst>);
   assert(It != VL.end() && "Expected at least one extract instruction.");

From 9e0ee0e104a2f10b04144837d6a138b04a0193f6 Mon Sep 17 00:00:00 2001
From: Hansang Bae <hansang.bae@intel.com>
Date: Thu, 15 Aug 2024 11:44:50 -0500
Subject: [PATCH 047/441] [OpenMP] Add support for pause with
 omp_pause_stop_tool (#97100)

This patch adds support for pause resource with a new enumerator
omp_pause_stop_tool. The expected behavior of this enumerator is
* omp_pause_resource: not allowed
* omp_pause_resource_all: equivalent to omp_pause_hard
---
 openmp/runtime/src/include/omp.h.var          |  3 +-
 openmp/runtime/src/include/omp_lib.F90.var    |  1 +
 openmp/runtime/src/include/omp_lib.h.var      |  2 +
 openmp/runtime/src/kmp.h                      |  3 +-
 openmp/runtime/src/kmp_ftn_entry.h            |  2 +
 openmp/runtime/src/kmp_runtime.cpp            |  3 +-
 .../runtime/test/ompt/misc/pause_stop_tool.c  | 44 +++++++++++++++++++
 7 files changed, 55 insertions(+), 3 deletions(-)
 create mode 100644 openmp/runtime/test/ompt/misc/pause_stop_tool.c

diff --git a/openmp/runtime/src/include/omp.h.var b/openmp/runtime/src/include/omp.h.var
index eb3ab7778606a35..dee46e51d6b340c 100644
--- a/openmp/runtime/src/include/omp.h.var
+++ b/openmp/runtime/src/include/omp.h.var
@@ -476,7 +476,8 @@
     typedef enum omp_pause_resource_t {
       omp_pause_resume = 0,
       omp_pause_soft = 1,
-      omp_pause_hard = 2
+      omp_pause_hard = 2,
+      omp_pause_stop_tool = 3
     } omp_pause_resource_t;
     extern int __KAI_KMPC_CONVENTION omp_pause_resource(omp_pause_resource_t, int);
     extern int __KAI_KMPC_CONVENTION omp_pause_resource_all(omp_pause_resource_t);
diff --git a/openmp/runtime/src/include/omp_lib.F90.var b/openmp/runtime/src/include/omp_lib.F90.var
index 63a3c93b8d92965..5133915c7d8cbd1 100644
--- a/openmp/runtime/src/include/omp_lib.F90.var
+++ b/openmp/runtime/src/include/omp_lib.F90.var
@@ -192,6 +192,7 @@
         integer (kind=omp_pause_resource_kind), parameter, public :: omp_pause_resume = 0
         integer (kind=omp_pause_resource_kind), parameter, public :: omp_pause_soft = 1
         integer (kind=omp_pause_resource_kind), parameter, public :: omp_pause_hard = 2
+        integer (kind=omp_pause_resource_kind), parameter, public :: omp_pause_stop_tool = 3
 
         integer (kind=omp_interop_fr_kind), parameter, public :: omp_ifr_cuda = 1
         integer (kind=omp_interop_fr_kind), parameter, public :: omp_ifr_cuda_driver = 2
diff --git a/openmp/runtime/src/include/omp_lib.h.var b/openmp/runtime/src/include/omp_lib.h.var
index a709a2f298f8c8c..db1dc889d129989 100644
--- a/openmp/runtime/src/include/omp_lib.h.var
+++ b/openmp/runtime/src/include/omp_lib.h.var
@@ -248,6 +248,8 @@
       parameter(omp_pause_soft=1)
       integer(kind=omp_pause_resource_kind)omp_pause_hard
       parameter(omp_pause_hard=2)
+      integer(kind=omp_pause_resource_kind)omp_pause_stop_tool
+      parameter(omp_pause_stop_tool=3)
 
       integer(kind=omp_interop_fr_kind)omp_ifr_cuda
       parameter(omp_ifr_cuda=1)
diff --git a/openmp/runtime/src/kmp.h b/openmp/runtime/src/kmp.h
index 4f4fb3292d63eec..04bf6c3b34dace2 100644
--- a/openmp/runtime/src/kmp.h
+++ b/openmp/runtime/src/kmp.h
@@ -4552,7 +4552,8 @@ extern int __kmpc_get_target_offload();
 typedef enum kmp_pause_status_t {
   kmp_not_paused = 0, // status is not paused, or, requesting resume
   kmp_soft_paused = 1, // status is soft-paused, or, requesting soft pause
-  kmp_hard_paused = 2 // status is hard-paused, or, requesting hard pause
+  kmp_hard_paused = 2, // status is hard-paused, or, requesting hard pause
+  kmp_stop_tool_paused = 3 // requesting stop_tool pause
 } kmp_pause_status_t;
 
 // This stores the pause state of the runtime
diff --git a/openmp/runtime/src/kmp_ftn_entry.h b/openmp/runtime/src/kmp_ftn_entry.h
index 713561734c481ab..9bd3ac973b3528b 100644
--- a/openmp/runtime/src/kmp_ftn_entry.h
+++ b/openmp/runtime/src/kmp_ftn_entry.h
@@ -1427,6 +1427,8 @@ int FTN_STDCALL KMP_EXPAND_NAME(FTN_PAUSE_RESOURCE)(kmp_pause_status_t kind,
 #ifdef KMP_STUB
   return 1; // just fail
 #else
+  if (kind == kmp_stop_tool_paused)
+    return 1; // stop_tool must not be specified
   if (device_num == KMP_EXPAND_NAME(FTN_GET_INITIAL_DEVICE)())
     return __kmpc_pause_resource(kind);
   else {
diff --git a/openmp/runtime/src/kmp_runtime.cpp b/openmp/runtime/src/kmp_runtime.cpp
index 06bc4939359e264..2c8d9304c46bc25 100644
--- a/openmp/runtime/src/kmp_runtime.cpp
+++ b/openmp/runtime/src/kmp_runtime.cpp
@@ -9132,7 +9132,8 @@ int __kmp_pause_resource(kmp_pause_status_t level) {
       __kmp_soft_pause();
       return 0;
     }
-  } else if (level == kmp_hard_paused) { // requesting hard pause
+  } else if (level == kmp_hard_paused || level == kmp_stop_tool_paused) {
+    // requesting hard pause or stop_tool pause
     if (__kmp_pause_status != kmp_not_paused) {
       // error message about already being paused
       return 1;
diff --git a/openmp/runtime/test/ompt/misc/pause_stop_tool.c b/openmp/runtime/test/ompt/misc/pause_stop_tool.c
new file mode 100644
index 000000000000000..c93913793244582
--- /dev/null
+++ b/openmp/runtime/test/ompt/misc/pause_stop_tool.c
@@ -0,0 +1,44 @@
+// RUN: %libomp-compile-and-run | FileCheck %s
+// REQUIRES: ompt
+#include "callback.h"
+#include <omp.h>
+int main() {
+  int x = 0;
+  int ret = 0;
+#pragma omp parallel
+#pragma omp single
+  x++;
+  // Expected to fail; omp_pause_stop_tool must not be specified
+  ret = omp_pause_resource(omp_pause_stop_tool, omp_get_initial_device());
+  printf("omp_pause_resource %s\n", ret ? "failed" : "succeeded");
+#pragma omp parallel
+#pragma omp single
+  x++;
+  // Expected to succeed
+  ret = omp_pause_resource_all(omp_pause_stop_tool);
+  printf("omp_pause_resource_all %s\n", ret ? "failed" : "succeeded");
+#pragma omp parallel
+#pragma omp single
+  x++;
+  printf("x = %d\n", x);
+  return 0;
+
+  // Check if
+  // -- omp_pause_resource/resource_all returns expected code
+  // -- OMPT interface is shut down as expected
+
+  // CHECK-NOT: {{^}}0: Could not register callback
+  // CHECK: 0: NULL_POINTER=[[NULL:.*$]]
+
+  // CHECK: ompt_event_parallel_begin
+  // CHECK: ompt_event_parallel_end
+
+  // CHECK: omp_pause_resource failed
+
+  // CHECK: ompt_event_parallel_begin
+  // CHECK: ompt_event_parallel_end
+
+  // CHECK: omp_pause_resource_all succeeded
+
+  // CHECK-NOT: ompt_event
+}

From 6c270a8b9f1e1b80a6016aafb438db7dd89bcb99 Mon Sep 17 00:00:00 2001
From: Koakuma <koachan@protonmail.com>
Date: Thu, 15 Aug 2024 23:46:09 +0700
Subject: [PATCH 048/441] [SPARC][Driver] Add -m(no-)v8plus flags handling
 (#98713)

Implement handling for `-m(no-)v8plus` flags to allow the user to switch
between V8 and V8+ mode with 32-bit code.

Currently it only toggles the V8+ feature bit, ABI and codegen changes
will be done in future patches.
---
 clang/include/clang/Driver/Options.td      | 4 ++++
 clang/lib/Driver/ToolChains/Arch/Sparc.cpp | 5 +++++
 clang/test/Driver/sparc-target-features.c  | 3 +++
 3 files changed, 12 insertions(+)

diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index acc1f2fde53979a..73e19b65dededb7 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -6149,6 +6149,10 @@ def mvis3 : Flag<["-"], "mvis3">, Group<m_sparc_Features_Group>;
 def mno_vis3 : Flag<["-"], "mno-vis3">, Group<m_sparc_Features_Group>;
 def mhard_quad_float : Flag<["-"], "mhard-quad-float">, Group<m_sparc_Features_Group>;
 def msoft_quad_float : Flag<["-"], "msoft-quad-float">, Group<m_sparc_Features_Group>;
+def mv8plus : Flag<["-"], "mv8plus">, Group<m_sparc_Features_Group>,
+  HelpText<"Enable V8+ mode, allowing use of 64-bit V9 instructions in 32-bit code">;
+def mno_v8plus : Flag<["-"], "mno-v8plus">, Group<m_sparc_Features_Group>,
+  HelpText<"Disable V8+ mode">;
 foreach i = 1 ... 7 in
   def ffixed_g#i : Flag<["-"], "ffixed-g"#i>, Group<m_sparc_Features_Group>,
     HelpText<"Reserve the G"#i#" register (SPARC only)">;
diff --git a/clang/lib/Driver/ToolChains/Arch/Sparc.cpp b/clang/lib/Driver/ToolChains/Arch/Sparc.cpp
index ae1a4ba7882627f..5a1fedbec06adfa 100644
--- a/clang/lib/Driver/ToolChains/Arch/Sparc.cpp
+++ b/clang/lib/Driver/ToolChains/Arch/Sparc.cpp
@@ -179,6 +179,11 @@ void sparc::getSparcTargetFeatures(const Driver &D, const ArgList &Args,
       Features.push_back("-hard-quad-float");
   }
 
+  if (Arg *A = Args.getLastArg(options::OPT_mv8plus, options::OPT_mno_v8plus)) {
+    if (A->getOption().matches(options::OPT_mv8plus))
+      Features.push_back("+v8plus");
+  }
+
   if (Args.hasArg(options::OPT_ffixed_g1))
     Features.push_back("+reserve-g1");
 
diff --git a/clang/test/Driver/sparc-target-features.c b/clang/test/Driver/sparc-target-features.c
index b36f63e7660e830..a839604ff1bc022 100644
--- a/clang/test/Driver/sparc-target-features.c
+++ b/clang/test/Driver/sparc-target-features.c
@@ -32,3 +32,6 @@
 // RUN: %clang --target=sparc -msoft-quad-float %s -### 2>&1 | FileCheck -check-prefix=SOFT-QUAD-FLOAT %s
 // HARD-QUAD-FLOAT: "-target-feature" "+hard-quad-float"
 // SOFT-QUAD-FLOAT: "-target-feature" "-hard-quad-float"
+
+// RUN: %clang --target=sparc -mv8plus %s -### 2>&1 | FileCheck -check-prefix=V8PLUS %s
+// V8PLUS: "-target-feature" "+v8plus"

From e61776a0edce86ef01efaa708f43476c58173cae Mon Sep 17 00:00:00 2001
From: Koakuma <koachan@protonmail.com>
Date: Thu, 15 Aug 2024 23:49:01 +0700
Subject: [PATCH 049/441] [SPARC][Utilities] Add names for SPARC ELF flags in
 LLVM binary utilities (#102843)

This allows us to use and print readable names in LLVM binary utilities.
---
 llvm/include/llvm/BinaryFormat/ELF.h          |  8 ++
 llvm/lib/ObjectYAML/ELFYAML.cpp               | 14 ++++
 .../llvm-readobj/ELF/Sparc/elf-headers.test   | 78 +++++++++++++++++++
 llvm/tools/llvm-readobj/ELFDumper.cpp         | 16 ++++
 4 files changed, 116 insertions(+)
 create mode 100644 llvm/test/tools/llvm-readobj/ELF/Sparc/elf-headers.test

diff --git a/llvm/include/llvm/BinaryFormat/ELF.h b/llvm/include/llvm/BinaryFormat/ELF.h
index 6827b5d95abfe01..2c9f70872962732 100644
--- a/llvm/include/llvm/BinaryFormat/ELF.h
+++ b/llvm/include/llvm/BinaryFormat/ELF.h
@@ -699,11 +699,19 @@ enum {
 
 // SPARC Specific e_flags
 enum : unsigned {
+  // ELF extension mask.
+  // All values are available for EM_SPARC32PLUS & EM_SPARCV9 objects, except
+  // EF_SPARC_32PLUS which is a EM_SPARC32PLUS-only flag.
+  //
+  // Note that those features are not mutually exclusive (one can set more than
+  // one flag in this group).
   EF_SPARC_EXT_MASK = 0xffff00,
   EF_SPARC_32PLUS = 0x000100,
   EF_SPARC_SUN_US1 = 0x000200,
   EF_SPARC_HAL_R1 = 0x000400,
   EF_SPARC_SUN_US3 = 0x000800,
+
+  // Memory model selection mask for EM_SPARCV9 objects.
   EF_SPARCV9_MM = 0x3,
   EF_SPARCV9_TSO = 0x0,
   EF_SPARCV9_PSO = 0x1,
diff --git a/llvm/lib/ObjectYAML/ELFYAML.cpp b/llvm/lib/ObjectYAML/ELFYAML.cpp
index 1a192f72ae4d606..89ffc383a4a6ece 100644
--- a/llvm/lib/ObjectYAML/ELFYAML.cpp
+++ b/llvm/lib/ObjectYAML/ELFYAML.cpp
@@ -547,6 +547,20 @@ void ScalarBitSetTraits<ELFYAML::ELF_EF>::bitset(IO &IO,
     BCase(EF_RISCV_RVE);
     BCase(EF_RISCV_TSO);
     break;
+  case ELF::EM_SPARC32PLUS:
+    BCase(EF_SPARC_32PLUS);
+    BCase(EF_SPARC_SUN_US1);
+    BCase(EF_SPARC_SUN_US3);
+    BCase(EF_SPARC_HAL_R1);
+    break;
+  case ELF::EM_SPARCV9:
+    BCase(EF_SPARC_SUN_US1);
+    BCase(EF_SPARC_SUN_US3);
+    BCase(EF_SPARC_HAL_R1);
+    BCaseMask(EF_SPARCV9_RMO, EF_SPARCV9_MM);
+    BCaseMask(EF_SPARCV9_PSO, EF_SPARCV9_MM);
+    BCaseMask(EF_SPARCV9_TSO, EF_SPARCV9_MM);
+    break;
   case ELF::EM_XTENSA:
     BCase(EF_XTENSA_XT_INSN);
     BCaseMask(EF_XTENSA_MACH_NONE, EF_XTENSA_MACH);
diff --git a/llvm/test/tools/llvm-readobj/ELF/Sparc/elf-headers.test b/llvm/test/tools/llvm-readobj/ELF/Sparc/elf-headers.test
new file mode 100644
index 000000000000000..c05cd76640ed3eb
--- /dev/null
+++ b/llvm/test/tools/llvm-readobj/ELF/Sparc/elf-headers.test
@@ -0,0 +1,78 @@
+## Test various combinations of ELF flag values for SPARC.
+
+# RUN: yaml2obj %s -o %t -DCLASS_NAME="ELFCLASS32" -DMACHINE_NAME="EM_SPARC" -DFLAG_NAME=""
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=LLVM-COMMON -DFLAG_VALUE=0x0
+# RUN: llvm-readelf -h %t | FileCheck %s --match-full-lines --check-prefix=GNU \
+# RUN:   -DFLAG_VALUE=0x0 -DGNU_FLAG_NAME=""
+
+# RUN: yaml2obj %s -o %t -DCLASS_NAME="ELFCLASS32" -DMACHINE_NAME="EM_SPARC32PLUS" -DFLAG_NAME="EF_SPARC_32PLUS"
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=LLVM-COMMON,LLVM-FLAG0 \
+# RUN:   -DFLAG0_NAME=EF_SPARC_32PLUS -DFLAG_VALUE=0x100
+# RUN: llvm-readelf -h %t | FileCheck %s --match-full-lines --check-prefix=GNU \
+# RUN:   -DFLAG_VALUE=0x100 -DGNU_FLAG_NAME=", V8+ ABI"
+
+# RUN: yaml2obj %s -o %t -DCLASS_NAME="ELFCLASS32" -DMACHINE_NAME="EM_SPARC32PLUS" \
+# RUN:   -DFLAG_NAME="EF_SPARC_32PLUS, EF_SPARC_SUN_US1"
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=LLVM-COMMON,LLVM-FLAG0,LLVM-FLAG1\
+# RUN:   -DFLAG_VALUE=0x300 \
+# RUN:   -DFLAG0_NAME=EF_SPARC_32PLUS -DFLAG1_NAME=EF_SPARC_SUN_US1
+# RUN: llvm-readelf -h %t | FileCheck %s --match-full-lines --check-prefix=GNU \
+# RUN:   -DFLAG_VALUE=0x300 \
+# RUN:   -DGNU_FLAG_NAME=", V8+ ABI, Sun UltraSPARC I extensions"
+
+# RUN: yaml2obj %s -o %t -DCLASS_NAME="ELFCLASS32" -DMACHINE_NAME="EM_SPARC32PLUS" \
+# RUN:   -DFLAG_NAME="EF_SPARC_32PLUS, EF_SPARC_SUN_US1, EF_SPARC_HAL_R1, EF_SPARC_SUN_US3"
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=LLVM-COMMON,LLVM-FLAG0,LLVM-FLAG1,LLVM-FLAG2,LLVM-FLAG3 \
+# RUN:   -DFLAG_VALUE=0xF00 \
+# RUN:   -DFLAG0_NAME=EF_SPARC_32PLUS -DFLAG1_NAME=EF_SPARC_HAL_R1 \
+# RUN:   -DFLAG2_NAME=EF_SPARC_SUN_US1 -DFLAG3_NAME=EF_SPARC_SUN_US3
+# RUN: llvm-readelf -h %t | FileCheck %s --match-full-lines --check-prefix=GNU \
+# RUN:   -DFLAG_VALUE=0xF00 \
+# RUN:   -DGNU_FLAG_NAME=", V8+ ABI, Sun UltraSPARC I extensions, HAL/Fujitsu R1 extensions, Sun UltraSPARC III extensions"
+
+# RUN: yaml2obj %s -o %t -DCLASS_NAME="ELFCLASS64" -DMACHINE_NAME="EM_SPARCV9" -DFLAG_NAME=""
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=LLVM-COMMON -DFLAG_VALUE=0x0
+# RUN: llvm-readelf -h %t | FileCheck %s --match-full-lines --check-prefix=GNU \
+# RUN:   -DFLAG_VALUE=0x0 -DGNU_FLAG_NAME=""
+
+# RUN: yaml2obj %s -o %t -DCLASS_NAME="ELFCLASS64" -DMACHINE_NAME="EM_SPARCV9" -DFLAG_NAME="EF_SPARC_SUN_US1"
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=LLVM-COMMON,LLVM-FLAG0 \
+# RUN:   -DFLAG_VALUE=0x200 -DFLAG0_NAME=EF_SPARC_SUN_US1
+# RUN: llvm-readelf -h %t | FileCheck %s --match-full-lines --check-prefix=GNU \
+# RUN:   -DFLAG_VALUE=0x200 -DGNU_FLAG_NAME=", Sun UltraSPARC I extensions"
+
+# RUN: yaml2obj %s -o %t -DCLASS_NAME="ELFCLASS64" -DMACHINE_NAME="EM_SPARCV9" \
+# RUN:   -DFLAG_NAME="EF_SPARC_SUN_US1, EF_SPARCV9_RMO"
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=LLVM-COMMON,LLVM-FLAG0,LLVM-FLAG1 \
+# RUN:   -DFLAG_VALUE=0x202 \
+# RUN:   -DFLAG0_NAME=EF_SPARCV9_RMO -DFLAG1_NAME=EF_SPARC_SUN_US1
+# RUN: llvm-readelf -h %t | FileCheck %s --match-full-lines --check-prefix=GNU \
+# RUN:   -DFLAG_VALUE=0x202 \
+# RUN:   -DGNU_FLAG_NAME=", Sun UltraSPARC I extensions, Relaxed Memory Ordering"
+
+# RUN: yaml2obj %s -o %t -DCLASS_NAME="ELFCLASS64" -DMACHINE_NAME="EM_SPARCV9" \
+# RUN:   -DFLAG_NAME="EF_SPARC_SUN_US1, EF_SPARC_HAL_R1, EF_SPARC_SUN_US3, EF_SPARCV9_PSO"
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=LLVM-COMMON,LLVM-FLAG0,LLVM-FLAG1,LLVM-FLAG2,LLVM-FLAG3 \
+# RUN:   -DFLAG_VALUE=0xE01 \
+# RUN:   -DFLAG0_NAME=EF_SPARCV9_PSO -DFLAG1_NAME=EF_SPARC_HAL_R1 \
+# RUN:   -DFLAG2_NAME=EF_SPARC_SUN_US1 -DFLAG3_NAME=EF_SPARC_SUN_US3
+# RUN: llvm-readelf -h %t | FileCheck %s --match-full-lines --check-prefix=GNU \
+# RUN:   -DFLAG_VALUE=0xE01\
+# RUN:   -DGNU_FLAG_NAME=", Sun UltraSPARC I extensions, HAL/Fujitsu R1 extensions, Sun UltraSPARC III extensions, Partial Store Ordering"
+
+--- !ELF
+FileHeader:
+  Class:           [[CLASS_NAME]]
+  Data:            ELFDATA2MSB
+  Type:            ET_REL
+  Machine:         [[MACHINE_NAME]]
+  Flags:           [ [[FLAG_NAME]] ]
+
+# LLVM-COMMON:          Flags [ ([[FLAG_VALUE]])
+# LLVM-FLAG0-NEXT:       [[FLAG0_NAME]]
+# LLVM-FLAG1-NEXT:       [[FLAG1_NAME]]
+# LLVM-FLAG2-NEXT:       [[FLAG2_NAME]]
+# LLVM-FLAG3-NEXT:       [[FLAG3_NAME]]
+# LLVM-COMMON-NEXT:     ]
+
+# GNU: Flags: [[FLAG_VALUE]][[GNU_FLAG_NAME]]
diff --git a/llvm/tools/llvm-readobj/ELFDumper.cpp b/llvm/tools/llvm-readobj/ELFDumper.cpp
index 2b5e2f4a9d347e3..b4804246875c606 100644
--- a/llvm/tools/llvm-readobj/ELFDumper.cpp
+++ b/llvm/tools/llvm-readobj/ELFDumper.cpp
@@ -1684,6 +1684,16 @@ const EnumEntry<unsigned> ElfHeaderRISCVFlags[] = {
   ENUM_ENT(EF_RISCV_TSO, "TSO"),
 };
 
+const EnumEntry<unsigned> ElfHeaderSPARCFlags[] = {
+    ENUM_ENT(EF_SPARC_32PLUS, "V8+ ABI"),
+    ENUM_ENT(EF_SPARC_SUN_US1, "Sun UltraSPARC I extensions"),
+    ENUM_ENT(EF_SPARC_HAL_R1, "HAL/Fujitsu R1 extensions"),
+    ENUM_ENT(EF_SPARC_SUN_US3, "Sun UltraSPARC III extensions"),
+    ENUM_ENT(EF_SPARCV9_TSO, "Total Store Ordering"),
+    ENUM_ENT(EF_SPARCV9_PSO, "Partial Store Ordering"),
+    ENUM_ENT(EF_SPARCV9_RMO, "Relaxed Memory Ordering"),
+};
+
 const EnumEntry<unsigned> ElfHeaderAVRFlags[] = {
   LLVM_READOBJ_ENUM_ENT(ELF, EF_AVR_ARCH_AVR1),
   LLVM_READOBJ_ENUM_ENT(ELF, EF_AVR_ARCH_AVR2),
@@ -3625,6 +3635,9 @@ template <class ELFT> void GNUELFDumper<ELFT>::printFileHeaders() {
         unsigned(ELF::EF_MIPS_ABI), unsigned(ELF::EF_MIPS_MACH));
   else if (e.e_machine == EM_RISCV)
     ElfFlags = printFlags(e.e_flags, ArrayRef(ElfHeaderRISCVFlags));
+  else if (e.e_machine == EM_SPARC32PLUS || e.e_machine == EM_SPARCV9)
+    ElfFlags = printFlags(e.e_flags, ArrayRef(ElfHeaderSPARCFlags),
+                          unsigned(ELF::EF_SPARCV9_MM));
   else if (e.e_machine == EM_AVR)
     ElfFlags = printFlags(e.e_flags, ArrayRef(ElfHeaderAVRFlags),
                           unsigned(ELF::EF_AVR_ARCH_MASK));
@@ -7118,6 +7131,9 @@ template <class ELFT> void LLVMELFDumper<ELFT>::printFileHeaders() {
       }
     } else if (E.e_machine == EM_RISCV)
       W.printFlags("Flags", E.e_flags, ArrayRef(ElfHeaderRISCVFlags));
+    else if (E.e_machine == EM_SPARC32PLUS || E.e_machine == EM_SPARCV9)
+      W.printFlags("Flags", E.e_flags, ArrayRef(ElfHeaderSPARCFlags),
+                   unsigned(ELF::EF_SPARCV9_MM));
     else if (E.e_machine == EM_AVR)
       W.printFlags("Flags", E.e_flags, ArrayRef(ElfHeaderAVRFlags),
                    unsigned(ELF::EF_AVR_ARCH_MASK));

From 54154f9f06e08b7ab3c7294352601ca4c6e5e160 Mon Sep 17 00:00:00 2001
From: jeffreytan81 <jeffreytan@meta.com>
Date: Thu, 15 Aug 2024 09:57:01 -0700
Subject: [PATCH 050/441] Fix single thread stepping timeout race condition
 (#104195)

This PR fixes a potential race condition in
https://github.com/llvm/llvm-project/pull/90930.

This race can happen because the original code set `m_info->m_isAlive =
true` **after** the timer thread is created. So if there is a context
switch happens and timer thread checks `m_info->m_isAlive` before main
thread got a chance to run `m_info->m_isAlive = true`, the timer thread
may treat `ThreadPlanSingleThreadTimeout` as not alive and simply exit
resulting in async interrupt never being sent to resume all threads
(deadlock).

The PR fixes the race by initializing all states **before** worker timer
thread creates.

Co-authored-by: jeffreytan81 <jeffreytan@fb.com>
---
 lldb/source/Target/ThreadPlanSingleThreadTimeout.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lldb/source/Target/ThreadPlanSingleThreadTimeout.cpp b/lldb/source/Target/ThreadPlanSingleThreadTimeout.cpp
index 0a939d55f4ce493..806ba95c508b7ca 100644
--- a/lldb/source/Target/ThreadPlanSingleThreadTimeout.cpp
+++ b/lldb/source/Target/ThreadPlanSingleThreadTimeout.cpp
@@ -29,10 +29,10 @@ ThreadPlanSingleThreadTimeout::ThreadPlanSingleThreadTimeout(
     : ThreadPlan(ThreadPlan::eKindSingleThreadTimeout, "Single thread timeout",
                  thread, eVoteNo, eVoteNoOpinion),
       m_info(info), m_state(State::WaitTimeout) {
-  // TODO: reuse m_timer_thread without recreation.
-  m_timer_thread = std::thread(TimeoutThreadFunc, this);
   m_info->m_isAlive = true;
   m_state = m_info->m_last_state;
+  // TODO: reuse m_timer_thread without recreation.
+  m_timer_thread = std::thread(TimeoutThreadFunc, this);
 }
 
 ThreadPlanSingleThreadTimeout::~ThreadPlanSingleThreadTimeout() {

From dd3f1313ae27a76cfce68e926fd90ac7408b3a21 Mon Sep 17 00:00:00 2001
From: Keith Smiley <keithbsmiley@gmail.com>
Date: Thu, 15 Aug 2024 09:59:29 -0700
Subject: [PATCH 051/441] [bazel] Enable more lit self tests (#104285)

I assume the intent of the initial `*/*.py` was to also collect things
in `*.py`, but that's not what bazel does unless you use `**/*.py` which
is what we're doing now. A few of these tests fail so I explicitly
disabled them until someone has time to debug.
---
 .../llvm/utils/lit/tests/BUILD.bazel                  | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/utils/bazel/llvm-project-overlay/llvm/utils/lit/tests/BUILD.bazel b/utils/bazel/llvm-project-overlay/llvm/utils/lit/tests/BUILD.bazel
index 13f6f815d399504..d89626a6ee9e67c 100644
--- a/utils/bazel/llvm-project-overlay/llvm/utils/lit/tests/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/llvm/utils/lit/tests/BUILD.bazel
@@ -34,7 +34,14 @@ expand_template(
         ] + glob(["Inputs/**"]),
     )
     for src in glob(
-        ["*/*.py"],
-        exclude = ["Inputs/**"],
+        ["**/*.py"],
+        exclude = [
+            "Inputs/**",
+            "discovery.py",  # TODO: debug and re-enable
+            "max-time.py",
+            "selecting.py",
+            "shtest-recursive-substitution.py",
+            "use-llvm-tool.py",
+        ],
     )
 ]

From d68d2172f9f1f0659b8b4bdbbeb1ccd290a614b5 Mon Sep 17 00:00:00 2001
From: Volodymyr Vasylkun <vvmposeydon@gmail.com>
Date: Thu, 15 Aug 2024 18:08:23 +0100
Subject: [PATCH 052/441] [InstCombine] Fold `ucmp/scmp(x, y) >> N` to
 `zext/sext(x < y)` when N is one less than the width of the result of
 `ucmp/scmp` (#104009)

Proof: https://alive2.llvm.org/ce/z/4diUqN

---------

Co-authored-by: Nikita Popov <github@npopov.com>
---
 .../InstCombine/InstCombineShifts.cpp         |  15 +++
 .../InstCombine/lshr-ashr-of-uscmp.ll         | 107 ++++++++++++++++++
 2 files changed, 122 insertions(+)
 create mode 100644 llvm/test/Transforms/InstCombine/lshr-ashr-of-uscmp.ll

diff --git a/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp b/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp
index 38f8a41214b6825..794b384d126eb66 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp
@@ -511,6 +511,21 @@ Instruction *InstCombinerImpl::commonShiftTransforms(BinaryOperator &I) {
   if (match(Op1, m_Or(m_Value(), m_SpecificInt(BitWidth - 1))))
     return replaceOperand(I, 1, ConstantInt::get(Ty, BitWidth - 1));
 
+  Instruction *CmpIntr;
+  if ((I.getOpcode() == Instruction::LShr ||
+       I.getOpcode() == Instruction::AShr) &&
+      match(Op0, m_OneUse(m_Instruction(CmpIntr))) &&
+      isa<CmpIntrinsic>(CmpIntr) &&
+      match(Op1, m_SpecificInt(Ty->getScalarSizeInBits() - 1))) {
+    Value *Cmp =
+        Builder.CreateICmp(cast<CmpIntrinsic>(CmpIntr)->getLTPredicate(),
+                           CmpIntr->getOperand(0), CmpIntr->getOperand(1));
+    return CastInst::Create(I.getOpcode() == Instruction::LShr
+                                ? Instruction::ZExt
+                                : Instruction::SExt,
+                            Cmp, Ty);
+  }
+
   return nullptr;
 }
 
diff --git a/llvm/test/Transforms/InstCombine/lshr-ashr-of-uscmp.ll b/llvm/test/Transforms/InstCombine/lshr-ashr-of-uscmp.ll
new file mode 100644
index 000000000000000..93082de93f97a4c
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/lshr-ashr-of-uscmp.ll
@@ -0,0 +1,107 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -passes=instcombine -S | FileCheck %s
+
+declare void @use(i8 %val)
+
+; ucmp/scmp(x, y) >> N folds to either zext(x < y) or sext(x < y)
+; if N is one less than the width of result of ucmp/scmp
+define i8 @ucmp_to_zext(i32 %x, i32 %y) {
+; CHECK-LABEL: define i8 @ucmp_to_zext(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i1 [[TMP1]] to i8
+; CHECK-NEXT:    ret i8 [[TMP2]]
+;
+  %1 = call i8 @llvm.ucmp(i32 %x, i32 %y)
+  %2 = lshr i8 %1, 7
+  ret i8 %2
+}
+
+define i8 @ucmp_to_sext(i32 %x, i32 %y) {
+; CHECK-LABEL: define i8 @ucmp_to_sext(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[TMP2:%.*]] = sext i1 [[TMP1]] to i8
+; CHECK-NEXT:    ret i8 [[TMP2]]
+;
+  %1 = call i8 @llvm.ucmp(i32 %x, i32 %y)
+  %2 = ashr i8 %1, 7
+  ret i8 %2
+}
+
+define i8 @scmp_to_zext(i32 %x, i32 %y) {
+; CHECK-LABEL: define i8 @scmp_to_zext(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i1 [[TMP1]] to i8
+; CHECK-NEXT:    ret i8 [[TMP2]]
+;
+  %1 = call i8 @llvm.scmp(i32 %x, i32 %y)
+  %2 = lshr i8 %1, 7
+  ret i8 %2
+}
+
+define i8 @scmp_to_sext(i32 %x, i32 %y) {
+; CHECK-LABEL: define i8 @scmp_to_sext(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[TMP2:%.*]] = sext i1 [[TMP1]] to i8
+; CHECK-NEXT:    ret i8 [[TMP2]]
+;
+  %1 = call i8 @llvm.scmp(i32 %x, i32 %y)
+  %2 = ashr i8 %1, 7
+  ret i8 %2
+}
+
+define <4 x i8> @scmp_to_sext_vec(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: define <4 x i8> @scmp_to_sext_vec(
+; CHECK-SAME: <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt <4 x i32> [[X]], [[Y]]
+; CHECK-NEXT:    [[TMP2:%.*]] = sext <4 x i1> [[TMP1]] to <4 x i8>
+; CHECK-NEXT:    ret <4 x i8> [[TMP2]]
+;
+  %1 = call <4 x i8> @llvm.scmp(<4 x i32> %x, <4 x i32> %y)
+  %2 = ashr <4 x i8> %1, <i8 7, i8 7, i8 7, i8 7>
+  ret <4 x i8> %2
+}
+
+; Negative test: incorrect shift amount
+define i8 @ucmp_to_zext_neg1(i32 %x, i32 %y) {
+; CHECK-LABEL: define i8 @ucmp_to_zext_neg1(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @llvm.ucmp.i8.i32(i32 [[X]], i32 [[Y]])
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr i8 [[TMP1]], 5
+; CHECK-NEXT:    ret i8 [[TMP2]]
+;
+  %1 = call i8 @llvm.ucmp(i32 %x, i32 %y)
+  %2 = lshr i8 %1, 5
+  ret i8 %2
+}
+
+; Negative test: shift amount is not a constant
+define i8 @ucmp_to_zext_neg2(i32 %x, i32 %y, i8 %s) {
+; CHECK-LABEL: define i8 @ucmp_to_zext_neg2(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], i8 [[S:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @llvm.ucmp.i8.i32(i32 [[X]], i32 [[Y]])
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr i8 [[TMP1]], [[S]]
+; CHECK-NEXT:    ret i8 [[TMP2]]
+;
+  %1 = call i8 @llvm.ucmp(i32 %x, i32 %y)
+  %2 = lshr i8 %1, %s
+  ret i8 %2
+}
+
+; Negative test: the result of ucmp/scmp is used more than once
+define i8 @ucmp_to_zext_neg3(i32 %x, i32 %y) {
+; CHECK-LABEL: define i8 @ucmp_to_zext_neg3(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @llvm.ucmp.i8.i32(i32 [[X]], i32 [[Y]])
+; CHECK-NEXT:    call void @use(i8 [[TMP1]])
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr i8 [[TMP1]], 7
+; CHECK-NEXT:    ret i8 [[TMP2]]
+;
+  %1 = call i8 @llvm.ucmp(i32 %x, i32 %y)
+  call void @use(i8 %1)
+  %2 = lshr i8 %1, 7
+  ret i8 %2
+}

From 062e69a647c7ea0bc3441223648f9989490abb7a Mon Sep 17 00:00:00 2001
From: Leandro Lupori <leandro.lupori@linaro.org>
Date: Thu, 15 Aug 2024 14:14:18 -0300
Subject: [PATCH 053/441] [flang][OpenMP] Fix 2 more regressions after #101009
 (#101538)

PR #101009 exposed a semantic check issue with OPTIONAL dummy
arguments.
Another issue occurred when using %{re,im,len,kind}, as these also
need to be skipped when handling variables with implicitly defined
DSAs.

These issues were found by Fujitsu testsuite.
---
 flang/lib/Semantics/check-call.cpp         |  3 ++-
 flang/lib/Semantics/resolve-directives.cpp | 15 ++++++++-------
 flang/test/Semantics/OpenMP/complex.f90    | 13 +++++++++++++
 flang/test/Semantics/OpenMP/present.f90    |  9 +++++++++
 4 files changed, 32 insertions(+), 8 deletions(-)
 create mode 100644 flang/test/Semantics/OpenMP/complex.f90
 create mode 100644 flang/test/Semantics/OpenMP/present.f90

diff --git a/flang/lib/Semantics/check-call.cpp b/flang/lib/Semantics/check-call.cpp
index 9fad1aa3dd0bf97..4708d51d3af4dd9 100644
--- a/flang/lib/Semantics/check-call.cpp
+++ b/flang/lib/Semantics/check-call.cpp
@@ -1645,7 +1645,8 @@ static void CheckPresent(evaluate::ActualArguments &arguments,
       } else {
         symbol = arg->GetAssumedTypeDummy();
       }
-      if (!symbol || !symbol->attrs().test(semantics::Attr::OPTIONAL)) {
+      if (!symbol ||
+          !symbol->GetUltimate().attrs().test(semantics::Attr::OPTIONAL)) {
         messages.Say(arg ? arg->sourceLocation() : messages.at(),
             "Argument of PRESENT() must be the name of a whole OPTIONAL dummy argument"_err_en_US);
       }
diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index d635a7b8b7874fa..cc9f1cc7ed2691d 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -2036,20 +2036,21 @@ void OmpAttributeVisitor::Post(const parser::OpenMPAllocatorsConstruct &x) {
 void OmpAttributeVisitor::Post(const parser::Name &name) {
   auto *symbol{name.symbol};
   auto IsPrivatizable = [](const Symbol *sym) {
+    auto *misc{sym->detailsIf<MiscDetails>()};
     return !IsProcedure(*sym) && !IsNamedConstant(*sym) &&
         !sym->owner().IsDerivedType() &&
         sym->owner().kind() != Scope::Kind::ImpliedDos &&
         !sym->detailsIf<semantics::AssocEntityDetails>() &&
-        !sym->detailsIf<semantics::NamelistDetails>();
+        !sym->detailsIf<semantics::NamelistDetails>() &&
+        (!misc ||
+            (misc->kind() != MiscDetails::Kind::ComplexPartRe &&
+                misc->kind() != MiscDetails::Kind::ComplexPartIm &&
+                misc->kind() != MiscDetails::Kind::KindParamInquiry &&
+                misc->kind() != MiscDetails::Kind::LenParamInquiry &&
+                misc->kind() != MiscDetails::Kind::ConstructName));
   };
 
   if (symbol && !dirContext_.empty() && GetContext().withinConstruct) {
-    // Exclude construct-names
-    if (auto *details{symbol->detailsIf<semantics::MiscDetails>()}) {
-      if (details->kind() == semantics::MiscDetails::Kind::ConstructName) {
-        return;
-      }
-    }
     if (IsPrivatizable(symbol) && !IsObjectWithDSA(*symbol)) {
       // TODO: create a separate function to go through the rules for
       //       predetermined, explicitly determined, and implicitly
diff --git a/flang/test/Semantics/OpenMP/complex.f90 b/flang/test/Semantics/OpenMP/complex.f90
new file mode 100644
index 000000000000000..62336c7e6b31abd
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/complex.f90
@@ -0,0 +1,13 @@
+! RUN: %flang_fc1 -fopenmp -fsyntax-only %s
+
+! Check that using %re/%im inside 'parallel' doesn't cause syntax errors.
+subroutine test_complex_re_im
+  complex :: cc(4) = (1,2)
+  integer :: i
+
+  !$omp parallel do private(cc)
+    do i = 1, 4
+      print *, cc(i)%re, cc(i)%im
+    end do
+  !$omp end parallel do
+end subroutine
diff --git a/flang/test/Semantics/OpenMP/present.f90 b/flang/test/Semantics/OpenMP/present.f90
new file mode 100644
index 000000000000000..31bdcc7a8c654ee
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/present.f90
@@ -0,0 +1,9 @@
+! RUN: %flang_fc1 -fopenmp -fsyntax-only %s
+
+! Check that using 'present' inside 'parallel' doesn't cause syntax errors.
+subroutine test_present(opt)
+  integer, optional :: opt
+  !$omp parallel
+    if (present(opt)) print *, "present"
+  !$omp end parallel
+end subroutine

From e398da2b37fcc2696e1f5c661e5372844f4e1550 Mon Sep 17 00:00:00 2001
From: Thurston Dang <thurston@google.com>
Date: Thu, 15 Aug 2024 10:11:29 -0700
Subject: [PATCH 054/441] Revert "[Clang] Overflow Pattern Exclusions
 (#100272)"

This reverts commit 9a666deecb9ff6ca3a6b12e6c2877e19b74b54da.

Reason: broke buildbots

e.g., fork-ubsan.test started failing at
https://lab.llvm.org/buildbot/#/builders/66/builds/2819/steps/9/logs/stdio

  Clang :: CodeGen/compound-assign-overflow.c
  Clang :: CodeGen/sanitize-atomic-int-overflow.c
started failing with https://lab.llvm.org/buildbot/#/builders/52/builds/1570
---
 clang/docs/ReleaseNotes.rst                   | 30 -------
 clang/docs/UndefinedBehaviorSanitizer.rst     | 42 ----------
 clang/include/clang/AST/Expr.h                |  9 --
 clang/include/clang/AST/Stmt.h                |  5 --
 clang/include/clang/Basic/LangOptions.def     |  2 -
 clang/include/clang/Basic/LangOptions.h       | 28 -------
 clang/include/clang/Driver/Options.td         |  5 --
 clang/include/clang/Driver/SanitizerArgs.h    |  1 -
 clang/lib/AST/Expr.cpp                        | 54 ------------
 clang/lib/CodeGen/CGExprScalar.cpp            | 41 +--------
 clang/lib/Driver/SanitizerArgs.cpp            | 37 ---------
 clang/lib/Driver/ToolChains/Clang.cpp         |  3 -
 clang/lib/Frontend/CompilerInvocation.cpp     | 13 ---
 clang/lib/Serialization/ASTReaderStmt.cpp     |  1 -
 clang/lib/Serialization/ASTWriterStmt.cpp     |  1 -
 .../CodeGen/overflow-idiom-exclusion-fp.c     | 83 -------------------
 16 files changed, 2 insertions(+), 353 deletions(-)
 delete mode 100644 clang/test/CodeGen/overflow-idiom-exclusion-fp.c

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index b6b7dd5705637a3..4872dbb7a556ad2 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -395,36 +395,6 @@ Moved checkers
 Sanitizers
 ----------
 
-- Added the ``-fsanitize-overflow-pattern-exclusion=`` flag which can be used
-  to disable specific overflow-dependent code patterns. The supported patterns
-  are: ``add-overflow-test``, ``negated-unsigned-const``, and
-  ``post-decr-while``. The sanitizer instrumentation can be toggled off for all
-  available patterns by specifying ``all``. Conversely, you can disable all
-  exclusions with ``none``.
-
-  .. code-block:: c++
-
-     /// specified with ``-fsanitize-overflow-pattern-exclusion=add-overflow-test``
-     int common_overflow_check_pattern(unsigned base, unsigned offset) {
-       if (base + offset < base) { /* ... */ } // The pattern of `a + b < a`, and other re-orderings, won't be instrumented
-     }
-
-     /// specified with ``-fsanitize-overflow-pattern-exclusion=negated-unsigned-const``
-     void negation_overflow() {
-       unsigned long foo = -1UL; // No longer causes a negation overflow warning
-       unsigned long bar = -2UL; // and so on...
-     }
-
-     /// specified with ``-fsanitize-overflow-pattern-exclusion=post-decr-while``
-     void while_post_decrement() {
-       unsigned char count = 16;
-       while (count--) { /* ... */} // No longer causes unsigned-integer-overflow sanitizer to trip
-     }
-
-  Many existing projects have a large amount of these code patterns present.
-  This new flag should allow those projects to enable integer sanitizers with
-  less noise.
-
 Python Binding Changes
 ----------------------
 - Fixed an issue that led to crashes when calling ``Type.get_exception_specification_kind``.
diff --git a/clang/docs/UndefinedBehaviorSanitizer.rst b/clang/docs/UndefinedBehaviorSanitizer.rst
index 9f3d980eefbea77..531d56e313826c7 100644
--- a/clang/docs/UndefinedBehaviorSanitizer.rst
+++ b/clang/docs/UndefinedBehaviorSanitizer.rst
@@ -293,48 +293,6 @@ To silence reports from unsigned integer overflow, you can set
 ``-fsanitize-recover=unsigned-integer-overflow``, is particularly useful for
 providing fuzzing signal without blowing up logs.
 
-Disabling instrumentation for common overflow patterns
-------------------------------------------------------
-
-There are certain overflow-dependent or overflow-prone code patterns which
-produce a lot of noise for integer overflow/truncation sanitizers. Negated
-unsigned constants, post-decrements in a while loop condition and simple
-overflow checks are accepted and pervasive code patterns. However, the signal
-received from sanitizers instrumenting these code patterns may be too noisy for
-some projects. To disable instrumentation for these common patterns one should
-use ``-fsanitize-overflow-pattern-exclusion=``.
-
-Currently, this option supports three overflow-dependent code idioms:
-
-``negated-unsigned-const``
-
-.. code-block:: c++
-
-    /// -fsanitize-overflow-pattern-exclusion=negated-unsigned-const
-    unsigned long foo = -1UL; // No longer causes a negation overflow warning
-    unsigned long bar = -2UL; // and so on...
-
-``post-decr-while``
-
-.. code-block:: c++
-
-    /// -fsanitize-overflow-pattern-exclusion=post-decr-while
-    unsigned char count = 16;
-    while (count--) { /* ... */ } // No longer causes unsigned-integer-overflow sanitizer to trip
-
-``add-overflow-test``
-
-.. code-block:: c++
-
-    /// -fsanitize-overflow-pattern-exclusion=add-overflow-test
-    if (base + offset < base) { /* ... */ } // The pattern of `a + b < a`, and other re-orderings,
-                                            // won't be instrumented (same for signed types)
-
-You can enable all exclusions with
-``-fsanitize-overflow-pattern-exclusion=all`` or disable all exclusions with
-``-fsanitize-overflow-pattern-exclusion=none``. Specifying ``none`` has
-precedence over other values.
-
 Issue Suppression
 =================
 
diff --git a/clang/include/clang/AST/Expr.h b/clang/include/clang/AST/Expr.h
index f5863524723a2ec..5b813bfc2faf908 100644
--- a/clang/include/clang/AST/Expr.h
+++ b/clang/include/clang/AST/Expr.h
@@ -4043,15 +4043,6 @@ class BinaryOperator : public Expr {
   void setHasStoredFPFeatures(bool B) { BinaryOperatorBits.HasFPFeatures = B; }
   bool hasStoredFPFeatures() const { return BinaryOperatorBits.HasFPFeatures; }
 
-  /// Set and get the bit that informs arithmetic overflow sanitizers whether
-  /// or not they should exclude certain BinaryOperators from instrumentation
-  void setExcludedOverflowPattern(bool B) {
-    BinaryOperatorBits.ExcludedOverflowPattern = B;
-  }
-  bool hasExcludedOverflowPattern() const {
-    return BinaryOperatorBits.ExcludedOverflowPattern;
-  }
-
   /// Get FPFeatures from trailing storage
   FPOptionsOverride getStoredFPFeatures() const {
     assert(hasStoredFPFeatures());
diff --git a/clang/include/clang/AST/Stmt.h b/clang/include/clang/AST/Stmt.h
index f1a2aac0a8b2f8f..bbd7634bcc3bfb3 100644
--- a/clang/include/clang/AST/Stmt.h
+++ b/clang/include/clang/AST/Stmt.h
@@ -650,11 +650,6 @@ class alignas(void *) Stmt {
     LLVM_PREFERRED_TYPE(bool)
     unsigned HasFPFeatures : 1;
 
-    /// Whether or not this BinaryOperator should be excluded from integer
-    /// overflow sanitization.
-    LLVM_PREFERRED_TYPE(bool)
-    unsigned ExcludedOverflowPattern : 1;
-
     SourceLocation OpLoc;
   };
 
diff --git a/clang/include/clang/Basic/LangOptions.def b/clang/include/clang/Basic/LangOptions.def
index 2e9f2c552aad8a2..d454a7ff2f8cf49 100644
--- a/clang/include/clang/Basic/LangOptions.def
+++ b/clang/include/clang/Basic/LangOptions.def
@@ -406,8 +406,6 @@ VALUE_LANGOPT(TrivialAutoVarInitMaxSize, 32, 0,
              "stop trivial automatic variable initialization if var size exceeds the specified size (in bytes). Must be greater than 0.")
 ENUM_LANGOPT(SignedOverflowBehavior, SignedOverflowBehaviorTy, 2, SOB_Undefined,
              "signed integer overflow handling")
-LANGOPT(IgnoreNegationOverflow, 1, 0, "ignore overflow caused by negation")
-LANGOPT(SanitizeOverflowIdioms, 1, 1, "enable instrumentation for common overflow idioms")
 ENUM_LANGOPT(ThreadModel  , ThreadModelKind, 2, ThreadModelKind::POSIX, "Thread Model")
 
 BENIGN_LANGOPT(ArrowDepth, 32, 256,
diff --git a/clang/include/clang/Basic/LangOptions.h b/clang/include/clang/Basic/LangOptions.h
index eb4cb4b5a7e93fa..91f1c2f2e6239ee 100644
--- a/clang/include/clang/Basic/LangOptions.h
+++ b/clang/include/clang/Basic/LangOptions.h
@@ -367,21 +367,6 @@ class LangOptionsBase {
     PerThread,
   };
 
-  /// Exclude certain code patterns from being instrumented by arithmetic
-  /// overflow sanitizers
-  enum OverflowPatternExclusionKind {
-    /// Don't exclude any overflow patterns from sanitizers
-    None = 1 << 0,
-    /// Exclude all overflow patterns (below)
-    All = 1 << 1,
-    /// if (a + b < a)
-    AddOverflowTest = 1 << 2,
-    /// -1UL
-    NegUnsignedConst = 1 << 3,
-    /// while (count--)
-    PostDecrInWhile = 1 << 4,
-  };
-
   enum class DefaultVisiblityExportMapping {
     None,
     /// map only explicit default visibilities to exported
@@ -570,11 +555,6 @@ class LangOptions : public LangOptionsBase {
   /// The default stream kind used for HIP kernel launching.
   GPUDefaultStreamKind GPUDefaultStream;
 
-  /// Which overflow patterns should be excluded from sanitizer instrumentation
-  unsigned OverflowPatternExclusionMask = 0;
-
-  std::vector<std::string> OverflowPatternExclusionValues;
-
   /// The seed used by the randomize structure layout feature.
   std::string RandstructSeed;
 
@@ -650,14 +630,6 @@ class LangOptions : public LangOptionsBase {
     return MSCompatibilityVersion >= MajorVersion * 100000U;
   }
 
-  bool isOverflowPatternExcluded(OverflowPatternExclusionKind Kind) const {
-    if (OverflowPatternExclusionMask & OverflowPatternExclusionKind::None)
-      return false;
-    if (OverflowPatternExclusionMask & OverflowPatternExclusionKind::All)
-      return true;
-    return OverflowPatternExclusionMask & Kind;
-  }
-
   /// Reset all of the options that are not considered when building a
   /// module.
   void resetNonModularOptions();
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 73e19b65dededb7..40df91dc3fe0e3d 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -2565,11 +2565,6 @@ defm sanitize_stats : BoolOption<"f", "sanitize-stats",
           "Disable">,
   BothFlags<[], [ClangOption], " sanitizer statistics gathering.">>,
   Group<f_clang_Group>;
-def fsanitize_overflow_pattern_exclusion_EQ : CommaJoined<["-"], "fsanitize-overflow-pattern-exclusion=">,
-  HelpText<"Specify the overflow patterns to exclude from artihmetic sanitizer instrumentation">,
-  Visibility<[ClangOption, CC1Option]>,
-  Values<"none,all,add-overflow-test,negated-unsigned-const,post-decr-while">,
-  MarshallingInfoStringVector<LangOpts<"OverflowPatternExclusionValues">>;
 def fsanitize_thread_memory_access : Flag<["-"], "fsanitize-thread-memory-access">,
                                      Group<f_clang_Group>,
                                      HelpText<"Enable memory access instrumentation in ThreadSanitizer (default)">;
diff --git a/clang/include/clang/Driver/SanitizerArgs.h b/clang/include/clang/Driver/SanitizerArgs.h
index e64ec463ca89070..47ef175302679f8 100644
--- a/clang/include/clang/Driver/SanitizerArgs.h
+++ b/clang/include/clang/Driver/SanitizerArgs.h
@@ -33,7 +33,6 @@ class SanitizerArgs {
   std::vector<std::string> BinaryMetadataIgnorelistFiles;
   int CoverageFeatures = 0;
   int BinaryMetadataFeatures = 0;
-  int OverflowPatternExclusions = 0;
   int MsanTrackOrigins = 0;
   bool MsanUseAfterDtor = true;
   bool MsanParamRetval = true;
diff --git a/clang/lib/AST/Expr.cpp b/clang/lib/AST/Expr.cpp
index 57475c66a94e356..9d5b8167d0ee628 100644
--- a/clang/lib/AST/Expr.cpp
+++ b/clang/lib/AST/Expr.cpp
@@ -4759,53 +4759,6 @@ ParenListExpr *ParenListExpr::CreateEmpty(const ASTContext &Ctx,
   return new (Mem) ParenListExpr(EmptyShell(), NumExprs);
 }
 
-/// Certain overflow-dependent code patterns can have their integer overflow
-/// sanitization disabled. Check for the common pattern `if (a + b < a)` and
-/// return the resulting BinaryOperator responsible for the addition so we can
-/// elide overflow checks during codegen.
-static std::optional<BinaryOperator *>
-getOverflowPatternBinOp(const BinaryOperator *E) {
-  Expr *Addition, *ComparedTo;
-  if (E->getOpcode() == BO_LT) {
-    Addition = E->getLHS();
-    ComparedTo = E->getRHS();
-  } else if (E->getOpcode() == BO_GT) {
-    Addition = E->getRHS();
-    ComparedTo = E->getLHS();
-  } else {
-    return {};
-  }
-
-  const Expr *AddLHS = nullptr, *AddRHS = nullptr;
-  BinaryOperator *BO = dyn_cast<BinaryOperator>(Addition);
-
-  if (BO && BO->getOpcode() == clang::BO_Add) {
-    // now store addends for lookup on other side of '>'
-    AddLHS = BO->getLHS();
-    AddRHS = BO->getRHS();
-  }
-
-  if (!AddLHS || !AddRHS)
-    return {};
-
-  const Decl *LHSDecl, *RHSDecl, *OtherDecl;
-
-  LHSDecl = AddLHS->IgnoreParenImpCasts()->getReferencedDeclOfCallee();
-  RHSDecl = AddRHS->IgnoreParenImpCasts()->getReferencedDeclOfCallee();
-  OtherDecl = ComparedTo->IgnoreParenImpCasts()->getReferencedDeclOfCallee();
-
-  if (!OtherDecl)
-    return {};
-
-  if (!LHSDecl && !RHSDecl)
-    return {};
-
-  if ((LHSDecl && LHSDecl == OtherDecl && LHSDecl != RHSDecl) ||
-      (RHSDecl && RHSDecl == OtherDecl && RHSDecl != LHSDecl))
-    return BO;
-  return {};
-}
-
 BinaryOperator::BinaryOperator(const ASTContext &Ctx, Expr *lhs, Expr *rhs,
                                Opcode opc, QualType ResTy, ExprValueKind VK,
                                ExprObjectKind OK, SourceLocation opLoc,
@@ -4815,15 +4768,8 @@ BinaryOperator::BinaryOperator(const ASTContext &Ctx, Expr *lhs, Expr *rhs,
   assert(!isCompoundAssignmentOp() &&
          "Use CompoundAssignOperator for compound assignments");
   BinaryOperatorBits.OpLoc = opLoc;
-  BinaryOperatorBits.ExcludedOverflowPattern = 0;
   SubExprs[LHS] = lhs;
   SubExprs[RHS] = rhs;
-  if (Ctx.getLangOpts().isOverflowPatternExcluded(
-          LangOptions::OverflowPatternExclusionKind::AddOverflowTest)) {
-    std::optional<BinaryOperator *> Result = getOverflowPatternBinOp(this);
-    if (Result.has_value())
-      Result.value()->BinaryOperatorBits.ExcludedOverflowPattern = 1;
-  }
   BinaryOperatorBits.HasFPFeatures = FPFeatures.requiresTrailingStorage();
   if (hasStoredFPFeatures())
     setStoredFPFeatures(FPFeatures);
diff --git a/clang/lib/CodeGen/CGExprScalar.cpp b/clang/lib/CodeGen/CGExprScalar.cpp
index 6eac2b4c54e1ba3..84392745ea6144d 100644
--- a/clang/lib/CodeGen/CGExprScalar.cpp
+++ b/clang/lib/CodeGen/CGExprScalar.cpp
@@ -24,7 +24,6 @@
 #include "clang/AST/Attr.h"
 #include "clang/AST/DeclObjC.h"
 #include "clang/AST/Expr.h"
-#include "clang/AST/ParentMapContext.h"
 #include "clang/AST/RecordLayout.h"
 #include "clang/AST/StmtVisitor.h"
 #include "clang/Basic/CodeGenOptions.h"
@@ -196,24 +195,13 @@ static bool CanElideOverflowCheck(const ASTContext &Ctx, const BinOpInfo &Op) {
   if (!Op.mayHaveIntegerOverflow())
     return true;
 
-  const UnaryOperator *UO = dyn_cast<UnaryOperator>(Op.E);
-
-  if (UO && UO->getOpcode() == UO_Minus &&
-      Ctx.getLangOpts().isOverflowPatternExcluded(
-          LangOptions::OverflowPatternExclusionKind::NegUnsignedConst) &&
-      UO->isIntegerConstantExpr(Ctx))
-    return true;
-
   // If a unary op has a widened operand, the op cannot overflow.
-  if (UO)
+  if (const auto *UO = dyn_cast<UnaryOperator>(Op.E))
     return !UO->canOverflow();
 
   // We usually don't need overflow checks for binops with widened operands.
   // Multiplication with promoted unsigned operands is a special case.
   const auto *BO = cast<BinaryOperator>(Op.E);
-  if (BO->hasExcludedOverflowPattern())
-    return true;
-
   auto OptionalLHSTy = getUnwidenedIntegerType(Ctx, BO->getLHS());
   if (!OptionalLHSTy)
     return false;
@@ -2778,26 +2766,6 @@ llvm::Value *ScalarExprEmitter::EmitIncDecConsiderOverflowBehavior(
   llvm_unreachable("Unknown SignedOverflowBehaviorTy");
 }
 
-/// For the purposes of overflow pattern exclusion, does this match the
-/// "while(i--)" pattern?
-static bool matchesPostDecrInWhile(const UnaryOperator *UO, bool isInc,
-                                   bool isPre, ASTContext &Ctx) {
-  if (isInc || isPre)
-    return false;
-
-  // -fsanitize-overflow-pattern-exclusion=post-decr-while
-  if (!Ctx.getLangOpts().isOverflowPatternExcluded(
-          LangOptions::OverflowPatternExclusionKind::PostDecrInWhile))
-    return false;
-
-  // all Parents (usually just one) must be a WhileStmt
-  for (const auto &Parent : Ctx.getParentMapContext().getParents(*UO))
-    if (!Parent.get<WhileStmt>())
-      return false;
-
-  return true;
-}
-
 namespace {
 /// Handles check and update for lastprivate conditional variables.
 class OMPLastprivateConditionalUpdateRAII {
@@ -2909,10 +2877,6 @@ ScalarExprEmitter::EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV,
   } else if (type->isIntegerType()) {
     QualType promotedType;
     bool canPerformLossyDemotionCheck = false;
-
-    bool excludeOverflowPattern =
-        matchesPostDecrInWhile(E, isInc, isPre, CGF.getContext());
-
     if (CGF.getContext().isPromotableIntegerType(type)) {
       promotedType = CGF.getContext().getPromotedIntegerType(type);
       assert(promotedType != type && "Shouldn't promote to the same type.");
@@ -2972,8 +2936,7 @@ ScalarExprEmitter::EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV,
     } else if (E->canOverflow() && type->isSignedIntegerOrEnumerationType()) {
       value = EmitIncDecConsiderOverflowBehavior(E, value, isInc);
     } else if (E->canOverflow() && type->isUnsignedIntegerType() &&
-               CGF.SanOpts.has(SanitizerKind::UnsignedIntegerOverflow) &&
-               !excludeOverflowPattern) {
+               CGF.SanOpts.has(SanitizerKind::UnsignedIntegerOverflow)) {
       value = EmitOverflowCheckedBinOp(createBinOpInfoFromIncDec(
           E, value, isInc, E->getFPFeaturesInEffect(CGF.getLangOpts())));
     } else {
diff --git a/clang/lib/Driver/SanitizerArgs.cpp b/clang/lib/Driver/SanitizerArgs.cpp
index a63ee944fd1bb41..1fd870b72286e55 100644
--- a/clang/lib/Driver/SanitizerArgs.cpp
+++ b/clang/lib/Driver/SanitizerArgs.cpp
@@ -119,10 +119,6 @@ static SanitizerMask parseArgValues(const Driver &D, const llvm::opt::Arg *A,
 static int parseCoverageFeatures(const Driver &D, const llvm::opt::Arg *A,
                                  bool DiagnoseErrors);
 
-static int parseOverflowPatternExclusionValues(const Driver &D,
-                                               const llvm::opt::Arg *A,
-                                               bool DiagnoseErrors);
-
 /// Parse -f(no-)?sanitize-metadata= flag values, diagnosing any invalid
 /// components. Returns OR of members of \c BinaryMetadataFeature enumeration.
 static int parseBinaryMetadataFeatures(const Driver &D, const llvm::opt::Arg *A,
@@ -792,13 +788,6 @@ SanitizerArgs::SanitizerArgs(const ToolChain &TC,
           << "fsanitize-trap=cfi";
   }
 
-  for (const auto *Arg :
-       Args.filtered(options::OPT_fsanitize_overflow_pattern_exclusion_EQ)) {
-    Arg->claim();
-    OverflowPatternExclusions |=
-        parseOverflowPatternExclusionValues(D, Arg, DiagnoseErrors);
-  }
-
   // Parse -f(no-)?sanitize-coverage flags if coverage is supported by the
   // enabled sanitizers.
   for (const auto *Arg : Args) {
@@ -1252,10 +1241,6 @@ void SanitizerArgs::addArgs(const ToolChain &TC, const llvm::opt::ArgList &Args,
   addSpecialCaseListOpt(Args, CmdArgs,
                         "-fsanitize-system-ignorelist=", SystemIgnorelistFiles);
 
-  if (OverflowPatternExclusions)
-    Args.AddAllArgs(CmdArgs,
-                    options::OPT_fsanitize_overflow_pattern_exclusion_EQ);
-
   if (MsanTrackOrigins)
     CmdArgs.push_back(Args.MakeArgString("-fsanitize-memory-track-origins=" +
                                          Twine(MsanTrackOrigins)));
@@ -1441,28 +1426,6 @@ SanitizerMask parseArgValues(const Driver &D, const llvm::opt::Arg *A,
   return Kinds;
 }
 
-static int parseOverflowPatternExclusionValues(const Driver &D,
-                                               const llvm::opt::Arg *A,
-                                               bool DiagnoseErrors) {
-  int Exclusions = 0;
-  for (int i = 0, n = A->getNumValues(); i != n; ++i) {
-    const char *Value = A->getValue(i);
-    int E =
-        llvm::StringSwitch<int>(Value)
-            .Case("none", LangOptionsBase::None)
-            .Case("all", LangOptionsBase::All)
-            .Case("add-overflow-test", LangOptionsBase::AddOverflowTest)
-            .Case("negated-unsigned-const", LangOptionsBase::NegUnsignedConst)
-            .Case("post-decr-while", LangOptionsBase::PostDecrInWhile)
-            .Default(0);
-    if (E == 0)
-      D.Diag(clang::diag::err_drv_unsupported_option_argument)
-          << A->getSpelling() << Value;
-    Exclusions |= E;
-  }
-  return Exclusions;
-}
-
 int parseCoverageFeatures(const Driver &D, const llvm::opt::Arg *A,
                           bool DiagnoseErrors) {
   assert(A->getOption().matches(options::OPT_fsanitize_coverage) ||
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index f2bc11839edd4d0..96aa930ea286129 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -7769,9 +7769,6 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
     Args.AddLastArg(CmdArgs, options::OPT_fgpu_default_stream_EQ);
   }
 
-  Args.AddAllArgs(CmdArgs,
-                  options::OPT_fsanitize_overflow_pattern_exclusion_EQ);
-
   Args.AddLastArg(CmdArgs, options::OPT_foffload_uniform_block,
                   options::OPT_fno_offload_uniform_block);
 
diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp
index 5a5f5cb79a12f2b..e3911c281985b72 100644
--- a/clang/lib/Frontend/CompilerInvocation.cpp
+++ b/clang/lib/Frontend/CompilerInvocation.cpp
@@ -4267,19 +4267,6 @@ bool CompilerInvocation::ParseLangArgs(LangOptions &Opts, ArgList &Args,
       Diags.Report(diag::err_drv_invalid_value) << A->getAsString(Args) << Val;
   }
 
-  if (auto *A = Args.getLastArg(OPT_fsanitize_overflow_pattern_exclusion_EQ)) {
-    for (int i = 0, n = A->getNumValues(); i != n; ++i) {
-      Opts.OverflowPatternExclusionMask |=
-          llvm::StringSwitch<unsigned>(A->getValue(i))
-              .Case("none", LangOptionsBase::None)
-              .Case("all", LangOptionsBase::All)
-              .Case("add-overflow-test", LangOptionsBase::AddOverflowTest)
-              .Case("negated-unsigned-const", LangOptionsBase::NegUnsignedConst)
-              .Case("post-decr-while", LangOptionsBase::PostDecrInWhile)
-              .Default(0);
-    }
-  }
-
   // Parse -fsanitize= arguments.
   parseSanitizerKinds("-fsanitize=", Args.getAllArgValues(OPT_fsanitize_EQ),
                       Diags, Opts.Sanitize);
diff --git a/clang/lib/Serialization/ASTReaderStmt.cpp b/clang/lib/Serialization/ASTReaderStmt.cpp
index 8ae07907a04abaa..a33f2a41a654974 100644
--- a/clang/lib/Serialization/ASTReaderStmt.cpp
+++ b/clang/lib/Serialization/ASTReaderStmt.cpp
@@ -1128,7 +1128,6 @@ void ASTStmtReader::VisitBinaryOperator(BinaryOperator *E) {
       (BinaryOperator::Opcode)CurrentUnpackingBits->getNextBits(/*Width=*/6));
   bool hasFP_Features = CurrentUnpackingBits->getNextBit();
   E->setHasStoredFPFeatures(hasFP_Features);
-  E->setExcludedOverflowPattern(CurrentUnpackingBits->getNextBit());
   E->setLHS(Record.readSubExpr());
   E->setRHS(Record.readSubExpr());
   E->setOperatorLoc(readSourceLocation());
diff --git a/clang/lib/Serialization/ASTWriterStmt.cpp b/clang/lib/Serialization/ASTWriterStmt.cpp
index c292d0a789c7cd9..038616a675b7271 100644
--- a/clang/lib/Serialization/ASTWriterStmt.cpp
+++ b/clang/lib/Serialization/ASTWriterStmt.cpp
@@ -1063,7 +1063,6 @@ void ASTStmtWriter::VisitBinaryOperator(BinaryOperator *E) {
   CurrentPackingBits.addBits(E->getOpcode(), /*Width=*/6);
   bool HasFPFeatures = E->hasStoredFPFeatures();
   CurrentPackingBits.addBit(HasFPFeatures);
-  CurrentPackingBits.addBit(E->hasExcludedOverflowPattern());
   Record.AddStmt(E->getLHS());
   Record.AddStmt(E->getRHS());
   Record.AddSourceLocation(E->getOperatorLoc());
diff --git a/clang/test/CodeGen/overflow-idiom-exclusion-fp.c b/clang/test/CodeGen/overflow-idiom-exclusion-fp.c
deleted file mode 100644
index 511a88cc7a28368..000000000000000
--- a/clang/test/CodeGen/overflow-idiom-exclusion-fp.c
+++ /dev/null
@@ -1,83 +0,0 @@
-// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fsanitize=signed-integer-overflow,unsigned-integer-overflow -fsanitize-overflow-pattern-exclusion=all -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fsanitize=signed-integer-overflow,unsigned-integer-overflow -fsanitize-overflow-pattern-exclusion=all -fwrapv -emit-llvm -o - %s | FileCheck %s
-
-// Check for potential false positives from patterns that _almost_ match classic overflow-dependent or overflow-prone code patterns
-extern unsigned a, b, c;
-extern int u, v, w;
-
-extern unsigned some(void);
-
-// Make sure all these still have handler paths, we shouldn't be excluding
-// instrumentation of any "near" patterns.
-// CHECK-LABEL: close_but_not_quite
-void close_but_not_quite(void) {
-  // CHECK: br i1{{.*}}handler.
-  if (a + b > a)
-    c = 9;
-
-  // CHECK: br i1{{.*}}handler.
-  if (a - b < a)
-    c = 9;
-
-  // CHECK: br i1{{.*}}handler.
-  if (a + b < a)
-    c = 9;
-
-  // CHECK: br i1{{.*}}handler.
-  if (a + b + 1 < a)
-    c = 9;
-
-  // CHECK: br i1{{.*}}handler.
-  // CHECK: br i1{{.*}}handler.
-  if (a + b < a + 1)
-    c = 9;
-
-  // CHECK: br i1{{.*}}handler.
-  if (b >= a + b)
-    c = 9;
-
-  // CHECK: br i1{{.*}}handler.
-  if (a + a < a)
-    c = 9;
-
-  // CHECK: br i1{{.*}}handler.
-  if (a + b == a)
-    c = 9;
-
-  // CHECK: br i1{{.*}}handler
-  // Although this can never actually overflow we are still checking that the
-  // sanitizer instruments it.
-  while (--a)
-    some();
-}
-
-// cvise'd kernel code that caused problems during development
-typedef unsigned _size_t;
-typedef enum { FSE_repeat_none } FSE_repeat;
-typedef enum { ZSTD_defaultAllowed } ZSTD_defaultPolicy_e;
-FSE_repeat ZSTD_selectEncodingType_repeatMode;
-ZSTD_defaultPolicy_e ZSTD_selectEncodingType_isDefaultAllowed;
-_size_t ZSTD_NCountCost(void);
-
-// CHECK-LABEL: ZSTD_selectEncodingType
-// CHECK: br i1{{.*}}handler
-void ZSTD_selectEncodingType(void) {
-  _size_t basicCost =
-             ZSTD_selectEncodingType_isDefaultAllowed ? ZSTD_NCountCost() : 0,
-         compressedCost = 3 + ZSTD_NCountCost();
-  if (basicCost <= compressedCost)
-    ZSTD_selectEncodingType_repeatMode = FSE_repeat_none;
-}
-
-// CHECK-LABEL: function_calls
-void function_calls(void) {
-  // CHECK: br i1{{.*}}handler
-  if (some() + b < some())
-    c = 9;
-}
-
-// CHECK-LABEL: not_quite_a_negated_unsigned_const
-void not_quite_a_negated_unsigned_const(void) {
-  // CHECK: br i1{{.*}}handler
-  a = -b;
-}

From 7332713b8eea9bb84d8481376f62b8de7c0ddb3a Mon Sep 17 00:00:00 2001
From: "Oleksandr T." <oleksandr.tarasiuk@outlook.com>
Date: Thu, 15 Aug 2024 20:47:14 +0300
Subject: [PATCH 055/441] [Clang] prevent null explicit object argument from
 being deduced (#104328)

Fixes #102025
---
 clang/docs/ReleaseNotes.rst                |  1 +
 clang/lib/Sema/SemaTemplateDeduction.cpp   |  3 +++
 clang/test/SemaCXX/cxx2b-deducing-this.cpp | 19 +++++++++++++++++++
 3 files changed, 23 insertions(+)

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 4872dbb7a556ad2..5ba9fcb040e3a61 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -257,6 +257,7 @@ Bug Fixes to C++ Support
 - Properly reject defaulted relational operators with invalid types for explicit object parameters,
   e.g., ``bool operator==(this int, const Foo&)`` (#GH100329), and rvalue reference parameters.
 - Properly reject defaulted copy/move assignment operators that have a non-reference explicit object parameter.
+- Fixed an assertion failure by preventing null explicit object arguments from being deduced. (#GH102025).
 
 Bug Fixes to AST Handling
 ^^^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/clang/lib/Sema/SemaTemplateDeduction.cpp b/clang/lib/Sema/SemaTemplateDeduction.cpp
index ec951d5ac06dbc6..fc883e423f47105 100644
--- a/clang/lib/Sema/SemaTemplateDeduction.cpp
+++ b/clang/lib/Sema/SemaTemplateDeduction.cpp
@@ -4462,6 +4462,9 @@ TemplateDeductionResult Sema::DeduceTemplateArguments(
       ParamTypesForArgChecking.push_back(ParamType);
 
       if (ParamIdx == 0 && HasExplicitObject) {
+        if (ObjectType.isNull())
+          return TemplateDeductionResult::InvalidExplicitArguments;
+
         if (auto Result = DeduceCallArgument(ParamType, 0,
                                              /*ExplicitObjectArgument=*/true);
             Result != TemplateDeductionResult::Success)
diff --git a/clang/test/SemaCXX/cxx2b-deducing-this.cpp b/clang/test/SemaCXX/cxx2b-deducing-this.cpp
index 23fb383fb73cbbe..bfcc5991cee5aca 100644
--- a/clang/test/SemaCXX/cxx2b-deducing-this.cpp
+++ b/clang/test/SemaCXX/cxx2b-deducing-this.cpp
@@ -1011,3 +1011,22 @@ struct X;
 static_assert(__is_same(X<B{0}>, X<B{0}>));
 static_assert(!__is_same(X<B{0}>, X<B{1}>));
 } // namespace defaulted_compare
+
+namespace GH102025 {
+struct Foo {
+  template <class T>
+  constexpr auto operator[](this T &&self, auto... i) // expected-note {{candidate template ignored: substitution failure [with T = Foo &, i:auto = <>]: member '_evaluate' used before its declaration}}
+      -> decltype(_evaluate(self, i...)) {
+    return self._evaluate(i...);
+  }
+
+private:
+  template <class T>
+  constexpr auto _evaluate(this T &&self, auto... i) -> decltype((i + ...));
+};
+
+int main() {
+  Foo foo;
+  return foo[]; // expected-error {{no viable overloaded operator[] for type 'Foo'}}
+}
+}

From 29b0a251c14b248848f2bbad1618b66a0c173336 Mon Sep 17 00:00:00 2001
From: zhijian lin <zhijian@ca.ibm.com>
Date: Thu, 15 Aug 2024 13:50:54 -0400
Subject: [PATCH 056/441] [llvm-objdump] Print out  xcoff file header for xcoff
 object file with option private-headers (#96350)

Print out the XCOFF file header and load section header for the XCOFF
object file using llvm-objdump with the --private-headers option.
---
 .../llvm-objdump/XCOFF/private-headers.test   | 52 ++++++++++++
 llvm/tools/llvm-objdump/XCOFFDump.cpp         | 82 ++++++++++++++++++-
 2 files changed, 132 insertions(+), 2 deletions(-)
 create mode 100644 llvm/test/tools/llvm-objdump/XCOFF/private-headers.test

diff --git a/llvm/test/tools/llvm-objdump/XCOFF/private-headers.test b/llvm/test/tools/llvm-objdump/XCOFF/private-headers.test
new file mode 100644
index 000000000000000..9f1e60705bedf3c
--- /dev/null
+++ b/llvm/test/tools/llvm-objdump/XCOFF/private-headers.test
@@ -0,0 +1,52 @@
+## Test the --private-headers option for XCOFF object files.
+
+# RUN: yaml2obj -DMAGIC=0x1DF --docnum=1 %s -o %t_xcoff32.o
+# RUN: yaml2obj -DMAGIC=0x1F7 --docnum=1 %s -o %t_xcoff64.o
+# RUN: llvm-objdump --private-headers %t_xcoff32.o | \
+# RUN:   FileCheck %s --check-prefixes=CHECK32 --match-full-lines --strict-whitespace
+# RUN: llvm-objdump --private-headers %t_xcoff64.o | \
+# RUN:   FileCheck %s --check-prefixes=CHECK64 --match-full-lines --strict-whitespace
+
+--- !XCOFF
+FileHeader:
+  MagicNumber:       [[MAGIC]]
+  CreationTime:      1234
+Sections:
+  - Name:            .text
+    Flags:           [ STYP_TEXT ]
+    SectionData:     "9061FFF880820000"
+  - Name:            .data
+    Flags:           [ STYP_DATA ]
+    SectionData:     "0000000000000FC0"
+
+#      CHECK32:---File Header:
+# CHECK32-NEXT:Magic:              0x1df
+# CHECK32-NEXT:NumberOfSections:   2
+# CHECK32-NEXT:Timestamp:          1970-01-01 00:20:34 (1234)
+# CHECK32-NEXT:SymbolTableOffset:  0x0
+# CHECK32-NEXT:SymbolTableEntries: 0
+# CHECK32-NEXT:OptionalHeaderSize: 0x0
+# CHECK32-NEXT:Flags:              0x0
+
+#      CHECK64:---File Header:
+# CHECK64-NEXT:Magic:              0x1f7
+# CHECK64-NEXT:NumberOfSections:   2
+# CHECK64-NEXT:Timestamp:          1970-01-01 00:20:34 (1234)
+# CHECK64-NEXT:SymbolTableOffset:  0x0
+# CHECK64-NEXT:SymbolTableEntries: 0
+# CHECK64-NEXT:OptionalHeaderSize: 0x0
+# CHECK64-NEXT:Flags:              0x0
+
+## Test if the creation time of XCOFF is zero and the number of symbols is negative.
+# RUN: yaml2obj -DMAGIC=0x1DF --docnum=2 %s -o %t_xcoff_timestamp.o
+# RUN: llvm-objdump --private-headers %t_xcoff_timestamp.o | \
+# RUN:   FileCheck %s --match-full-lines
+
+--- !XCOFF
+FileHeader:
+  MagicNumber:          0x1DF
+  CreationTime:         0
+  EntriesInSymbolTable: -1
+
+# CHECK: Timestamp:          None (0)
+# CHECK: SymbolTableEntries: Reserved Value (-1)
diff --git a/llvm/tools/llvm-objdump/XCOFFDump.cpp b/llvm/tools/llvm-objdump/XCOFFDump.cpp
index d9c00c096209831..5617313cdbf721b 100644
--- a/llvm/tools/llvm-objdump/XCOFFDump.cpp
+++ b/llvm/tools/llvm-objdump/XCOFFDump.cpp
@@ -20,7 +20,9 @@
 #include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/Endian.h"
+#include "llvm/Support/Format.h"
 #include "llvm/Support/FormattedStream.h"
+#include "llvm/Support/raw_ostream.h"
 #include <algorithm>
 
 using namespace llvm;
@@ -30,10 +32,86 @@ using namespace llvm::support;
 
 namespace {
 class XCOFFDumper : public objdump::Dumper {
+  const XCOFFObjectFile &Obj;
+  unsigned Width;
+
 public:
-  XCOFFDumper(const object::XCOFFObjectFile &O) : Dumper(O) {}
-  void printPrivateHeaders() override {}
+  XCOFFDumper(const object::XCOFFObjectFile &O) : Dumper(O), Obj(O) {}
+
+private:
+  void printPrivateHeaders() override;
+  void printFileHeader();
+  FormattedString formatName(StringRef Name);
+  void printHex(StringRef Name, uint64_t Value);
+  void printNumber(StringRef Name, uint64_t Value);
+  void printStrHex(StringRef Name, StringRef Str, uint64_t Value);
+  void setWidth(unsigned W) { Width = W; };
 };
+
+void XCOFFDumper::printPrivateHeaders() { printFileHeader(); }
+
+FormattedString XCOFFDumper::formatName(StringRef Name) {
+  return FormattedString(Name, Width, FormattedString::JustifyLeft);
+}
+
+void XCOFFDumper::printHex(StringRef Name, uint64_t Value) {
+  outs() << formatName(Name) << format_hex(Value, 0) << "\n";
+}
+
+void XCOFFDumper::printNumber(StringRef Name, uint64_t Value) {
+  outs() << formatName(Name) << format_decimal(Value, 0) << "\n";
+}
+
+void XCOFFDumper::printStrHex(StringRef Name, StringRef Str, uint64_t Value) {
+  outs() << formatName(Name) << Str << " (" << format_decimal(Value, 0)
+         << ")\n";
+}
+
+void XCOFFDumper::printFileHeader() {
+  setWidth(20);
+  outs() << "\n---File Header:\n";
+  printHex("Magic:", Obj.getMagic());
+  printNumber("NumberOfSections:", Obj.getNumberOfSections());
+
+  int32_t Timestamp = Obj.getTimeStamp();
+  if (Timestamp > 0) {
+    // This handling of the timestamp assumes that the host system's time_t is
+    // compatible with AIX time_t. If a platform is not compatible, the lit
+    // tests will let us know.
+    time_t TimeDate = Timestamp;
+
+    char FormattedTime[20] = {};
+
+    size_t BytesFormatted = std::strftime(FormattedTime, sizeof(FormattedTime),
+                                          "%F %T", std::gmtime(&TimeDate));
+    assert(BytesFormatted && "The size of the buffer FormattedTime is less "
+                             "than the size of the date/time string.");
+    printStrHex("Timestamp:", FormattedTime, Timestamp);
+  } else {
+    // Negative timestamp values are reserved for future use.
+    printStrHex("Timestamp:", Timestamp == 0 ? "None" : "Reserved Value",
+                Timestamp);
+  }
+
+  // The number of symbol table entries is an unsigned value in 64-bit objects
+  // and a signed value (with negative values being 'reserved') in 32-bit
+  // objects.
+  if (Obj.is64Bit()) {
+    printHex("SymbolTableOffset:", Obj.getSymbolTableOffset64());
+    printNumber("SymbolTableEntries:", Obj.getNumberOfSymbolTableEntries64());
+  } else {
+    printHex("SymbolTableOffset:", Obj.getSymbolTableOffset32());
+    int32_t SymTabEntries = Obj.getRawNumberOfSymbolTableEntries32();
+    if (SymTabEntries >= 0)
+      printNumber("SymbolTableEntries:", SymTabEntries);
+    else
+      printStrHex("SymbolTableEntries:", "Reserved Value", SymTabEntries);
+  }
+
+  printHex("OptionalHeaderSize:", Obj.getOptionalHeaderSize());
+  printHex("Flags:", Obj.getFlags());
+}
+
 } // namespace
 
 std::unique_ptr<objdump::Dumper>

From 75ea8e803a6560b5e16644bb25d6340926156503 Mon Sep 17 00:00:00 2001
From: Timm Baeder <tbaeder@redhat.com>
Date: Thu, 15 Aug 2024 19:59:55 +0200
Subject: [PATCH 057/441] [clang][Interp] Call move function for certain
 primitive types (#104437)

---
 clang/lib/AST/Interp/Descriptor.cpp | 8 ++++++++
 clang/test/AST/Interp/lifetimes.cpp | 8 ++++++++
 2 files changed, 16 insertions(+)

diff --git a/clang/lib/AST/Interp/Descriptor.cpp b/clang/lib/AST/Interp/Descriptor.cpp
index 1b7d9f03f8ef0d0..47b8885c9ae3796 100644
--- a/clang/lib/AST/Interp/Descriptor.cpp
+++ b/clang/lib/AST/Interp/Descriptor.cpp
@@ -284,6 +284,14 @@ static BlockDtorFn getDtorPrim(PrimType Type) {
 }
 
 static BlockMoveFn getMovePrim(PrimType Type) {
+  if (Type == PT_Float)
+    return moveTy<PrimConv<PT_Float>::T>;
+  if (Type == PT_IntAP)
+    return moveTy<PrimConv<PT_IntAP>::T>;
+  if (Type == PT_IntAPS)
+    return moveTy<PrimConv<PT_IntAPS>::T>;
+  if (Type == PT_MemberPtr)
+    return moveTy<PrimConv<PT_MemberPtr>::T>;
   COMPOSITE_TYPE_SWITCH(Type, return moveTy<T>, return nullptr);
 }
 
diff --git a/clang/test/AST/Interp/lifetimes.cpp b/clang/test/AST/Interp/lifetimes.cpp
index 9fca54fe11120aa..9a99485c4a40bff 100644
--- a/clang/test/AST/Interp/lifetimes.cpp
+++ b/clang/test/AST/Interp/lifetimes.cpp
@@ -60,3 +60,11 @@ namespace MoveFnWorks {
   }
   static_assert(dtor_calls_dtor(), "");
 }
+
+namespace PrimitiveMoveFn {
+  /// This used to crash.
+  void test() {
+    const float y = 100;
+    const float &x = y;
+  }
+}

From 7493ea22f8027dc163ca521a71150d264891853c Mon Sep 17 00:00:00 2001
From: earnol <earnol@users.noreply.github.com>
Date: Thu, 15 Aug 2024 14:11:27 -0400
Subject: [PATCH 058/441] [test]Fix test error due to CRT dependency (#104462)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Remove CRT dependency in headers stdint.h, stdio.h inside test
clang/test/CodeGen/bit-int-ubsan.c

---------

Co-authored-by: Eänolituri Lómitaurë <vladislav.aranov@ericsson.com>
Co-authored-by: Aaron Ballman <aaron@aaronballman.com>
Co-authored-by: Paul Kirth <paulkirth@google.com>
---
 clang/test/CodeGen/bit-int-ubsan.c | 22 ++--------------------
 1 file changed, 2 insertions(+), 20 deletions(-)

diff --git a/clang/test/CodeGen/bit-int-ubsan.c b/clang/test/CodeGen/bit-int-ubsan.c
index 35f96963c181d15..10c0d8a23f8f971 100644
--- a/clang/test/CodeGen/bit-int-ubsan.c
+++ b/clang/test/CodeGen/bit-int-ubsan.c
@@ -1,11 +1,9 @@
 // REQUIRES: x86-registered-target
-// RUN: %clang -Wno-constant-conversion -Wno-array-bounds -Wno-division-by-zero -Wno-shift-negative-value -Wno-shift-count-negative -Wno-int-to-pointer-cast -fsanitize=array-bounds,enum,float-cast-overflow,integer-divide-by-zero,implicit-unsigned-integer-truncation,implicit-signed-integer-truncation,implicit-integer-sign-change,unsigned-integer-overflow,signed-integer-overflow,shift-base,shift-exponent -O0 -S -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -Wno-constant-conversion -Wno-array-bounds -Wno-division-by-zero -Wno-shift-negative-value -Wno-shift-count-negative -Wno-int-to-pointer-cast -fsanitize=array-bounds,enum,float-cast-overflow,integer-divide-by-zero,implicit-unsigned-integer-truncation,implicit-signed-integer-truncation,implicit-integer-sign-change,unsigned-integer-overflow,signed-integer-overflow,shift-base,shift-exponent -O0 -emit-llvm -o - %s | FileCheck %s
 
 // The runtime test checking the _BitInt ubsan feature is located in compiler-rt/test/ubsan/TestCases/Integer/bit-int.c
 
-#include <stdint.h>
-#include <stdio.h>
-
+typedef unsigned int uint32_t;
 uint32_t float_divide_by_zero() {
   float f = 1.0f / 0.0f;
   // CHECK: constant { i16, i16, [8 x i8] } { i16 1, i16 32, [8 x i8] c"'float'\00" }
@@ -78,19 +76,3 @@ uint32_t negative_shift5(unsigned _BitInt(37) x)
   return x >> c;
   // CHECK: constant { i16, i16, [20 x i8] } { i16 2, i16 {{([[:xdigit:]]{2})}}, [20 x i8] c"'_BitInt(68)'\00D\00\00\00\00\00" }
 }
-
-int main(int argc, char **argv) {
-  // clang-format off
-  uint64_t result =
-      1ULL +
-      implicit_unsigned_integer_truncation() +
-      (uint32_t)array_bounds() +
-      float_cast_overflow() +
-      (uint64_t)implicit_signed_integer_truncation() +
-      negative_shift1(5) +
-      negative_shift2(5) +
-      negative_shift3(5) +
-      negative_shift5(5);
-  // clang-format on
-  printf("%u\n", (uint32_t)(result & 0xFFFFFFFF));
-}

From 52337d5f9d108f04b2ed06069b21a255c232dc1f Mon Sep 17 00:00:00 2001
From: Tim Northover <t.p.northover@gmail.com>
Date: Thu, 15 Aug 2024 19:19:30 +0100
Subject: [PATCH 059/441] llvm-objdump: ensure a MachO symbol isn't STAB before
 looking up secion (#86667)

The section field has been repurposed for some STAB symbol types, and if
we blindly look it up we'll produce an error and terminate. Logic
already existed

Existing stabs test had a section that was in range. Unfortunately I
don't know of an easy way to produce stabs entries in LLVM (I thought
they died in the 90s until this came up) so I just binary-edited it to
cause a failure on existing llvm-objdump.
---
 .../llvm-objdump/macho-stabs-in-syms.yaml     | 263 ++++++++++++++++++
 llvm/tools/llvm-objdump/llvm-objdump.cpp      |  20 +-
 2 files changed, 273 insertions(+), 10 deletions(-)
 create mode 100644 llvm/test/tools/llvm-objdump/macho-stabs-in-syms.yaml

diff --git a/llvm/test/tools/llvm-objdump/macho-stabs-in-syms.yaml b/llvm/test/tools/llvm-objdump/macho-stabs-in-syms.yaml
new file mode 100644
index 000000000000000..3850473b219cead
--- /dev/null
+++ b/llvm/test/tools/llvm-objdump/macho-stabs-in-syms.yaml
@@ -0,0 +1,263 @@
+# Check we don't error out on MachO files with stabs symbol entries that
+# repurpose the section field so it's out of range for normal symbols.
+
+# RUN: yaml2obj %s -o %t
+# RUN: llvm-objdump --syms %t | FileCheck %s
+
+# CHECK: 0000000000000000      d  *UND*
+--- !mach-o
+FileHeader:
+  magic:           0xFEEDFACF
+  cputype:         0x100000C
+  cpusubtype:      0x0
+  filetype:        0x2
+  ncmds:           16
+  sizeofcmds:      744
+  flags:           0x200085
+  reserved:        0x0
+LoadCommands:
+  - cmd:             LC_SEGMENT_64
+    cmdsize:         72
+    segname:         __PAGEZERO
+    vmaddr:          0
+    vmsize:          4294967296
+    fileoff:         0
+    filesize:        0
+    maxprot:         0
+    initprot:        0
+    nsects:          0
+    flags:           0
+  - cmd:             LC_SEGMENT_64
+    cmdsize:         232
+    segname:         __TEXT
+    vmaddr:          4294967296
+    vmsize:          16384
+    fileoff:         0
+    filesize:        16384
+    maxprot:         5
+    initprot:        5
+    nsects:          2
+    flags:           0
+    Sections:
+      - sectname:        __text
+        segname:         __TEXT
+        addr:            0x100003FA0
+        size:            8
+        offset:          0x3FA0
+        align:           2
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x80000400
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         00008052C0035FD6
+      - sectname:        __unwind_info
+        segname:         __TEXT
+        addr:            0x100003FA8
+        size:            88
+        offset:          0x3FA8
+        align:           2
+        reloff:          0x0
+        nreloc:          0
+        flags:           0x0
+        reserved1:       0x0
+        reserved2:       0x0
+        reserved3:       0x0
+        content:         010000001C000000000000001C000000000000001C00000002000000A03F00004000000040000000A83F00000000000040000000000000000000000000000000030000000C00010010000100000000000000000200000000
+  - cmd:             LC_SEGMENT_64
+    cmdsize:         72
+    segname:         __LINKEDIT
+    vmaddr:          4294983680
+    vmsize:          16384
+    fileoff:         16384
+    filesize:        688
+    maxprot:         1
+    initprot:        1
+    nsects:          0
+    flags:           0
+  - cmd:             LC_DYLD_CHAINED_FIXUPS
+    cmdsize:         16
+    dataoff:         16384
+    datasize:        56
+  - cmd:             LC_DYLD_EXPORTS_TRIE
+    cmdsize:         16
+    dataoff:         16440
+    datasize:        48
+  - cmd:             LC_SYMTAB
+    cmdsize:         24
+    symoff:          16496
+    nsyms:           11
+    stroff:          16672
+    strsize:         120
+  - cmd:             LC_DYSYMTAB
+    cmdsize:         80
+    ilocalsym:       0
+    nlocalsym:       9
+    iextdefsym:      9
+    nextdefsym:      2
+    iundefsym:       11
+    nundefsym:       0
+    tocoff:          0
+    ntoc:            0
+    modtaboff:       0
+    nmodtab:         0
+    extrefsymoff:    0
+    nextrefsyms:     0
+    indirectsymoff:  0
+    nindirectsyms:   0
+    extreloff:       0
+    nextrel:         0
+    locreloff:       0
+    nlocrel:         0
+  - cmd:             LC_LOAD_DYLINKER
+    cmdsize:         32
+    name:            12
+    Content:         '/usr/lib/dyld'
+    ZeroPadBytes:    7
+  - cmd:             LC_UUID
+    cmdsize:         24
+    uuid:            73BD5072-2F45-39FD-8013-8102F002C240
+  - cmd:             LC_BUILD_VERSION
+    cmdsize:         32
+    platform:        1
+    minos:           917504
+    sdk:             918528
+    ntools:          1
+    Tools:
+      - tool:            3
+        version:         72942592
+  - cmd:             LC_SOURCE_VERSION
+    cmdsize:         16
+    version:         0
+  - cmd:             LC_MAIN
+    cmdsize:         24
+    entryoff:        16288
+    stacksize:       0
+  - cmd:             LC_LOAD_DYLIB
+    cmdsize:         56
+    dylib:
+      name:            24
+      timestamp:       2
+      current_version: 88171522
+      compatibility_version: 65536
+    Content:         '/usr/lib/libSystem.B.dylib'
+    ZeroPadBytes:    6
+  - cmd:             LC_FUNCTION_STARTS
+    cmdsize:         16
+    dataoff:         16488
+    datasize:        8
+  - cmd:             LC_DATA_IN_CODE
+    cmdsize:         16
+    dataoff:         16496
+    datasize:        0
+  - cmd:             LC_CODE_SIGNATURE
+    cmdsize:         16
+    dataoff:         16800
+    datasize:        272
+LinkEditData:
+  ExportTrie:
+    TerminalSize:    0
+    NodeOffset:      0
+    Name:            ''
+    Flags:           0x0
+    Address:         0x0
+    Other:           0x0
+    ImportName:      ''
+    Children:
+      - TerminalSize:    0
+        NodeOffset:      18
+        Name:            _
+        Flags:           0x0
+        Address:         0x0
+        Other:           0x0
+        ImportName:      ''
+        Children:
+          - TerminalSize:    2
+            NodeOffset:      9
+            Name:            _mh_execute_header
+            Flags:           0x0
+            Address:         0x0
+            Other:           0x0
+            ImportName:      ''
+          - TerminalSize:    3
+            NodeOffset:      13
+            Name:            main
+            Flags:           0x0
+            Address:         0x3FA0
+            Other:           0x0
+            ImportName:      ''
+  NameList:
+    - n_strx:          1
+      n_type:          0x64
+      # This line has been changed from a real object file to move the section
+      # out of valid range for normal symbols.
+      n_sect:          42
+      n_desc:          0
+      n_value:         0
+    - n_strx:          28
+      n_type:          0x64
+      n_sect:          0
+      n_desc:          0
+      n_value:         0
+    - n_strx:          68
+      n_type:          0x64
+      n_sect:          0
+      n_desc:          0
+      n_value:         0
+    - n_strx:          74
+      n_type:          0x66
+      n_sect:          0
+      n_desc:          1
+      n_value:         1713522175
+    - n_strx:          1
+      n_type:          0x2E
+      n_sect:          1
+      n_desc:          0
+      n_value:         4294983584
+    - n_strx:          22
+      n_type:          0x24
+      n_sect:          1
+      n_desc:          0
+      n_value:         4294983584
+    - n_strx:          1
+      n_type:          0x24
+      n_sect:          0
+      n_desc:          0
+      n_value:         8
+    - n_strx:          1
+      n_type:          0x4E
+      n_sect:          1
+      n_desc:          0
+      n_value:         4294983584
+    - n_strx:          1
+      n_type:          0x64
+      n_sect:          1
+      n_desc:          0
+      n_value:         0
+    - n_strx:          2
+      n_type:          0xF
+      n_sect:          1
+      n_desc:          16
+      n_value:         4294967296
+    - n_strx:          22
+      n_type:          0xF
+      n_sect:          1
+      n_desc:          0
+      n_value:         4294983584
+  StringTable:
+    - ' '
+    - __mh_execute_header
+    - _main
+    - '/Users/tim/llvm/llvm-project/build.rel/'
+    - tmp.c
+    - '/Users/tim/llvm/llvm-project/build.rel/tmp.o'
+    - ''
+  FunctionStarts:  [ 0x3FA0 ]
+  ChainedFixups:   [ 0x0, 0x0, 0x0, 0x0, 0x20, 0x0, 0x0, 0x0, 0x30, 0x0,
+                     0x0, 0x0, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+                     0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+                     0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+                     0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+                     0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ]
+...
diff --git a/llvm/tools/llvm-objdump/llvm-objdump.cpp b/llvm/tools/llvm-objdump/llvm-objdump.cpp
index 768a976cd920275..b69d14b4e7609aa 100644
--- a/llvm/tools/llvm-objdump/llvm-objdump.cpp
+++ b/llvm/tools/llvm-objdump/llvm-objdump.cpp
@@ -2885,16 +2885,6 @@ void Dumper::printSymbol(const SymbolRef &Symbol,
     reportUniqueWarning(AddrOrErr.takeError());
     return;
   }
-  uint64_t Address = *AddrOrErr;
-  section_iterator SecI = unwrapOrError(Symbol.getSection(), FileName);
-  if (SecI != O.section_end() && shouldAdjustVA(*SecI))
-    Address += AdjustVMA;
-  if ((Address < StartAddress) || (Address > StopAddress))
-    return;
-  SymbolRef::Type Type =
-      unwrapOrError(Symbol.getType(), FileName, ArchiveName, ArchitectureName);
-  uint32_t Flags =
-      unwrapOrError(Symbol.getFlags(), FileName, ArchiveName, ArchitectureName);
 
   // Don't ask a Mach-O STAB symbol for its section unless you know that
   // STAB symbol's section field refers to a valid section index. Otherwise
@@ -2913,6 +2903,16 @@ void Dumper::printSymbol(const SymbolRef &Symbol,
                                  : unwrapOrError(Symbol.getSection(), FileName,
                                                  ArchiveName, ArchitectureName);
 
+  uint64_t Address = *AddrOrErr;
+  if (Section != O.section_end() && shouldAdjustVA(*Section))
+    Address += AdjustVMA;
+  if ((Address < StartAddress) || (Address > StopAddress))
+    return;
+  SymbolRef::Type Type =
+      unwrapOrError(Symbol.getType(), FileName, ArchiveName, ArchitectureName);
+  uint32_t Flags =
+      unwrapOrError(Symbol.getFlags(), FileName, ArchiveName, ArchitectureName);
+
   StringRef Name;
   if (Type == SymbolRef::ST_Debug && Section != O.section_end()) {
     if (Expected<StringRef> NameOrErr = Section->getName())

From 47721d46187f89c12a13d07b5857496301cf5d6e Mon Sep 17 00:00:00 2001
From: royitaqi <royitaqi@users.noreply.github.com>
Date: Thu, 15 Aug 2024 11:26:24 -0700
Subject: [PATCH 060/441] [lldb] Realpath symlinks for breakpoints (#102223)

Improve the chance of resolving file/line breakpoints by realpath'ing the support files before doing a second match attempt, with some conditions applied.

A working [hello-world example](https://github.com/royitaqi/lldb_demos/blob/main/realpath/README.md).

See [patch](https://github.com/llvm/llvm-project/pull/102223) for more info about problem/motivation, details of the feature, new settings, telemetries and tests.
---
 lldb/include/lldb/Symbol/CompileUnit.h        |   9 +-
 lldb/include/lldb/Target/Statistics.h         |   5 +
 lldb/include/lldb/Target/Target.h             |   3 +
 lldb/include/lldb/Utility/FileSpecList.h      |   8 +-
 lldb/include/lldb/Utility/RealpathPrefixes.h  |  77 +++++
 lldb/include/lldb/lldb-forward.h              |   1 +
 .../Breakpoint/BreakpointResolverFileLine.cpp |  12 +-
 lldb/source/Symbol/CompileUnit.cpp            |  14 +-
 lldb/source/Target/Statistics.cpp             |  12 +
 lldb/source/Target/Target.cpp                 |   8 +
 lldb/source/Target/TargetProperties.td        |   3 +
 lldb/source/Utility/CMakeLists.txt            |   1 +
 lldb/source/Utility/FileSpecList.cpp          | 108 ++++--
 lldb/source/Utility/RealpathPrefixes.cpp      |  70 ++++
 .../Makefile                                  |   3 +
 .../TestBreakpoint.py                         | 158 +++++++++
 .../main.c                                    |  10 +
 .../real/bar.h                                |   3 +
 .../real/foo.h                                |   3 +
 .../real/qux.h                                |   3 +
 .../symlink1/foo.h                            |   1 +
 .../symlink2                                  |   1 +
 .../to-be-mapped/README.md                    |   1 +
 lldb/unittests/Core/CMakeLists.txt            |   1 -
 lldb/unittests/Core/FileSpecListTest.cpp      | 125 -------
 lldb/unittests/Utility/CMakeLists.txt         |   3 +
 lldb/unittests/Utility/FileSpecListTest.cpp   | 323 ++++++++++++++++++
 .../unittests/Utility/MockSymlinkFileSystem.h |  66 ++++
 .../Utility/RealpathPrefixesTest.cpp          | 140 ++++++++
 29 files changed, 1002 insertions(+), 170 deletions(-)
 create mode 100644 lldb/include/lldb/Utility/RealpathPrefixes.h
 create mode 100644 lldb/source/Utility/RealpathPrefixes.cpp
 create mode 100644 lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/Makefile
 create mode 100644 lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/TestBreakpoint.py
 create mode 100644 lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/main.c
 create mode 100644 lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/real/bar.h
 create mode 100644 lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/real/foo.h
 create mode 100644 lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/real/qux.h
 create mode 120000 lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/symlink1/foo.h
 create mode 120000 lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/symlink2
 create mode 100644 lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/to-be-mapped/README.md
 delete mode 100644 lldb/unittests/Core/FileSpecListTest.cpp
 create mode 100644 lldb/unittests/Utility/FileSpecListTest.cpp
 create mode 100644 lldb/unittests/Utility/MockSymlinkFileSystem.h
 create mode 100644 lldb/unittests/Utility/RealpathPrefixesTest.cpp

diff --git a/lldb/include/lldb/Symbol/CompileUnit.h b/lldb/include/lldb/Symbol/CompileUnit.h
index c20a37e32830755..c5bb080d2118490 100644
--- a/lldb/include/lldb/Symbol/CompileUnit.h
+++ b/lldb/include/lldb/Symbol/CompileUnit.h
@@ -19,11 +19,13 @@
 #include "lldb/Utility/Stream.h"
 #include "lldb/Utility/UserID.h"
 #include "lldb/lldb-enumerations.h"
+#include "lldb/lldb-forward.h"
 
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
 
 namespace lldb_private {
+
 /// \class CompileUnit CompileUnit.h "lldb/Symbol/CompileUnit.h"
 /// A class that describes a compilation unit.
 ///
@@ -389,10 +391,15 @@ class CompileUnit : public std::enable_shared_from_this<CompileUnit>,
   ///     A SymbolContext list class that will get any matching
   ///     entries appended to.
   ///
+  /// \param[in] realpath_prefixes
+  ///     Paths that start with one of the prefixes in this list will be
+  ///     realpath'ed to resolve any symlinks.
+  ///
   /// \see enum SymbolContext::Scope
   void ResolveSymbolContext(const SourceLocationSpec &src_location_spec,
                             lldb::SymbolContextItem resolve_scope,
-                            SymbolContextList &sc_list);
+                            SymbolContextList &sc_list,
+                            RealpathPrefixes *realpath_prefixes = nullptr);
 
   /// Get whether compiler optimizations were enabled for this compile unit
   ///
diff --git a/lldb/include/lldb/Target/Statistics.h b/lldb/include/lldb/Target/Statistics.h
index 35bd7f8a66e055e..5193d099a5494d8 100644
--- a/lldb/include/lldb/Target/Statistics.h
+++ b/lldb/include/lldb/Target/Statistics.h
@@ -10,6 +10,7 @@
 #define LLDB_TARGET_STATISTICS_H
 
 #include "lldb/Utility/ConstString.h"
+#include "lldb/Utility/RealpathPrefixes.h"
 #include "lldb/Utility/Stream.h"
 #include "lldb/lldb-forward.h"
 #include "llvm/ADT/StringMap.h"
@@ -184,6 +185,8 @@ class TargetStats {
   void SetFirstPrivateStopTime();
   void SetFirstPublicStopTime();
   void IncreaseSourceMapDeduceCount();
+  void IncreaseSourceRealpathAttemptCount(uint32_t count);
+  void IncreaseSourceRealpathCompatibleCount(uint32_t count);
 
   StatsDuration &GetCreateTime() { return m_create_time; }
   StatsSuccessFail &GetExpressionStats() { return m_expr_eval; }
@@ -198,6 +201,8 @@ class TargetStats {
   StatsSuccessFail m_frame_var{"frameVariable"};
   std::vector<intptr_t> m_module_identifiers;
   uint32_t m_source_map_deduce_count = 0;
+  uint32_t m_source_realpath_attempt_count = 0;
+  uint32_t m_source_realpath_compatible_count = 0;
   void CollectStats(Target &target);
 };
 
diff --git a/lldb/include/lldb/Target/Target.h b/lldb/include/lldb/Target/Target.h
index 119dff4d498199f..7f4d607f5427dff 100644
--- a/lldb/include/lldb/Target/Target.h
+++ b/lldb/include/lldb/Target/Target.h
@@ -34,6 +34,7 @@
 #include "lldb/Utility/ArchSpec.h"
 #include "lldb/Utility/Broadcaster.h"
 #include "lldb/Utility/LLDBAssert.h"
+#include "lldb/Utility/RealpathPrefixes.h"
 #include "lldb/Utility/Timeout.h"
 #include "lldb/lldb-public.h"
 
@@ -117,6 +118,8 @@ class TargetProperties : public Properties {
 
   InlineStrategy GetInlineStrategy() const;
 
+  RealpathPrefixes GetSourceRealpathPrefixes() const;
+
   llvm::StringRef GetArg0() const;
 
   void SetArg0(llvm::StringRef arg);
diff --git a/lldb/include/lldb/Utility/FileSpecList.h b/lldb/include/lldb/Utility/FileSpecList.h
index 6eb3bb9971f13ad..d091a9246e08277 100644
--- a/lldb/include/lldb/Utility/FileSpecList.h
+++ b/lldb/include/lldb/Utility/FileSpecList.h
@@ -64,10 +64,16 @@ class SupportFileList {
   /// \param[in] file
   ///     The file specification to search for.
   ///
+  /// \param[in] realpath_prefixes
+  ///     Paths that start with one of the prefixes in this list will be
+  ///     realpath'ed to resolve any symlinks.
+  ///
   /// \return
   ///     The index of the file that matches \a file if it is found,
   ///     else UINT32_MAX is returned.
-  size_t FindCompatibleIndex(size_t idx, const FileSpec &file) const;
+  size_t
+  FindCompatibleIndex(size_t idx, const FileSpec &file,
+                      RealpathPrefixes *realpath_prefixes = nullptr) const;
 
   template <class... Args> void EmplaceBack(Args &&...args) {
     m_files.push_back(
diff --git a/lldb/include/lldb/Utility/RealpathPrefixes.h b/lldb/include/lldb/Utility/RealpathPrefixes.h
new file mode 100644
index 000000000000000..daa2a479d6a2469
--- /dev/null
+++ b/lldb/include/lldb/Utility/RealpathPrefixes.h
@@ -0,0 +1,77 @@
+//===-- RealpathPrefixes.h --------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLDB_UTILITY_REALPATHPREFIXES_H
+#define LLDB_UTILITY_REALPATHPREFIXES_H
+
+#include "lldb/lldb-forward.h"
+#include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/Support/VirtualFileSystem.h"
+
+#include <optional>
+#include <string>
+#include <vector>
+
+namespace lldb_private {
+
+class RealpathPrefixes {
+public:
+  /// \param[in] file_spec_list
+  ///     Prefixes are obtained from FileSpecList, through FileSpec::GetPath(),
+  ///     which ensures that the paths are normalized. For example:
+  ///     "./foo/.." -> ""
+  ///     "./foo/../bar" -> "bar"
+  ///
+  /// \param[in] fs
+  ///     An optional filesystem to use for realpath'ing. If not set, the real
+  ///     filesystem will be used.
+  explicit RealpathPrefixes(const FileSpecList &file_spec_list,
+                            llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> fs =
+                                llvm::vfs::getRealFileSystem());
+
+  std::optional<FileSpec> ResolveSymlinks(const FileSpec &file_spec);
+
+  // If/when Statistics.h/cpp is moved into Utility, we can remove these
+  // methods, hold a (weak) pointer to `TargetStats` and directly increment
+  // on that object.
+  void IncreaseSourceRealpathAttemptCount() {
+    ++m_source_realpath_attempt_count;
+  }
+  uint32_t GetSourceRealpathAttemptCount() const {
+    return m_source_realpath_attempt_count;
+  }
+  void IncreaseSourceRealpathCompatibleCount() {
+    ++m_source_realpath_compatible_count;
+  }
+  uint32_t GetSourceRealpathCompatibleCount() const {
+    return m_source_realpath_compatible_count;
+  }
+
+private:
+  // Paths that start with one of the prefixes in this list will be realpath'ed
+  // to resolve any symlinks.
+  //
+  // Wildcard prefixes:
+  // - "" (empty string) will match all paths.
+  // - "/" will match all absolute paths.
+  std::vector<std::string> m_prefixes;
+
+  // The filesystem to use for realpath'ing.
+  llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> m_fs;
+
+  // The optional Target instance to gather statistics.
+  lldb::TargetWP m_target;
+
+  // Statistics that we temprarily hold here, to be gathered into TargetStats
+  uint32_t m_source_realpath_attempt_count = 0;
+  uint32_t m_source_realpath_compatible_count = 0;
+};
+
+} // namespace lldb_private
+
+#endif // LLDB_UTILITY_REALPATHPREFIXES_H
diff --git a/lldb/include/lldb/lldb-forward.h b/lldb/include/lldb/lldb-forward.h
index 1024501e05bcac7..337eff696fcf3f5 100644
--- a/lldb/include/lldb/lldb-forward.h
+++ b/lldb/include/lldb/lldb-forward.h
@@ -175,6 +175,7 @@ class Queue;
 class QueueImpl;
 class QueueItem;
 class REPL;
+class RealpathPrefixes;
 class RecognizedStackFrame;
 class RegisterCheckpoint;
 class RegisterContext;
diff --git a/lldb/source/Breakpoint/BreakpointResolverFileLine.cpp b/lldb/source/Breakpoint/BreakpointResolverFileLine.cpp
index 16c4ee1b88d1620..508754082cae8a5 100644
--- a/lldb/source/Breakpoint/BreakpointResolverFileLine.cpp
+++ b/lldb/source/Breakpoint/BreakpointResolverFileLine.cpp
@@ -15,6 +15,7 @@
 #include "lldb/Target/Target.h"
 #include "lldb/Utility/LLDBLog.h"
 #include "lldb/Utility/Log.h"
+#include "lldb/Utility/RealpathPrefixes.h"
 #include "lldb/Utility/StreamString.h"
 #include <optional>
 
@@ -290,16 +291,25 @@ Searcher::CallbackReturn BreakpointResolverFileLine::SearchCallback(
   const uint32_t line = m_location_spec.GetLine().value_or(0);
   const std::optional<uint16_t> column = m_location_spec.GetColumn();
 
+  Target &target = GetBreakpoint()->GetTarget();
+  RealpathPrefixes realpath_prefixes = target.GetSourceRealpathPrefixes();
+
   const size_t num_comp_units = context.module_sp->GetNumCompileUnits();
   for (size_t i = 0; i < num_comp_units; i++) {
     CompUnitSP cu_sp(context.module_sp->GetCompileUnitAtIndex(i));
     if (cu_sp) {
       if (filter.CompUnitPasses(*cu_sp))
         cu_sp->ResolveSymbolContext(m_location_spec, eSymbolContextEverything,
-                                    sc_list);
+                                    sc_list, &realpath_prefixes);
     }
   }
 
+  // Gather stats into the Target
+  target.GetStatistics().IncreaseSourceRealpathAttemptCount(
+      realpath_prefixes.GetSourceRealpathAttemptCount());
+  target.GetStatistics().IncreaseSourceRealpathCompatibleCount(
+      realpath_prefixes.GetSourceRealpathCompatibleCount());
+
   FilterContexts(sc_list);
 
   DeduceSourceMapping(sc_list);
diff --git a/lldb/source/Symbol/CompileUnit.cpp b/lldb/source/Symbol/CompileUnit.cpp
index ddeacf18e855ee3..db8f8ce6bcbc923 100644
--- a/lldb/source/Symbol/CompileUnit.cpp
+++ b/lldb/source/Symbol/CompileUnit.cpp
@@ -213,11 +213,12 @@ VariableListSP CompileUnit::GetVariableList(bool can_create) {
   return m_variables;
 }
 
-std::vector<uint32_t> FindFileIndexes(const SupportFileList &files,
-                                      const FileSpec &file) {
+std::vector<uint32_t>
+FindFileIndexes(const SupportFileList &files, const FileSpec &file,
+                RealpathPrefixes *realpath_prefixes = nullptr) {
   std::vector<uint32_t> result;
   uint32_t idx = -1;
-  while ((idx = files.FindCompatibleIndex(idx + 1, file)) !=
+  while ((idx = files.FindCompatibleIndex(idx + 1, file, realpath_prefixes)) !=
          UINT32_MAX)
     result.push_back(idx);
   return result;
@@ -247,7 +248,8 @@ uint32_t CompileUnit::FindLineEntry(uint32_t start_idx, uint32_t line,
 
 void CompileUnit::ResolveSymbolContext(
     const SourceLocationSpec &src_location_spec,
-    SymbolContextItem resolve_scope, SymbolContextList &sc_list) {
+    SymbolContextItem resolve_scope, SymbolContextList &sc_list,
+    RealpathPrefixes *realpath_prefixes) {
   const FileSpec file_spec = src_location_spec.GetFileSpec();
   const uint32_t line = src_location_spec.GetLine().value_or(0);
   const bool check_inlines = src_location_spec.GetCheckInlines();
@@ -275,8 +277,8 @@ void CompileUnit::ResolveSymbolContext(
     return;
   }
 
-  std::vector<uint32_t> file_indexes = FindFileIndexes(GetSupportFiles(),
-                                                       file_spec);
+  std::vector<uint32_t> file_indexes =
+      FindFileIndexes(GetSupportFiles(), file_spec, realpath_prefixes);
   const size_t num_file_indexes = file_indexes.size();
   if (num_file_indexes == 0)
     return;
diff --git a/lldb/source/Target/Statistics.cpp b/lldb/source/Target/Statistics.cpp
index 583d1524881fc38..390e04cebf6be6c 100644
--- a/lldb/source/Target/Statistics.cpp
+++ b/lldb/source/Target/Statistics.cpp
@@ -192,6 +192,10 @@ TargetStats::ToJSON(Target &target,
   }
   target_metrics_json.try_emplace("sourceMapDeduceCount",
                                   m_source_map_deduce_count);
+  target_metrics_json.try_emplace("sourceRealpathAttemptCount",
+                                  m_source_realpath_attempt_count);
+  target_metrics_json.try_emplace("sourceRealpathCompatibleCount",
+                                  m_source_realpath_compatible_count);
   return target_metrics_json;
 }
 
@@ -220,6 +224,14 @@ void TargetStats::IncreaseSourceMapDeduceCount() {
   ++m_source_map_deduce_count;
 }
 
+void TargetStats::IncreaseSourceRealpathAttemptCount(uint32_t count) {
+  m_source_realpath_attempt_count += count;
+}
+
+void TargetStats::IncreaseSourceRealpathCompatibleCount(uint32_t count) {
+  m_source_realpath_compatible_count += count;
+}
+
 bool DebuggerStats::g_collecting_stats = false;
 
 llvm::json::Value DebuggerStats::ReportStatistics(
diff --git a/lldb/source/Target/Target.cpp b/lldb/source/Target/Target.cpp
index 129683c43f0c1a0..5a5d689e03fbc04 100644
--- a/lldb/source/Target/Target.cpp
+++ b/lldb/source/Target/Target.cpp
@@ -60,6 +60,7 @@
 #include "lldb/Utility/LLDBAssert.h"
 #include "lldb/Utility/LLDBLog.h"
 #include "lldb/Utility/Log.h"
+#include "lldb/Utility/RealpathPrefixes.h"
 #include "lldb/Utility/State.h"
 #include "lldb/Utility/StreamString.h"
 #include "lldb/Utility/Timer.h"
@@ -4354,6 +4355,13 @@ InlineStrategy TargetProperties::GetInlineStrategy() const {
       static_cast<InlineStrategy>(g_target_properties[idx].default_uint_value));
 }
 
+// Returning RealpathPrefixes, but the setting's type is FileSpecList. We do
+// this because we want the FileSpecList to normalize the file paths for us.
+RealpathPrefixes TargetProperties::GetSourceRealpathPrefixes() const {
+  const uint32_t idx = ePropertySourceRealpathPrefixes;
+  return RealpathPrefixes(GetPropertyAtIndexAs<FileSpecList>(idx, {}));
+}
+
 llvm::StringRef TargetProperties::GetArg0() const {
   const uint32_t idx = ePropertyArg0;
   return GetPropertyAtIndexAs<llvm::StringRef>(
diff --git a/lldb/source/Target/TargetProperties.td b/lldb/source/Target/TargetProperties.td
index ef538678670fea0..421252aa4aea26b 100644
--- a/lldb/source/Target/TargetProperties.td
+++ b/lldb/source/Target/TargetProperties.td
@@ -150,6 +150,9 @@ let Definition = "target" in {
     DefaultEnumValue<"eInlineBreakpointsAlways">,
     EnumValues<"OptionEnumValues(g_inline_breakpoint_enums)">,
     Desc<"The strategy to use when settings breakpoints by file and line. Breakpoint locations can end up being inlined by the compiler, so that a compile unit 'a.c' might contain an inlined function from another source file. Usually this is limited to breakpoint locations from inlined functions from header or other include files, or more accurately non-implementation source files. Sometimes code might #include implementation files and cause inlined breakpoint locations in inlined implementation files. Always checking for inlined breakpoint locations can be expensive (memory and time), so if you have a project with many headers and find that setting breakpoints is slow, then you can change this setting to headers. This setting allows you to control exactly which strategy is used when setting file and line breakpoints.">;
+  def SourceRealpathPrefixes: Property<"source-realpath-prefixes", "FileSpecList">,
+    DefaultStringValue<"">,
+    Desc<"Realpath any source paths that start with one of these prefixes. If the debug info contains symlinks which match the original source file's basename but don't match its location that the user will use to set breakpoints, then this setting can help resolve breakpoints correctly. This handles both symlinked files and directories. Wild card prefixes: An empty string matches all paths. A forward slash matches absolute paths.">;
   def DisassemblyFlavor: Property<"x86-disassembly-flavor", "Enum">,
     DefaultEnumValue<"eX86DisFlavorDefault">,
     EnumValues<"OptionEnumValues(g_x86_dis_flavor_value_types)">,
diff --git a/lldb/source/Utility/CMakeLists.txt b/lldb/source/Utility/CMakeLists.txt
index e9954d66cd1a528..397db0e89760234 100644
--- a/lldb/source/Utility/CMakeLists.txt
+++ b/lldb/source/Utility/CMakeLists.txt
@@ -51,6 +51,7 @@ add_lldb_library(lldbUtility NO_INTERNAL_DEPENDENCIES
   Log.cpp
   NameMatches.cpp
   ProcessInfo.cpp
+  RealpathPrefixes.cpp
   RegisterValue.cpp
   RegularExpression.cpp
   Instrumentation.cpp
diff --git a/lldb/source/Utility/FileSpecList.cpp b/lldb/source/Utility/FileSpecList.cpp
index 7647e04a8204516..5852367f77827f6 100644
--- a/lldb/source/Utility/FileSpecList.cpp
+++ b/lldb/source/Utility/FileSpecList.cpp
@@ -7,7 +7,12 @@
 //===----------------------------------------------------------------------===//
 
 #include "lldb/Utility/FileSpecList.h"
+#include "lldb/Target/Statistics.h"
+#include "lldb/Target/Target.h"
 #include "lldb/Utility/ConstString.h"
+#include "lldb/Utility/LLDBLog.h"
+#include "lldb/Utility/Log.h"
+#include "lldb/Utility/RealpathPrefixes.h"
 #include "lldb/Utility/Stream.h"
 
 #include <cstdint>
@@ -108,52 +113,85 @@ size_t SupportFileList::FindFileIndex(size_t start_idx,
                          });
 }
 
-size_t SupportFileList::FindCompatibleIndex(size_t start_idx,
-                                            const FileSpec &file_spec) const {
-  const size_t num_files = m_files.size();
-  if (start_idx >= num_files)
-    return UINT32_MAX;
+enum IsCompatibleResult {
+  kNoMatch = 0,
+  kOnlyFileMatch = 1,
+  kBothDirectoryAndFileMatch = 2,
+};
 
+IsCompatibleResult IsCompatible(const FileSpec &curr_file,
+                                const FileSpec &file_spec) {
   const bool file_spec_relative = file_spec.IsRelative();
   const bool file_spec_case_sensitive = file_spec.IsCaseSensitive();
   // When looking for files, we will compare only the filename if the directory
   // argument is empty in file_spec
   const bool full = !file_spec.GetDirectory().IsEmpty();
 
+  // Always start by matching the filename first
+  if (!curr_file.FileEquals(file_spec))
+    return IsCompatibleResult::kNoMatch;
+
+  // Only compare the full name if the we were asked to and if the current
+  // file entry has a directory. If it doesn't have a directory then we only
+  // compare the filename.
+  if (FileSpec::Equal(curr_file, file_spec, full)) {
+    return IsCompatibleResult::kBothDirectoryAndFileMatch;
+  } else if (curr_file.IsRelative() || file_spec_relative) {
+    llvm::StringRef curr_file_dir = curr_file.GetDirectory().GetStringRef();
+    if (curr_file_dir.empty())
+      // Basename match only for this file in the list
+      return IsCompatibleResult::kBothDirectoryAndFileMatch;
+
+    // Check if we have a relative path in our file list, or if "file_spec" is
+    // relative, if so, check if either ends with the other.
+    llvm::StringRef file_spec_dir = file_spec.GetDirectory().GetStringRef();
+    // We have a relative path in our file list, it matches if the
+    // specified path ends with this path, but we must ensure the full
+    // component matches (we don't want "foo/bar.cpp" to match "oo/bar.cpp").
+    auto is_suffix = [](llvm::StringRef a, llvm::StringRef b,
+                        bool case_sensitive) -> bool {
+      if (case_sensitive ? a.consume_back(b) : a.consume_back_insensitive(b))
+        return a.empty() || a.ends_with("/");
+      return false;
+    };
+    const bool case_sensitive =
+        file_spec_case_sensitive || curr_file.IsCaseSensitive();
+    if (is_suffix(curr_file_dir, file_spec_dir, case_sensitive) ||
+        is_suffix(file_spec_dir, curr_file_dir, case_sensitive))
+      return IsCompatibleResult::kBothDirectoryAndFileMatch;
+  }
+  return IsCompatibleResult::kOnlyFileMatch;
+}
+
+size_t SupportFileList::FindCompatibleIndex(
+    size_t start_idx, const FileSpec &file_spec,
+    RealpathPrefixes *realpath_prefixes) const {
+  const size_t num_files = m_files.size();
+  if (start_idx >= num_files)
+    return UINT32_MAX;
+
   for (size_t idx = start_idx; idx < num_files; ++idx) {
     const FileSpec &curr_file = m_files[idx]->GetSpecOnly();
 
-    // Always start by matching the filename first
-    if (!curr_file.FileEquals(file_spec))
-      continue;
-
-    // Only compare the full name if the we were asked to and if the current
-    // file entry has the a directory. If it doesn't have a directory then we
-    // only compare the filename.
-    if (FileSpec::Equal(curr_file, file_spec, full)) {
+    IsCompatibleResult result = IsCompatible(curr_file, file_spec);
+    if (result == IsCompatibleResult::kBothDirectoryAndFileMatch)
       return idx;
-    } else if (curr_file.IsRelative() || file_spec_relative) {
-      llvm::StringRef curr_file_dir = curr_file.GetDirectory().GetStringRef();
-      if (curr_file_dir.empty())
-        return idx; // Basename match only for this file in the list
-
-      // Check if we have a relative path in our file list, or if "file_spec" is
-      // relative, if so, check if either ends with the other.
-      llvm::StringRef file_spec_dir = file_spec.GetDirectory().GetStringRef();
-      // We have a relative path in our file list, it matches if the
-      // specified path ends with this path, but we must ensure the full
-      // component matches (we don't want "foo/bar.cpp" to match "oo/bar.cpp").
-      auto is_suffix = [](llvm::StringRef a, llvm::StringRef b,
-                          bool case_sensitive) -> bool {
-        if (case_sensitive ? a.consume_back(b) : a.consume_back_insensitive(b))
-          return a.empty() || a.ends_with("/");
-        return false;
-      };
-      const bool case_sensitive =
-          file_spec_case_sensitive || curr_file.IsCaseSensitive();
-      if (is_suffix(curr_file_dir, file_spec_dir, case_sensitive) ||
-          is_suffix(file_spec_dir, curr_file_dir, case_sensitive))
-        return idx;
+
+    if (realpath_prefixes && result == IsCompatibleResult::kOnlyFileMatch) {
+      if (std::optional<FileSpec> resolved_curr_file =
+              realpath_prefixes->ResolveSymlinks(curr_file)) {
+        if (IsCompatible(*resolved_curr_file, file_spec) ==
+            IsCompatibleResult::kBothDirectoryAndFileMatch) {
+          // Stats and logging.
+          realpath_prefixes->IncreaseSourceRealpathCompatibleCount();
+          Log *log = GetLog(LLDBLog::Source);
+          LLDB_LOGF(log,
+                    "Realpath'ed support file %s is compatible to input file",
+                    resolved_curr_file->GetPath().c_str());
+          // We found a match
+          return idx;
+        }
+      }
     }
   }
 
diff --git a/lldb/source/Utility/RealpathPrefixes.cpp b/lldb/source/Utility/RealpathPrefixes.cpp
new file mode 100644
index 000000000000000..14c81ee6a1f5711
--- /dev/null
+++ b/lldb/source/Utility/RealpathPrefixes.cpp
@@ -0,0 +1,70 @@
+//===-- RealpathPrefixes.cpp ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "lldb/Utility/RealpathPrefixes.h"
+
+#include "lldb/Utility/FileSpec.h"
+#include "lldb/Utility/FileSpecList.h"
+#include "lldb/Utility/LLDBLog.h"
+#include "lldb/Utility/Log.h"
+
+using namespace lldb_private;
+
+RealpathPrefixes::RealpathPrefixes(
+    const FileSpecList &file_spec_list,
+    llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> fs)
+    : m_fs(fs) {
+  m_prefixes.reserve(file_spec_list.GetSize());
+  for (const FileSpec &file_spec : file_spec_list) {
+    m_prefixes.emplace_back(file_spec.GetPath());
+  }
+}
+
+std::optional<FileSpec>
+RealpathPrefixes::ResolveSymlinks(const FileSpec &file_spec) {
+  if (m_prefixes.empty())
+    return std::nullopt;
+
+  // Test if `b` is a *path* prefix of `a` (not just *string* prefix).
+  // E.g. "/foo/bar" is a path prefix of "/foo/bar/baz" but not "/foo/barbaz".
+  auto is_path_prefix = [](llvm::StringRef a, llvm::StringRef b,
+                           bool case_sensitive,
+                           llvm::sys::path::Style style) -> bool {
+    if (case_sensitive ? a.consume_front(b) : a.consume_front_insensitive(b))
+      // If `b` isn't "/", then it won't end with "/" because it comes from
+      // `FileSpec`. After `a` consumes `b`, `a` should either be empty (i.e.
+      // `a` == `b`) or end with "/" (the remainder of `a` is a subdirectory).
+      return b == "/" || a.empty() ||
+             llvm::sys::path::is_separator(a[0], style);
+    return false;
+  };
+  std::string file_spec_path = file_spec.GetPath();
+  for (const std::string &prefix : m_prefixes) {
+    if (is_path_prefix(file_spec_path, prefix, file_spec.IsCaseSensitive(),
+                       file_spec.GetPathStyle())) {
+      // Stats and logging.
+      IncreaseSourceRealpathAttemptCount();
+      Log *log = GetLog(LLDBLog::Source);
+      LLDB_LOGF(log, "Realpath'ing support file %s", file_spec_path.c_str());
+
+      // One prefix matched. Try to realpath.
+      llvm::SmallString<PATH_MAX> buff;
+      std::error_code ec = m_fs->getRealPath(file_spec_path, buff);
+      if (ec)
+        return std::nullopt;
+      FileSpec realpath(buff, file_spec.GetPathStyle());
+
+      // Only return realpath if it is different from the original file_spec.
+      if (realpath != file_spec)
+        return realpath;
+      return std::nullopt;
+    }
+  }
+  // No prefix matched
+  return std::nullopt;
+}
diff --git a/lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/Makefile b/lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/Makefile
new file mode 100644
index 000000000000000..10495940055b63d
--- /dev/null
+++ b/lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/Makefile
@@ -0,0 +1,3 @@
+C_SOURCES := main.c
+
+include Makefile.rules
diff --git a/lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/TestBreakpoint.py b/lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/TestBreakpoint.py
new file mode 100644
index 000000000000000..5dc2af73f3647d4
--- /dev/null
+++ b/lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/TestBreakpoint.py
@@ -0,0 +1,158 @@
+"""
+Test lldb breakpoint with symlinks/realpath and source-map.
+"""
+
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil, lldbplatformutil
+
+
+class BreakpointTestCase(TestBase):
+    NO_DEBUG_INFO_TESTCASE = True
+
+    def setUp(self):
+        # Call super's setUp().
+        TestBase.setUp(self)
+        # Find the line number to break inside main().
+        self.line_in_main = line_number("main.c", "// Set break point at this line.")
+        self.line_in_foo = line_number("real/foo.h", "// Set break point at this line.")
+        self.line_in_bar = line_number("real/bar.h", "// Set break point at this line.")
+        self.line_in_qux = line_number("real/qux.h", "// Set break point at this line.")
+        # disable "There is a running process, kill it and restart?" prompt
+        self.runCmd("settings set auto-confirm true")
+        self.addTearDownHook(lambda: self.runCmd("settings clear auto-confirm"))
+
+    def buildAndCreateTarget(self):
+        self.build()
+        exe = self.getBuildArtifact("a.out")
+
+        # Create a target by the debugger.
+        target = self.dbg.CreateTarget(exe)
+        self.assertTrue(target, VALID_TARGET)
+
+    @skipIf(oslist=["windows"])
+    def test_file_line_breakpoint_realpath_and_source_map(self):
+        """Test file/line breakpoint with realpathing and source-mapping."""
+        self.buildAndCreateTarget()
+        cwd = os.getcwd()
+
+        ######################################################################
+        # Baseline
+        # --------------------------------------------------------------------
+        # Breakpoints should be resolved with paths which are in the line-table.
+        lldbutil.run_break_set_by_file_and_line(
+            self, "main.c", self.line_in_main, num_expected_locations=1, loc_exact=True
+        )
+        lldbutil.run_break_set_by_file_and_line(
+            self,
+            "symlink1/foo.h",
+            self.line_in_foo,
+            num_expected_locations=1,
+            loc_exact=True,
+        )
+        lldbutil.run_break_set_by_file_and_line(
+            self,
+            "symlink2/bar.h",
+            self.line_in_bar,
+            num_expected_locations=1,
+            loc_exact=True,
+        )
+        lldbutil.run_break_set_by_file_and_line(
+            self,
+            "symlink2/qux.h",
+            self.line_in_qux,
+            num_expected_locations=1,
+            loc_exact=True,
+        )
+
+        ######################################################################
+        # Symlinked file
+        # --------------------------------------------------------------------
+        # - `symlink1/foo.h` is a symlink file, pointing at `real/foo.h`
+        # - main.c includes `symlink1/foo.h`.
+        # - As a result, the line-table contains a support file `(test_base_dir)/symlink1/foo.h`
+        # - Setting a breakpoint for `real/foo.h` won't be resolved, because it doesn't match the above path.
+        # - Setting a realpath prefix to the current working directory will cause the above support file to be realpath'ed to `(test_base_dir)/real/foo.h`
+        # - Now setting a breakpoint for `real/foo.h` will be resolved.
+        lldbutil.run_break_set_by_file_and_line(
+            self,
+            "real/foo.h",
+            self.line_in_foo,
+            num_expected_locations=0,
+            loc_exact=True,
+        )
+        self.runCmd(f'settings set target.source-realpath-prefixes "{cwd}"')
+        lldbutil.run_break_set_by_file_and_line(
+            self,
+            "real/foo.h",
+            self.line_in_foo,
+            num_expected_locations=1,
+            loc_exact=True,
+        )
+        # Clear settings so that the test below won't be affected.
+        self.runCmd("settings clear target.source-realpath-prefixes")
+
+        ######################################################################
+        # Symlinked directory
+        # --------------------------------------------------------------------
+        # - `symlink2` is a symlink directory, pointing at `real`.
+        # - So, `symlink2/bar.h` will be realpath'ed to `real/bar.h`.
+        # - main.c includes `symlink2/bar.h`.
+        # - As a result, the line-table contains a support file `(test_base_dir)/symlink2/bar.h`
+        # - Setting a breakpoint for `real/bar.h` won't be resolved, because it doesn't match the above path.
+        # - Setting a realpath prefix to the current working directory will cause the above support file to be realpath'ed to `(test_base_dir)/real/bar.h`
+        # - Now setting a breakpoint for `real/bar.h` will be resolved.
+        lldbutil.run_break_set_by_file_and_line(
+            self,
+            "real/bar.h",
+            self.line_in_foo,
+            num_expected_locations=0,
+            loc_exact=True,
+        )
+        self.runCmd(f'settings set target.source-realpath-prefixes "{cwd}"')
+        lldbutil.run_break_set_by_file_and_line(
+            self,
+            "real/bar.h",
+            self.line_in_foo,
+            num_expected_locations=1,
+            loc_exact=True,
+        )
+        # Clear settings so that the test below won't be affected.
+        self.runCmd("settings clear target.source-realpath-prefixes")
+
+        ######################################################################
+        # Symlink + source-map
+        # --------------------------------------------------------------------
+        # - `symlink2` is a symlink directory, pointing at `real`.
+        # - So, `symlink2/qux.h` will be realpath'ed to `real/qux.h`.
+        # - main.c includes `symlink2/qux.h`.
+        # - As a result, the line-table contains a support file `(test_base_dir)/symlink2/qux.h`
+        # - Setting a realpath prefix to the current working directory will cause the above support file to be realpath'ed to `(test_base_dir)/real/qux.h`
+        # - Setting a breakpoint for `to-be-mapped/qux.h` won't be resolved, because it doesn't match the above path.
+        # - After setting a source-map, setting the same breakpoint will be resolved, because the input path `to-be-mapped/qux.h` is reverse-mapped to `real/qux.h`, which matches the realpath'ed support file.
+        lldbutil.run_break_set_by_file_and_line(
+            self,
+            "real/qux.h",
+            self.line_in_foo,
+            num_expected_locations=0,
+            loc_exact=True,
+        )
+        self.runCmd(f'settings set target.source-realpath-prefixes "{cwd}"')
+        lldbutil.run_break_set_by_file_and_line(
+            self,
+            "to-be-mapped/qux.h",
+            self.line_in_foo,
+            num_expected_locations=0,
+            loc_exact=True,
+        )
+        self.runCmd('settings set target.source-map "real" "to-be-mapped"')
+        lldbutil.run_break_set_by_file_and_line(
+            self,
+            "to-be-mapped/qux.h",
+            self.line_in_foo,
+            num_expected_locations=1,
+            loc_exact=True,
+        )
+        # Clear settings so that the test below won't be affected.
+        self.runCmd("settings clear target.source-realpath-prefixes")
diff --git a/lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/main.c b/lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/main.c
new file mode 100644
index 000000000000000..716e7a91715f9b2
--- /dev/null
+++ b/lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/main.c
@@ -0,0 +1,10 @@
+#include "symlink1/foo.h"
+#include "symlink2/bar.h"
+#include "symlink2/qux.h"
+
+int main(int argc, char const *argv[]) {
+  int a = foo();    // 1
+  int b = bar();    // 2
+  int c = qux();    // 3
+  return a + b - c; // Set break point at this line.
+}
diff --git a/lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/real/bar.h b/lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/real/bar.h
new file mode 100644
index 000000000000000..40155eb2075ab65
--- /dev/null
+++ b/lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/real/bar.h
@@ -0,0 +1,3 @@
+int bar() {
+  return 2; // Set break point at this line.
+}
diff --git a/lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/real/foo.h b/lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/real/foo.h
new file mode 100644
index 000000000000000..a4ceefecf012389
--- /dev/null
+++ b/lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/real/foo.h
@@ -0,0 +1,3 @@
+int foo() {
+  return 1; // Set break point at this line.
+}
diff --git a/lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/real/qux.h b/lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/real/qux.h
new file mode 100644
index 000000000000000..a90e0feba30aa55
--- /dev/null
+++ b/lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/real/qux.h
@@ -0,0 +1,3 @@
+int qux() {
+  return 3; // Set break point at this line.
+}
diff --git a/lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/symlink1/foo.h b/lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/symlink1/foo.h
new file mode 120000
index 000000000000000..c9001b4753bba67
--- /dev/null
+++ b/lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/symlink1/foo.h
@@ -0,0 +1 @@
+../real/foo.h
\ No newline at end of file
diff --git a/lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/symlink2 b/lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/symlink2
new file mode 120000
index 000000000000000..ac558a3e1bf4442
--- /dev/null
+++ b/lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/symlink2
@@ -0,0 +1 @@
+real
\ No newline at end of file
diff --git a/lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/to-be-mapped/README.md b/lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/to-be-mapped/README.md
new file mode 100644
index 000000000000000..5067af8eed047aa
--- /dev/null
+++ b/lldb/test/API/functionalities/breakpoint/breakpoint_with_realpath_and_source_map/to-be-mapped/README.md
@@ -0,0 +1 @@
+This is an empty folder just so that `settings set target.source-map "real" "to-be-mapped"` can be run successfully - it requires tha the latter path is valid.
diff --git a/lldb/unittests/Core/CMakeLists.txt b/lldb/unittests/Core/CMakeLists.txt
index d40c357e3f463be..949963fd4034639 100644
--- a/lldb/unittests/Core/CMakeLists.txt
+++ b/lldb/unittests/Core/CMakeLists.txt
@@ -3,7 +3,6 @@ add_lldb_unittest(LLDBCoreTests
   DiagnosticEventTest.cpp
   DumpDataExtractorTest.cpp
   DumpRegisterInfoTest.cpp
-  FileSpecListTest.cpp
   FormatEntityTest.cpp
   MangledTest.cpp
   ModuleSpecTest.cpp
diff --git a/lldb/unittests/Core/FileSpecListTest.cpp b/lldb/unittests/Core/FileSpecListTest.cpp
deleted file mode 100644
index e63f4a00bc3a944..000000000000000
--- a/lldb/unittests/Core/FileSpecListTest.cpp
+++ /dev/null
@@ -1,125 +0,0 @@
-//===-- FileSpecListTest.cpp ----------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "gtest/gtest.h"
-
-#include "lldb/Utility/FileSpecList.h"
-
-using namespace lldb_private;
-
-static FileSpec PosixSpec(llvm::StringRef path) {
-  return FileSpec(path, FileSpec::Style::posix);
-}
-
-static FileSpec WindowsSpec(llvm::StringRef path) {
-  return FileSpec(path, FileSpec::Style::windows);
-}
-
-TEST(SupportFileListTest, RelativePathMatchesPosix) {
-
-  const FileSpec fullpath = PosixSpec("/build/src/main.cpp");
-  const FileSpec relative = PosixSpec("./src/main.cpp");
-  const FileSpec basename = PosixSpec("./main.cpp");
-  const FileSpec full_wrong = PosixSpec("/other/wrong/main.cpp");
-  const FileSpec rel_wrong = PosixSpec("./wrong/main.cpp");
-  // Make sure these don't match "src/main.cpp" as we want to match full
-  // directories only
-  const FileSpec rel2_wrong = PosixSpec("asrc/main.cpp");
-  const FileSpec rel3_wrong = PosixSpec("rc/main.cpp");
-
-  SupportFileList files;
-  files.Append(fullpath);
-  files.Append(relative);
-  files.Append(basename);
-  files.Append(full_wrong);
-  files.Append(rel_wrong);
-  files.Append(rel2_wrong);
-  files.Append(rel3_wrong);
-
-  // Make sure the full path only matches the first entry
-  EXPECT_EQ((size_t)0, files.FindCompatibleIndex(0, fullpath));
-  EXPECT_EQ((size_t)1, files.FindCompatibleIndex(1, fullpath));
-  EXPECT_EQ((size_t)2, files.FindCompatibleIndex(2, fullpath));
-  EXPECT_EQ((size_t)UINT32_MAX, files.FindCompatibleIndex(3, fullpath));
-  // Make sure the relative path matches the all of the entries that contain
-  // the relative path
-  EXPECT_EQ((size_t)0, files.FindCompatibleIndex(0, relative));
-  EXPECT_EQ((size_t)1, files.FindCompatibleIndex(1, relative));
-  EXPECT_EQ((size_t)2, files.FindCompatibleIndex(2, relative));
-  EXPECT_EQ((size_t)UINT32_MAX, files.FindCompatibleIndex(3, relative));
-
-  // Make sure looking file a file using the basename matches all entries
-  EXPECT_EQ((size_t)0, files.FindCompatibleIndex(0, basename));
-  EXPECT_EQ((size_t)1, files.FindCompatibleIndex(1, basename));
-  EXPECT_EQ((size_t)2, files.FindCompatibleIndex(2, basename));
-  EXPECT_EQ((size_t)3, files.FindCompatibleIndex(3, basename));
-  EXPECT_EQ((size_t)4, files.FindCompatibleIndex(4, basename));
-  EXPECT_EQ((size_t)5, files.FindCompatibleIndex(5, basename));
-  EXPECT_EQ((size_t)6, files.FindCompatibleIndex(6, basename));
-
-  // Make sure that paths that have a common suffix don't return values that
-  // don't match on directory delimiters.
-  EXPECT_EQ((size_t)2, files.FindCompatibleIndex(0, rel2_wrong));
-  EXPECT_EQ((size_t)5, files.FindCompatibleIndex(3, rel2_wrong));
-  EXPECT_EQ((size_t)UINT32_MAX, files.FindCompatibleIndex(6, rel2_wrong));
-
-  EXPECT_EQ((size_t)2, files.FindCompatibleIndex(0, rel3_wrong));
-  EXPECT_EQ((size_t)6, files.FindCompatibleIndex(3, rel3_wrong));
-}
-
-TEST(SupportFileListTest, RelativePathMatchesWindows) {
-
-  const FileSpec fullpath = WindowsSpec(R"(C:\build\src\main.cpp)");
-  const FileSpec relative = WindowsSpec(R"(.\src\main.cpp)");
-  const FileSpec basename = WindowsSpec(R"(.\main.cpp)");
-  const FileSpec full_wrong = WindowsSpec(R"(\other\wrong\main.cpp)");
-  const FileSpec rel_wrong = WindowsSpec(R"(.\wrong\main.cpp)");
-  // Make sure these don't match "src\main.cpp" as we want to match full
-  // directories only
-  const FileSpec rel2_wrong = WindowsSpec(R"(asrc\main.cpp)");
-  const FileSpec rel3_wrong = WindowsSpec(R"("rc\main.cpp)");
-
-  SupportFileList files;
-  files.Append(fullpath);
-  files.Append(relative);
-  files.Append(basename);
-  files.Append(full_wrong);
-  files.Append(rel_wrong);
-  files.Append(rel2_wrong);
-  files.Append(rel3_wrong);
-
-  // Make sure the full path only matches the first entry
-  EXPECT_EQ((size_t)0, files.FindCompatibleIndex(0, fullpath));
-  EXPECT_EQ((size_t)1, files.FindCompatibleIndex(1, fullpath));
-  EXPECT_EQ((size_t)2, files.FindCompatibleIndex(2, fullpath));
-  EXPECT_EQ((size_t)UINT32_MAX, files.FindCompatibleIndex(3, fullpath));
-  // Make sure the relative path matches the all of the entries that contain
-  // the relative path
-  EXPECT_EQ((size_t)0, files.FindCompatibleIndex(0, relative));
-  EXPECT_EQ((size_t)1, files.FindCompatibleIndex(1, relative));
-  EXPECT_EQ((size_t)2, files.FindCompatibleIndex(2, relative));
-  EXPECT_EQ((size_t)UINT32_MAX, files.FindCompatibleIndex(3, relative));
-
-  // Make sure looking file a file using the basename matches all entries
-  EXPECT_EQ((size_t)0, files.FindCompatibleIndex(0, basename));
-  EXPECT_EQ((size_t)1, files.FindCompatibleIndex(1, basename));
-  EXPECT_EQ((size_t)2, files.FindCompatibleIndex(2, basename));
-  EXPECT_EQ((size_t)3, files.FindCompatibleIndex(3, basename));
-  EXPECT_EQ((size_t)4, files.FindCompatibleIndex(4, basename));
-  EXPECT_EQ((size_t)5, files.FindCompatibleIndex(5, basename));
-  EXPECT_EQ((size_t)6, files.FindCompatibleIndex(6, basename));
-
-  // Make sure that paths that have a common suffix don't return values that
-  // don't match on directory delimiters.
-  EXPECT_EQ((size_t)2, files.FindCompatibleIndex(0, rel2_wrong));
-  EXPECT_EQ((size_t)5, files.FindCompatibleIndex(3, rel2_wrong));
-  EXPECT_EQ((size_t)UINT32_MAX, files.FindCompatibleIndex(6, rel2_wrong));
-
-  EXPECT_EQ((size_t)2, files.FindCompatibleIndex(0, rel3_wrong));
-  EXPECT_EQ((size_t)6, files.FindCompatibleIndex(3, rel3_wrong));
-}
diff --git a/lldb/unittests/Utility/CMakeLists.txt b/lldb/unittests/Utility/CMakeLists.txt
index 8e12815d51541c9..40e0959fc01d142 100644
--- a/lldb/unittests/Utility/CMakeLists.txt
+++ b/lldb/unittests/Utility/CMakeLists.txt
@@ -12,6 +12,7 @@ add_lldb_unittest(UtilityTests
   DataExtractorTest.cpp
   EnvironmentTest.cpp
   EventTest.cpp
+  FileSpecListTest.cpp
   FileSpecTest.cpp
   FlagsTest.cpp
   ListenerTest.cpp
@@ -22,6 +23,7 @@ add_lldb_unittest(UtilityTests
   ProcessInstanceInfoTest.cpp
   RangeMapTest.cpp
   RangeTest.cpp
+  RealpathPrefixesTest.cpp
   RegisterValueTest.cpp
   RegularExpressionTest.cpp
   ScalarTest.cpp
@@ -49,6 +51,7 @@ add_lldb_unittest(UtilityTests
   XcodeSDKTest.cpp
 
   LINK_LIBS
+      lldbTarget
       lldbUtility
       lldbUtilityHelpers
       LLVMTestingSupport
diff --git a/lldb/unittests/Utility/FileSpecListTest.cpp b/lldb/unittests/Utility/FileSpecListTest.cpp
new file mode 100644
index 000000000000000..4c04f434261301a
--- /dev/null
+++ b/lldb/unittests/Utility/FileSpecListTest.cpp
@@ -0,0 +1,323 @@
+//===-- FileSpecListTest.cpp ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "gtest/gtest.h"
+
+#include "MockSymlinkFileSystem.h"
+#include "lldb/Utility/FileSpecList.h"
+#include "lldb/Utility/RealpathPrefixes.h"
+
+using namespace lldb_private;
+
+static FileSpec PosixSpec(llvm::StringRef path) {
+  return FileSpec(path, FileSpec::Style::posix);
+}
+
+static FileSpec WindowsSpec(llvm::StringRef path) {
+  return FileSpec(path, FileSpec::Style::windows);
+}
+
+TEST(SupportFileListTest, RelativePathMatchesPosix) {
+
+  const FileSpec fullpath = PosixSpec("/build/src/main.cpp");
+  const FileSpec relative = PosixSpec("./src/main.cpp");
+  const FileSpec basename = PosixSpec("./main.cpp");
+  const FileSpec full_wrong = PosixSpec("/other/wrong/main.cpp");
+  const FileSpec rel_wrong = PosixSpec("./wrong/main.cpp");
+  // Make sure these don't match "src/main.cpp" as we want to match full
+  // directories only
+  const FileSpec rel2_wrong = PosixSpec("asrc/main.cpp");
+  const FileSpec rel3_wrong = PosixSpec("rc/main.cpp");
+
+  SupportFileList files;
+  files.Append(fullpath);
+  files.Append(relative);
+  files.Append(basename);
+  files.Append(full_wrong);
+  files.Append(rel_wrong);
+  files.Append(rel2_wrong);
+  files.Append(rel3_wrong);
+
+  // Make sure the full path only matches the first entry
+  EXPECT_EQ((size_t)0, files.FindCompatibleIndex(0, fullpath));
+  EXPECT_EQ((size_t)1, files.FindCompatibleIndex(1, fullpath));
+  EXPECT_EQ((size_t)2, files.FindCompatibleIndex(2, fullpath));
+  EXPECT_EQ((size_t)UINT32_MAX, files.FindCompatibleIndex(3, fullpath));
+  // Make sure the relative path matches the all of the entries that contain
+  // the relative path
+  EXPECT_EQ((size_t)0, files.FindCompatibleIndex(0, relative));
+  EXPECT_EQ((size_t)1, files.FindCompatibleIndex(1, relative));
+  EXPECT_EQ((size_t)2, files.FindCompatibleIndex(2, relative));
+  EXPECT_EQ((size_t)UINT32_MAX, files.FindCompatibleIndex(3, relative));
+
+  // Make sure looking file a file using the basename matches all entries
+  EXPECT_EQ((size_t)0, files.FindCompatibleIndex(0, basename));
+  EXPECT_EQ((size_t)1, files.FindCompatibleIndex(1, basename));
+  EXPECT_EQ((size_t)2, files.FindCompatibleIndex(2, basename));
+  EXPECT_EQ((size_t)3, files.FindCompatibleIndex(3, basename));
+  EXPECT_EQ((size_t)4, files.FindCompatibleIndex(4, basename));
+  EXPECT_EQ((size_t)5, files.FindCompatibleIndex(5, basename));
+  EXPECT_EQ((size_t)6, files.FindCompatibleIndex(6, basename));
+
+  // Make sure that paths that have a common suffix don't return values that
+  // don't match on directory delimiters.
+  EXPECT_EQ((size_t)2, files.FindCompatibleIndex(0, rel2_wrong));
+  EXPECT_EQ((size_t)5, files.FindCompatibleIndex(3, rel2_wrong));
+  EXPECT_EQ((size_t)UINT32_MAX, files.FindCompatibleIndex(6, rel2_wrong));
+
+  EXPECT_EQ((size_t)2, files.FindCompatibleIndex(0, rel3_wrong));
+  EXPECT_EQ((size_t)6, files.FindCompatibleIndex(3, rel3_wrong));
+}
+
+TEST(SupportFileListTest, RelativePathMatchesWindows) {
+
+  const FileSpec fullpath = WindowsSpec(R"(C:\build\src\main.cpp)");
+  const FileSpec relative = WindowsSpec(R"(.\src\main.cpp)");
+  const FileSpec basename = WindowsSpec(R"(.\main.cpp)");
+  const FileSpec full_wrong = WindowsSpec(R"(\other\wrong\main.cpp)");
+  const FileSpec rel_wrong = WindowsSpec(R"(.\wrong\main.cpp)");
+  // Make sure these don't match "src\main.cpp" as we want to match full
+  // directories only
+  const FileSpec rel2_wrong = WindowsSpec(R"(asrc\main.cpp)");
+  const FileSpec rel3_wrong = WindowsSpec(R"("rc\main.cpp)");
+
+  SupportFileList files;
+  files.Append(fullpath);
+  files.Append(relative);
+  files.Append(basename);
+  files.Append(full_wrong);
+  files.Append(rel_wrong);
+  files.Append(rel2_wrong);
+  files.Append(rel3_wrong);
+
+  // Make sure the full path only matches the first entry
+  EXPECT_EQ((size_t)0, files.FindCompatibleIndex(0, fullpath));
+  EXPECT_EQ((size_t)1, files.FindCompatibleIndex(1, fullpath));
+  EXPECT_EQ((size_t)2, files.FindCompatibleIndex(2, fullpath));
+  EXPECT_EQ((size_t)UINT32_MAX, files.FindCompatibleIndex(3, fullpath));
+  // Make sure the relative path matches the all of the entries that contain
+  // the relative path
+  EXPECT_EQ((size_t)0, files.FindCompatibleIndex(0, relative));
+  EXPECT_EQ((size_t)1, files.FindCompatibleIndex(1, relative));
+  EXPECT_EQ((size_t)2, files.FindCompatibleIndex(2, relative));
+  EXPECT_EQ((size_t)UINT32_MAX, files.FindCompatibleIndex(3, relative));
+
+  // Make sure looking file a file using the basename matches all entries
+  EXPECT_EQ((size_t)0, files.FindCompatibleIndex(0, basename));
+  EXPECT_EQ((size_t)1, files.FindCompatibleIndex(1, basename));
+  EXPECT_EQ((size_t)2, files.FindCompatibleIndex(2, basename));
+  EXPECT_EQ((size_t)3, files.FindCompatibleIndex(3, basename));
+  EXPECT_EQ((size_t)4, files.FindCompatibleIndex(4, basename));
+  EXPECT_EQ((size_t)5, files.FindCompatibleIndex(5, basename));
+  EXPECT_EQ((size_t)6, files.FindCompatibleIndex(6, basename));
+
+  // Make sure that paths that have a common suffix don't return values that
+  // don't match on directory delimiters.
+  EXPECT_EQ((size_t)2, files.FindCompatibleIndex(0, rel2_wrong));
+  EXPECT_EQ((size_t)5, files.FindCompatibleIndex(3, rel2_wrong));
+  EXPECT_EQ((size_t)UINT32_MAX, files.FindCompatibleIndex(6, rel2_wrong));
+
+  EXPECT_EQ((size_t)2, files.FindCompatibleIndex(0, rel3_wrong));
+  EXPECT_EQ((size_t)6, files.FindCompatibleIndex(3, rel3_wrong));
+}
+
+// Support file is a symlink to the breakpoint file.
+// Absolute paths are used.
+// A matching prefix is set.
+// Should find it compatible.
+TEST(SupportFileListTest, SymlinkedAbsolutePaths) {
+  // Prepare FS
+  llvm::IntrusiveRefCntPtr<MockSymlinkFileSystem> fs(new MockSymlinkFileSystem(
+      FileSpec("/symlink_dir/foo.h"), FileSpec("/real_dir/foo.h")));
+
+  // Prepare RealpathPrefixes
+  FileSpecList file_spec_list;
+  file_spec_list.EmplaceBack("/symlink_dir");
+  RealpathPrefixes prefixes(file_spec_list, fs);
+
+  // Prepare support file list
+  SupportFileList support_file_list;
+  support_file_list.EmplaceBack(FileSpec("/symlink_dir/foo.h"));
+
+  // Test
+  size_t ret = support_file_list.FindCompatibleIndex(
+      0, FileSpec("/real_dir/foo.h"), &prefixes);
+  EXPECT_EQ(ret, (size_t)0);
+}
+
+// Support file is a symlink to the breakpoint file.
+// Absolute paths are used.
+// A matching prefix is set, which is the root directory.
+// Should find it compatible.
+TEST(SupportFileListTest, RootDirectory) {
+  // Prepare FS
+  llvm::IntrusiveRefCntPtr<MockSymlinkFileSystem> fs(new MockSymlinkFileSystem(
+      FileSpec("/symlink_dir/foo.h"), FileSpec("/real_dir/foo.h")));
+
+  // Prepare RealpathPrefixes
+  FileSpecList file_spec_list;
+  file_spec_list.EmplaceBack("/");
+  RealpathPrefixes prefixes(file_spec_list, fs);
+
+  // Prepare support file list
+  SupportFileList support_file_list;
+  support_file_list.EmplaceBack(FileSpec("/symlink_dir/foo.h"));
+
+  // Test
+  size_t ret = support_file_list.FindCompatibleIndex(
+      0, FileSpec("/real_dir/foo.h"), &prefixes);
+  EXPECT_EQ(ret, (size_t)0);
+}
+
+// Support file is a symlink to the breakpoint file.
+// Relative paths are used.
+// A matching prefix is set.
+// Should find it compatible.
+TEST(SupportFileListTest, SymlinkedRelativePaths) {
+  // Prepare FS
+  llvm::IntrusiveRefCntPtr<MockSymlinkFileSystem> fs(new MockSymlinkFileSystem(
+      FileSpec("symlink_dir/foo.h"), FileSpec("real_dir/foo.h")));
+
+  // Prepare RealpathPrefixes
+  FileSpecList file_spec_list;
+  file_spec_list.EmplaceBack("symlink_dir");
+  RealpathPrefixes prefixes(file_spec_list, fs);
+
+  // Prepare support file list
+  SupportFileList support_file_list;
+  support_file_list.EmplaceBack(FileSpec("symlink_dir/foo.h"));
+
+  // Test
+  size_t ret = support_file_list.FindCompatibleIndex(
+      0, FileSpec("real_dir/foo.h"), &prefixes);
+  EXPECT_EQ(ret, (size_t)0);
+}
+
+// Support file is a symlink to the breakpoint file.
+// A matching prefix is set.
+// Input file only match basename and not directory.
+// Should find it incompatible.
+TEST(SupportFileListTest, RealpathOnlyMatchFileName) {
+  // Prepare FS
+  llvm::IntrusiveRefCntPtr<MockSymlinkFileSystem> fs(new MockSymlinkFileSystem(
+      FileSpec("symlink_dir/foo.h"), FileSpec("real_dir/foo.h")));
+
+  // Prepare RealpathPrefixes
+  FileSpecList file_spec_list;
+  file_spec_list.EmplaceBack("symlink_dir");
+  RealpathPrefixes prefixes(file_spec_list, fs);
+
+  // Prepare support file list
+  SupportFileList support_file_list;
+  support_file_list.EmplaceBack(FileSpec("symlink_dir/foo.h"));
+
+  // Test
+  size_t ret = support_file_list.FindCompatibleIndex(
+      0, FileSpec("some_other_dir/foo.h"), &prefixes);
+  EXPECT_EQ(ret, UINT32_MAX);
+}
+
+// Support file is a symlink to the breakpoint file.
+// A prefix is set, which is a matching string prefix, but not a path prefix.
+// Should find it incompatible.
+TEST(SupportFileListTest, DirectoryMatchStringPrefixButNotWholeDirectoryName) {
+  // Prepare FS
+  llvm::IntrusiveRefCntPtr<MockSymlinkFileSystem> fs(new MockSymlinkFileSystem(
+      FileSpec("symlink_dir/foo.h"), FileSpec("real_dir/foo.h")));
+
+  // Prepare RealpathPrefixes
+  FileSpecList file_spec_list;
+  file_spec_list.EmplaceBack("symlink"); // This is a string prefix of the
+                                         // symlink but not a path prefix.
+  RealpathPrefixes prefixes(file_spec_list, fs);
+
+  // Prepare support file list
+  SupportFileList support_file_list;
+  support_file_list.EmplaceBack(FileSpec("symlink_dir/foo.h"));
+
+  // Test
+  size_t ret = support_file_list.FindCompatibleIndex(
+      0, FileSpec("real_dir/foo.h"), &prefixes);
+  EXPECT_EQ(ret, UINT32_MAX);
+}
+
+// Support file is a symlink to the breakpoint file.
+// A matching prefix is set.
+// However, the breakpoint is set with a partial path.
+// Should find it compatible.
+TEST(SupportFileListTest, PartialBreakpointPath) {
+  // Prepare FS
+  llvm::IntrusiveRefCntPtr<MockSymlinkFileSystem> fs(new MockSymlinkFileSystem(
+      FileSpec("symlink_dir/foo.h"), FileSpec("/real_dir/foo.h")));
+
+  // Prepare RealpathPrefixes
+  FileSpecList file_spec_list;
+  file_spec_list.EmplaceBack("symlink_dir");
+  RealpathPrefixes prefixes(file_spec_list, fs);
+
+  // Prepare support file list
+  SupportFileList support_file_list;
+  support_file_list.EmplaceBack(FileSpec("symlink_dir/foo.h"));
+
+  // Test
+  size_t ret = support_file_list.FindCompatibleIndex(
+      0, FileSpec("real_dir/foo.h"), &prefixes);
+  EXPECT_EQ(ret, (size_t)0);
+}
+
+// Support file is a symlink to the breakpoint file.
+// A matching prefix is set.
+// However, the basename is different between the symlink and its target.
+// Should find it incompatible.
+TEST(SupportFileListTest, DifferentBasename) {
+  // Prepare FS
+  llvm::IntrusiveRefCntPtr<MockSymlinkFileSystem> fs(new MockSymlinkFileSystem(
+      FileSpec("/symlink_dir/foo.h"), FileSpec("/real_dir/bar.h")));
+
+  // Prepare RealpathPrefixes
+  FileSpecList file_spec_list;
+  file_spec_list.EmplaceBack("/symlink_dir");
+  RealpathPrefixes prefixes(file_spec_list, fs);
+
+  // Prepare support file list
+  SupportFileList support_file_list;
+  support_file_list.EmplaceBack(FileSpec("/symlink_dir/foo.h"));
+
+  // Test
+  size_t ret = support_file_list.FindCompatibleIndex(
+      0, FileSpec("real_dir/bar.h"), &prefixes);
+  EXPECT_EQ(ret, UINT32_MAX);
+}
+
+// No prefixes are configured.
+// The support file and the breakpoint file are different.
+// Should find it incompatible.
+TEST(SupportFileListTest, NoPrefixes) {
+  // Prepare support file list
+  SupportFileList support_file_list;
+  support_file_list.EmplaceBack(FileSpec("/real_dir/bar.h"));
+
+  // Test
+  size_t ret = support_file_list.FindCompatibleIndex(
+      0, FileSpec("/real_dir/foo.h"), nullptr);
+  EXPECT_EQ(ret, UINT32_MAX);
+}
+
+// No prefixes are configured.
+// The support file and the breakpoint file are the same.
+// Should find it compatible.
+TEST(SupportFileListTest, SameFile) {
+  // Prepare support file list
+  SupportFileList support_file_list;
+  support_file_list.EmplaceBack(FileSpec("/real_dir/foo.h"));
+
+  // Test
+  size_t ret = support_file_list.FindCompatibleIndex(
+      0, FileSpec("/real_dir/foo.h"), nullptr);
+  EXPECT_EQ(ret, (size_t)0);
+}
diff --git a/lldb/unittests/Utility/MockSymlinkFileSystem.h b/lldb/unittests/Utility/MockSymlinkFileSystem.h
new file mode 100644
index 000000000000000..7fa1f93bfa38a9a
--- /dev/null
+++ b/lldb/unittests/Utility/MockSymlinkFileSystem.h
@@ -0,0 +1,66 @@
+//===-- MockSymlinkFileSystem.h
+//--------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "lldb/Utility/FileSpec.h"
+#include "llvm/Support/VirtualFileSystem.h"
+
+namespace lldb_private {
+
+// A mock file system that realpath's a given symlink to a given realpath.
+class MockSymlinkFileSystem : public llvm::vfs::FileSystem {
+public:
+  // Treat all files as non-symlinks.
+  MockSymlinkFileSystem() = default;
+
+  /// Treat \a symlink as a symlink to \a realpath. Treat all other files as
+  /// non-symlinks.
+  MockSymlinkFileSystem(FileSpec &&symlink, FileSpec &&realpath,
+                        FileSpec::Style style = FileSpec::Style::native)
+      : m_symlink(std::move(symlink)), m_realpath(std::move(realpath)),
+        m_style(style) {}
+
+  /// If \a Path matches the symlink given in the ctor, put the realpath given
+  /// in the ctor into \a Output.
+  std::error_code getRealPath(const llvm::Twine &Path,
+                              llvm::SmallVectorImpl<char> &Output) override {
+    if (FileSpec(Path.str(), m_style) == m_symlink) {
+      std::string path = m_realpath.GetPath();
+      Output.assign(path.begin(), path.end());
+    } else {
+      Path.toVector(Output);
+    }
+    return {};
+  }
+
+  // Implement the rest of the interface
+  llvm::ErrorOr<llvm::vfs::Status> status(const llvm::Twine &Path) override {
+    return llvm::errc::operation_not_permitted;
+  }
+  llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
+  openFileForRead(const llvm::Twine &Path) override {
+    return llvm::errc::operation_not_permitted;
+  }
+  llvm::vfs::directory_iterator dir_begin(const llvm::Twine &Dir,
+                                          std::error_code &EC) override {
+    return llvm::vfs::directory_iterator();
+  }
+  std::error_code setCurrentWorkingDirectory(const llvm::Twine &Path) override {
+    return llvm::errc::operation_not_permitted;
+  }
+  llvm::ErrorOr<std::string> getCurrentWorkingDirectory() const override {
+    return llvm::errc::operation_not_permitted;
+  }
+
+private:
+  FileSpec m_symlink;
+  FileSpec m_realpath;
+  FileSpec::Style m_style;
+};
+
+} // namespace lldb_private
diff --git a/lldb/unittests/Utility/RealpathPrefixesTest.cpp b/lldb/unittests/Utility/RealpathPrefixesTest.cpp
new file mode 100644
index 000000000000000..872ddf1fd223aa9
--- /dev/null
+++ b/lldb/unittests/Utility/RealpathPrefixesTest.cpp
@@ -0,0 +1,140 @@
+//===-- RealpathPrefixesTest.cpp
+//--------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "gtest/gtest.h"
+
+#include "MockSymlinkFileSystem.h"
+#include "lldb/Utility/FileSpecList.h"
+#include "lldb/Utility/RealpathPrefixes.h"
+
+using namespace lldb_private;
+
+// Should resolve a symlink which match an absolute prefix
+TEST(RealpathPrefixesTest, MatchingAbsolutePrefix) {
+  // Prepare FS
+  llvm::IntrusiveRefCntPtr<MockSymlinkFileSystem> fs(new MockSymlinkFileSystem(
+      FileSpec("/dir1/link.h"), FileSpec("/dir2/real.h")));
+
+  // Prepare RealpathPrefixes
+  FileSpecList file_spec_list;
+  file_spec_list.EmplaceBack("/dir1");
+  RealpathPrefixes prefixes(file_spec_list, fs);
+
+  // Test
+  std::optional<FileSpec> ret =
+      prefixes.ResolveSymlinks(FileSpec("/dir1/link.h"));
+  EXPECT_EQ(ret, FileSpec("/dir2/real.h"));
+}
+
+// Should resolve a symlink which match a relative prefix
+TEST(RealpathPrefixesTest, MatchingRelativePrefix) {
+  // Prepare FS
+  llvm::IntrusiveRefCntPtr<MockSymlinkFileSystem> fs(new MockSymlinkFileSystem(
+      FileSpec("dir1/link.h"), FileSpec("dir2/real.h")));
+
+  // Prepare RealpathPrefixes
+  FileSpecList file_spec_list;
+  file_spec_list.EmplaceBack("dir1");
+  RealpathPrefixes prefixes(file_spec_list, fs);
+
+  // Test
+  std::optional<FileSpec> ret =
+      prefixes.ResolveSymlinks(FileSpec("dir1/link.h"));
+  EXPECT_EQ(ret, FileSpec("dir2/real.h"));
+}
+
+// Should resolve in Windows and/or with a case-insensitive support file
+TEST(RealpathPrefixesTest, WindowsAndCaseInsensitive) {
+  // Prepare FS
+  llvm::IntrusiveRefCntPtr<MockSymlinkFileSystem> fs(new MockSymlinkFileSystem(
+      FileSpec("f:\\dir1\\link.h", FileSpec::Style::windows),
+      FileSpec("f:\\dir2\\real.h", FileSpec::Style::windows),
+      FileSpec::Style::windows));
+
+  // Prepare RealpathPrefixes
+  FileSpecList file_spec_list;
+  file_spec_list.EmplaceBack(FileSpec("f:\\dir1", FileSpec::Style::windows));
+  RealpathPrefixes prefixes(file_spec_list, fs);
+
+  // Test
+  std::optional<FileSpec> ret = prefixes.ResolveSymlinks(
+      FileSpec("F:\\DIR1\\LINK.H", FileSpec::Style::windows));
+  EXPECT_EQ(ret, FileSpec("f:\\dir2\\real.h", FileSpec::Style::windows));
+}
+
+// Should resolve a symlink when there is mixture of matching and mismatching
+// prefixex
+TEST(RealpathPrefixesTest, MatchingAndMismatchingPrefix) {
+  // Prepare FS
+  llvm::IntrusiveRefCntPtr<MockSymlinkFileSystem> fs(new MockSymlinkFileSystem(
+      FileSpec("/dir1/link.h"), FileSpec("/dir2/real.h")));
+
+  // Prepare RealpathPrefixes
+  FileSpecList file_spec_list;
+  file_spec_list.EmplaceBack("/fake/path1");
+  file_spec_list.EmplaceBack("/dir1"); // Matching prefix
+  file_spec_list.EmplaceBack("/fake/path2");
+  RealpathPrefixes prefixes(file_spec_list, fs);
+
+  // Test
+  std::optional<FileSpec> ret =
+      prefixes.ResolveSymlinks(FileSpec("/dir1/link.h"));
+  EXPECT_EQ(ret, FileSpec("/dir2/real.h"));
+}
+
+// Should resolve a symlink when the prefixes matches after normalization
+TEST(RealpathPrefixesTest, ComplexPrefixes) {
+  // Prepare FS
+  llvm::IntrusiveRefCntPtr<MockSymlinkFileSystem> fs(new MockSymlinkFileSystem(
+      FileSpec("dir1/link.h"), FileSpec("dir2/real.h")));
+
+  // Prepare RealpathPrefixes
+  FileSpecList file_spec_list;
+  file_spec_list.EmplaceBack("./dir1/foo/../bar/.."); // Equivalent to "/dir1"
+  RealpathPrefixes prefixes(file_spec_list, fs);
+
+  // Test
+  std::optional<FileSpec> ret =
+      prefixes.ResolveSymlinks(FileSpec("dir1/link.h"));
+  EXPECT_EQ(ret, FileSpec("dir2/real.h"));
+}
+
+// Should not resolve a symlink which doesn't match any prefixes
+TEST(RealpathPrefixesTest, MismatchingPrefixes) {
+  // Prepare FS
+  llvm::IntrusiveRefCntPtr<MockSymlinkFileSystem> fs(new MockSymlinkFileSystem(
+      FileSpec("/dir1/link.h"), FileSpec("/dir2/real.h")));
+
+  // Prepare RealpathPrefixes
+  FileSpecList file_spec_list;
+  file_spec_list.EmplaceBack("/dir3");
+  RealpathPrefixes prefixes(file_spec_list, fs);
+
+  // Test
+  std::optional<FileSpec> ret =
+      prefixes.ResolveSymlinks(FileSpec("/dir1/link.h"));
+  EXPECT_EQ(ret, std::nullopt);
+}
+
+// Should not resolve a realpath
+TEST(RealpathPrefixesTest, Realpath) {
+  // Prepare FS
+  llvm::IntrusiveRefCntPtr<MockSymlinkFileSystem> fs(
+      new MockSymlinkFileSystem());
+
+  // Prepare RealpathPrefixes
+  FileSpecList file_spec_list;
+  file_spec_list.EmplaceBack("/symlink_dir");
+  RealpathPrefixes prefixes(file_spec_list, fs);
+
+  // Test
+  std::optional<FileSpec> ret =
+      prefixes.ResolveSymlinks(FileSpec("/dir/real.h"));
+  EXPECT_EQ(ret, std::nullopt);
+}

From 7ab2d504a1b30ce7a1338d23b0f443c8b24f94c4 Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Thu, 15 Aug 2024 18:26:52 +0000
Subject: [PATCH 061/441] [gn build] Port 47721d46187f

---
 llvm/utils/gn/secondary/lldb/source/Utility/BUILD.gn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/utils/gn/secondary/lldb/source/Utility/BUILD.gn b/llvm/utils/gn/secondary/lldb/source/Utility/BUILD.gn
index 07b0a7ac66823ca..5faa365bb7bdbcd 100644
--- a/llvm/utils/gn/secondary/lldb/source/Utility/BUILD.gn
+++ b/llvm/utils/gn/secondary/lldb/source/Utility/BUILD.gn
@@ -35,6 +35,7 @@ static_library("Utility") {
     "Log.cpp",
     "NameMatches.cpp",
     "ProcessInfo.cpp",
+    "RealpathPrefixes.cpp",
     "RegisterValue.cpp",
     "RegularExpression.cpp",
     "Scalar.cpp",

From 6cbd96e24c6a60cbc3dbb849d2ed7afc39c77a80 Mon Sep 17 00:00:00 2001
From: "Oleksandr T." <oleksandr.tarasiuk@outlook.com>
Date: Thu, 15 Aug 2024 21:42:39 +0300
Subject: [PATCH 062/441] [Clang] handle both gnu and cpp11 attributes to
 ensure correct parsing inside extern block (#102864)

Fixes #101990
---
 clang/docs/ReleaseNotes.rst      |  1 +
 clang/lib/Parse/ParseDeclCXX.cpp |  5 ++++-
 clang/test/Parser/attr-order.cpp | 13 +++++++++++++
 3 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 5ba9fcb040e3a61..68cf79928bda021 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -257,6 +257,7 @@ Bug Fixes to C++ Support
 - Properly reject defaulted relational operators with invalid types for explicit object parameters,
   e.g., ``bool operator==(this int, const Foo&)`` (#GH100329), and rvalue reference parameters.
 - Properly reject defaulted copy/move assignment operators that have a non-reference explicit object parameter.
+- Clang now properly handles the order of attributes in `extern` blocks. (#GH101990).
 - Fixed an assertion failure by preventing null explicit object arguments from being deduced. (#GH102025).
 
 Bug Fixes to AST Handling
diff --git a/clang/lib/Parse/ParseDeclCXX.cpp b/clang/lib/Parse/ParseDeclCXX.cpp
index aac89d910bbc839..d45a738fe4c5969 100644
--- a/clang/lib/Parse/ParseDeclCXX.cpp
+++ b/clang/lib/Parse/ParseDeclCXX.cpp
@@ -425,7 +425,10 @@ Decl *Parser::ParseLinkage(ParsingDeclSpec &DS, DeclaratorContext Context) {
       [[fallthrough]];
     default:
       ParsedAttributes DeclAttrs(AttrFactory);
-      MaybeParseCXX11Attributes(DeclAttrs);
+      ParsedAttributes DeclSpecAttrs(AttrFactory);
+      while (MaybeParseCXX11Attributes(DeclAttrs) ||
+             MaybeParseGNUAttributes(DeclSpecAttrs))
+        ;
       ParseExternalDeclaration(DeclAttrs, DeclSpecAttrs);
       continue;
     }
diff --git a/clang/test/Parser/attr-order.cpp b/clang/test/Parser/attr-order.cpp
index 10bad38cac64470..21e30e5b6f9df1a 100644
--- a/clang/test/Parser/attr-order.cpp
+++ b/clang/test/Parser/attr-order.cpp
@@ -31,3 +31,16 @@ template <int a>
 
 template <int a>
 [[noreturn]] __declspec(dllexport) __attribute__((cdecl)) void k(); // ok
+
+extern "C" {
+  __attribute__ ((__warn_unused_result__)) [[__maybe_unused__]] int l(int); // ok
+  [[__maybe_unused__]] __attribute__ ((__warn_unused_result__)) int m(int); // ok
+}
+
+extern "C" {
+  __attribute__ ((__warn_unused_result__)) [[__maybe_unused__]] int n (int); // ok
+  __attribute__ ((__warn_unused_result__)) [[__maybe_unused__]] static int o (int x) { return x; }; // ok
+}
+
+extern "C" __attribute__ ((__warn_unused_result__)) [[__maybe_unused__]] int p(int); // ok
+extern "C" [[__maybe_unused__]] __attribute__ ((__warn_unused_result__)) int q(int); // ok

From 85da39debd8ee8c3186d88b462a924ea57b812ec Mon Sep 17 00:00:00 2001
From: Nikolas Klauser <nikolasklauser@berlin.de>
Date: Thu, 15 Aug 2024 20:45:49 +0200
Subject: [PATCH 063/441] [libc++] Remove the allocator<const T> extension
 (#102655)

In LLVM 19 removed the extension with an opt-in macro. This finally
removes that option too and removes a few `const_cast`s where I know
that they exist only to support this extension.
---
 libcxx/docs/ReleaseNotes/20.rst               |  2 +-
 libcxx/include/__memory/allocator.h           | 95 -------------------
 .../__memory/uninitialized_algorithms.h       | 16 ++--
 3 files changed, 7 insertions(+), 106 deletions(-)

diff --git a/libcxx/docs/ReleaseNotes/20.rst b/libcxx/docs/ReleaseNotes/20.rst
index b319067394099a2..fe9f4c1973cdb43 100644
--- a/libcxx/docs/ReleaseNotes/20.rst
+++ b/libcxx/docs/ReleaseNotes/20.rst
@@ -62,7 +62,7 @@ Deprecations and Removals
   ``_LIBCPP_ENABLE_REMOVED_WEEKDAY_RELATIONAL_OPERATORS`` macro that was used to re-enable this extension will be
   ignored in LLVM 20.
 
-- TODO: The ``_LIBCPP_ENABLE_REMOVED_ALLOCATOR_CONST`` macro will no longer have an effect.
+- The ``_LIBCPP_ENABLE_REMOVED_ALLOCATOR_CONST`` macro no longer has any effect. ``std::allocator<const T>`` is not supported as an extension anymore, please migrate any code that uses e.g. ``std::vector<const T>`` to be standards conforming.
 
 Upcoming Deprecations and Removals
 ----------------------------------
diff --git a/libcxx/include/__memory/allocator.h b/libcxx/include/__memory/allocator.h
index ae1f549626ee49e..0dbdc41d3c3d148 100644
--- a/libcxx/include/__memory/allocator.h
+++ b/libcxx/include/__memory/allocator.h
@@ -47,22 +47,6 @@ class _LIBCPP_TEMPLATE_VIS allocator<void> {
     typedef allocator<_Up> other;
   };
 };
-
-// TODO(LLVM 20): Remove the escape hatch
-#  ifdef _LIBCPP_ENABLE_REMOVED_ALLOCATOR_CONST
-template <>
-class _LIBCPP_TEMPLATE_VIS allocator<const void> {
-public:
-  _LIBCPP_DEPRECATED_IN_CXX17 typedef const void* pointer;
-  _LIBCPP_DEPRECATED_IN_CXX17 typedef const void* const_pointer;
-  _LIBCPP_DEPRECATED_IN_CXX17 typedef const void value_type;
-
-  template <class _Up>
-  struct _LIBCPP_DEPRECATED_IN_CXX17 rebind {
-    typedef allocator<_Up> other;
-  };
-};
-#  endif // _LIBCPP_ENABLE_REMOVED_ALLOCATOR_CONST
 #endif   // _LIBCPP_STD_VER <= 17
 
 // This class provides a non-trivial default constructor to the class that derives from it
@@ -171,85 +155,6 @@ class _LIBCPP_TEMPLATE_VIS allocator : private __non_trivial_if<!is_void<_Tp>::v
 #endif
 };
 
-// TODO(LLVM 20): Remove the escape hatch
-#ifdef _LIBCPP_ENABLE_REMOVED_ALLOCATOR_CONST
-template <class _Tp>
-class _LIBCPP_TEMPLATE_VIS allocator<const _Tp>
-    : private __non_trivial_if<!is_void<_Tp>::value, allocator<const _Tp> > {
-  static_assert(!is_volatile<_Tp>::value, "std::allocator does not support volatile types");
-
-public:
-  typedef size_t size_type;
-  typedef ptrdiff_t difference_type;
-  typedef const _Tp value_type;
-  typedef true_type propagate_on_container_move_assignment;
-#  if _LIBCPP_STD_VER <= 23 || defined(_LIBCPP_ENABLE_CXX26_REMOVED_ALLOCATOR_MEMBERS)
-  _LIBCPP_DEPRECATED_IN_CXX23 typedef true_type is_always_equal;
-#  endif
-
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 allocator() _NOEXCEPT = default;
-
-  template <class _Up>
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 allocator(const allocator<_Up>&) _NOEXCEPT {}
-
-  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 const _Tp* allocate(size_t __n) {
-    if (__n > allocator_traits<allocator>::max_size(*this))
-      __throw_bad_array_new_length();
-    if (__libcpp_is_constant_evaluated()) {
-      return static_cast<const _Tp*>(::operator new(__n * sizeof(_Tp)));
-    } else {
-      return static_cast<const _Tp*>(std::__libcpp_allocate(__n * sizeof(_Tp), _LIBCPP_ALIGNOF(_Tp)));
-    }
-  }
-
-#  if _LIBCPP_STD_VER >= 23
-  [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr allocation_result<const _Tp*> allocate_at_least(size_t __n) {
-    return {allocate(__n), __n};
-  }
-#  endif
-
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void deallocate(const _Tp* __p, size_t __n) {
-    if (__libcpp_is_constant_evaluated()) {
-      ::operator delete(const_cast<_Tp*>(__p));
-    } else {
-      std::__libcpp_deallocate((void*)const_cast<_Tp*>(__p), __n * sizeof(_Tp), _LIBCPP_ALIGNOF(_Tp));
-    }
-  }
-
-  // C++20 Removed members
-#  if _LIBCPP_STD_VER <= 17
-  _LIBCPP_DEPRECATED_IN_CXX17 typedef const _Tp* pointer;
-  _LIBCPP_DEPRECATED_IN_CXX17 typedef const _Tp* const_pointer;
-  _LIBCPP_DEPRECATED_IN_CXX17 typedef const _Tp& reference;
-  _LIBCPP_DEPRECATED_IN_CXX17 typedef const _Tp& const_reference;
-
-  template <class _Up>
-  struct _LIBCPP_DEPRECATED_IN_CXX17 rebind {
-    typedef allocator<_Up> other;
-  };
-
-  _LIBCPP_DEPRECATED_IN_CXX17 _LIBCPP_HIDE_FROM_ABI const_pointer address(const_reference __x) const _NOEXCEPT {
-    return std::addressof(__x);
-  }
-
-  _LIBCPP_NODISCARD _LIBCPP_HIDE_FROM_ABI _LIBCPP_DEPRECATED_IN_CXX17 const _Tp* allocate(size_t __n, const void*) {
-    return allocate(__n);
-  }
-
-  _LIBCPP_DEPRECATED_IN_CXX17 _LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT {
-    return size_type(~0) / sizeof(_Tp);
-  }
-
-  template <class _Up, class... _Args>
-  _LIBCPP_DEPRECATED_IN_CXX17 _LIBCPP_HIDE_FROM_ABI void construct(_Up* __p, _Args&&... __args) {
-    ::new ((void*)__p) _Up(std::forward<_Args>(__args)...);
-  }
-
-  _LIBCPP_DEPRECATED_IN_CXX17 _LIBCPP_HIDE_FROM_ABI void destroy(pointer __p) { __p->~_Tp(); }
-#  endif
-};
-#endif // _LIBCPP_ENABLE_REMOVED_ALLOCATOR_CONST
-
 template <class _Tp, class _Up>
 inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool
 operator==(const allocator<_Tp>&, const allocator<_Up>&) _NOEXCEPT {
diff --git a/libcxx/include/__memory/uninitialized_algorithms.h b/libcxx/include/__memory/uninitialized_algorithms.h
index 7475ef5cf85def4..72db3a266fdd4d9 100644
--- a/libcxx/include/__memory/uninitialized_algorithms.h
+++ b/libcxx/include/__memory/uninitialized_algorithms.h
@@ -562,17 +562,13 @@ struct __allocator_has_trivial_copy_construct<allocator<_Type>, _Type> : true_ty
 
 template <class _Alloc,
           class _In,
-          class _RawTypeIn = __remove_const_t<_In>,
           class _Out,
-          __enable_if_t<
-              // using _RawTypeIn because of the allocator<T const> extension
-              is_trivially_copy_constructible<_RawTypeIn>::value && is_trivially_copy_assignable<_RawTypeIn>::value &&
-                  is_same<__remove_const_t<_In>, __remove_const_t<_Out> >::value &&
-                  __allocator_has_trivial_copy_construct<_Alloc, _RawTypeIn>::value,
-              int> = 0>
+          __enable_if_t<is_trivially_copy_constructible<_In>::value && is_trivially_copy_assignable<_In>::value &&
+                            is_same<__remove_const_t<_In>, __remove_const_t<_Out> >::value &&
+                            __allocator_has_trivial_copy_construct<_Alloc, _In>::value,
+                        int> = 0>
 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Out*
 __uninitialized_allocator_copy_impl(_Alloc&, _In* __first1, _In* __last1, _Out* __first2) {
-  // TODO: Remove the const_cast once we drop support for std::allocator<T const>
   if (__libcpp_is_constant_evaluated()) {
     while (__first1 != __last1) {
       std::__construct_at(std::__to_address(__first2), *__first1);
@@ -581,7 +577,7 @@ __uninitialized_allocator_copy_impl(_Alloc&, _In* __first1, _In* __last1, _Out*
     }
     return __first2;
   } else {
-    return std::copy(__first1, __last1, const_cast<_RawTypeIn*>(__first2));
+    return std::copy(__first1, __last1, __first2);
   }
 }
 
@@ -642,7 +638,7 @@ __uninitialized_allocator_relocate(_Alloc& __alloc, _Tp* __first, _Tp* __last, _
     __guard.__complete();
     std::__allocator_destroy(__alloc, __first, __last);
   } else {
-    __builtin_memcpy(const_cast<__remove_const_t<_Tp>*>(__result), __first, sizeof(_Tp) * (__last - __first));
+    __builtin_memcpy(__result, __first, sizeof(_Tp) * (__last - __first));
   }
 }
 

From 8ffdc8765661b2f6e6fed32de9fd95c76ff7dc9f Mon Sep 17 00:00:00 2001
From: Keith Smiley <keithbsmiley@gmail.com>
Date: Thu, 15 Aug 2024 11:50:16 -0700
Subject: [PATCH 064/441] [bazel] Port 47721d46187f89c12a13d07b5857496301cf5d6e
 (#104481)

Made more difficult by many header circular dependencies
---
 .../bazel/llvm-project-overlay/lldb/BUILD.bazel  | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel b/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel
index b4889bbd46addd8..7841045afa6d219 100644
--- a/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel
@@ -564,7 +564,7 @@ cc_library(
     strip_include_prefix = "include",
     deps = [
         ":Headers",
-        ":Utility",
+        ":UtilityHeaders",
         "//llvm:Object",
         "//llvm:Support",
         "//llvm:TargetParser",
@@ -617,7 +617,6 @@ cc_library(
         ":InterpreterHeaders",
         ":SymbolHeaders",
         ":TargetHeaders",
-        ":Utility",
         "//clang:driver",
         "//llvm:Demangle",
         "//llvm:Support",
@@ -679,7 +678,10 @@ cc_library(
     name = "TargetHeaders",
     hdrs = glob(["include/lldb/Target/**/*.h"]),
     strip_include_prefix = "include",
-    deps = [":AppleArm64ExceptionClass"],
+    deps = [
+        ":AppleArm64ExceptionClass",
+        ":BreakpointHeaders",
+    ],
 )
 
 cc_library(
@@ -721,13 +723,21 @@ cc_library(
     deps = [":Headers"],
 )
 
+cc_library(
+    name = "UtilityHeaders",
+    hdrs = glob(["include/lldb/Utility/**/*.h"]),
+    strip_include_prefix = "include",
+)
+
 cc_library(
     name = "Utility",
     srcs = glob(["source/Utility/**/*.cpp"]),
     hdrs = glob(["include/lldb/Utility/**/*.h"]),
     strip_include_prefix = "include",
     deps = [
+        ":CoreHeaders",
         ":Headers",
+        ":TargetHeaders",
         ":UtilityPrivateHeaders",
         "//llvm:BinaryFormat",
         "//llvm:Support",

From 6bbbd301473a14a52d7ea1c5dae38ee20f97f1f2 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Thu, 15 Aug 2024 11:56:21 -0700
Subject: [PATCH 065/441] [llvm-objdump] Fix a warning

This patch fixes:

  llvm/tools/llvm-objdump/XCOFFDump.cpp:85:12: error: unused variable
  'BytesFormatted' [-Werror,-Wunused-variable]
---
 llvm/tools/llvm-objdump/XCOFFDump.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/tools/llvm-objdump/XCOFFDump.cpp b/llvm/tools/llvm-objdump/XCOFFDump.cpp
index 5617313cdbf721b..524bf07c372a9bc 100644
--- a/llvm/tools/llvm-objdump/XCOFFDump.cpp
+++ b/llvm/tools/llvm-objdump/XCOFFDump.cpp
@@ -86,6 +86,7 @@ void XCOFFDumper::printFileHeader() {
                                           "%F %T", std::gmtime(&TimeDate));
     assert(BytesFormatted && "The size of the buffer FormattedTime is less "
                              "than the size of the date/time string.");
+    (void)BytesFormatted;
     printStrHex("Timestamp:", FormattedTime, Timestamp);
   } else {
     // Negative timestamp values are reserved for future use.

From b5e63cc533b5e752eb475ac657f09b9bb5eb2373 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Thu, 15 Aug 2024 22:59:24 +0400
Subject: [PATCH 066/441] clang/AMDGPU: Emit atomicrmw for
 __builtin_amdgcn_global_atomic_fadd_{f32|f64} (#96872)

Need to emit syncscope and new metadata to get the native instruction,
most of the time.
---
 clang/lib/CodeGen/CGBuiltin.cpp               | 41 +++++++++++++------
 .../CodeGenOpenCL/builtins-amdgcn-gfx11.cl    |  2 +-
 .../builtins-fp-atomics-gfx12.cl              |  8 ++--
 .../CodeGenOpenCL/builtins-fp-atomics-gfx8.cl |  4 +-
 .../builtins-fp-atomics-gfx90a.cl             |  8 ++--
 .../builtins-fp-atomics-gfx940.cl             |  8 ++--
 6 files changed, 43 insertions(+), 28 deletions(-)

diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index 1c0baeaee036321..f424ddaa175400c 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -58,6 +58,7 @@
 #include "llvm/IR/MDBuilder.h"
 #include "llvm/IR/MatrixBuilder.h"
 #include "llvm/IR/MemoryModelRelaxationAnnotations.h"
+#include "llvm/Support/AMDGPUAddrSpace.h"
 #include "llvm/Support/ConvertUTF.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/ScopedPrinter.h"
@@ -18919,8 +18920,6 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
     Function *F = CGM.getIntrinsic(Intrin, { Src0->getType() });
     return Builder.CreateCall(F, { Src0, Builder.getFalse() });
   }
-  case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f64:
-  case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f32:
   case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_v2f16:
   case AMDGPU::BI__builtin_amdgcn_global_atomic_fmin_f64:
   case AMDGPU::BI__builtin_amdgcn_global_atomic_fmax_f64:
@@ -18932,18 +18931,11 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
     Intrinsic::ID IID;
     llvm::Type *ArgTy = llvm::Type::getDoubleTy(getLLVMContext());
     switch (BuiltinID) {
-    case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f32:
-      ArgTy = llvm::Type::getFloatTy(getLLVMContext());
-      IID = Intrinsic::amdgcn_global_atomic_fadd;
-      break;
     case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_v2f16:
       ArgTy = llvm::FixedVectorType::get(
           llvm::Type::getHalfTy(getLLVMContext()), 2);
       IID = Intrinsic::amdgcn_global_atomic_fadd;
       break;
-    case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f64:
-      IID = Intrinsic::amdgcn_global_atomic_fadd;
-      break;
     case AMDGPU::BI__builtin_amdgcn_global_atomic_fmin_f64:
       IID = Intrinsic::amdgcn_global_atomic_fmin;
       break;
@@ -19366,7 +19358,9 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
   case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_v2bf16:
   case AMDGPU::BI__builtin_amdgcn_ds_faddf:
   case AMDGPU::BI__builtin_amdgcn_ds_fminf:
-  case AMDGPU::BI__builtin_amdgcn_ds_fmaxf: {
+  case AMDGPU::BI__builtin_amdgcn_ds_fmaxf:
+  case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f32:
+  case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f64: {
     llvm::AtomicRMWInst::BinOp BinOp;
     switch (BuiltinID) {
     case AMDGPU::BI__builtin_amdgcn_atomic_inc32:
@@ -19382,6 +19376,8 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
     case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_f32:
     case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_v2f16:
     case AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_v2bf16:
+    case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f32:
+    case AMDGPU::BI__builtin_amdgcn_global_atomic_fadd_f64:
       BinOp = llvm::AtomicRMWInst::FAdd;
       break;
     case AMDGPU::BI__builtin_amdgcn_ds_fminf:
@@ -19416,9 +19412,14 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
       ProcessOrderScopeAMDGCN(EmitScalarExpr(E->getArg(2)),
                               EmitScalarExpr(E->getArg(3)), AO, SSID);
     } else {
-      // The ds_atomic_fadd_* builtins do not have syncscope/order arguments.
-      SSID = llvm::SyncScope::System;
-      AO = AtomicOrdering::SequentiallyConsistent;
+      // Most of the builtins do not have syncscope/order arguments. For DS
+      // atomics the scope doesn't really matter, as they implicitly operate at
+      // workgroup scope.
+      //
+      // The global/flat cases need to use agent scope to consistently produce
+      // the native instruction instead of a cmpxchg expansion.
+      SSID = getLLVMContext().getOrInsertSyncScopeID("agent");
+      AO = AtomicOrdering::Monotonic;
 
       // The v2bf16 builtin uses i16 instead of a natural bfloat type.
       if (BuiltinID == AMDGPU::BI__builtin_amdgcn_ds_atomic_fadd_v2bf16) {
@@ -19432,6 +19433,20 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
         Builder.CreateAtomicRMW(BinOp, Ptr, Val, AO, SSID);
     if (Volatile)
       RMW->setVolatile(true);
+
+    unsigned AddrSpace = Ptr.getType()->getAddressSpace();
+    if (AddrSpace != llvm::AMDGPUAS::LOCAL_ADDRESS) {
+      // Most targets require "amdgpu.no.fine.grained.memory" to emit the native
+      // instruction for flat and global operations.
+      llvm::MDTuple *EmptyMD = MDNode::get(getLLVMContext(), {});
+      RMW->setMetadata("amdgpu.no.fine.grained.memory", EmptyMD);
+
+      // Most targets require "amdgpu.ignore.denormal.mode" to emit the native
+      // instruction, but this only matters for float fadd.
+      if (BinOp == llvm::AtomicRMWInst::FAdd && Val->getType()->isFloatTy())
+        RMW->setMetadata("amdgpu.ignore.denormal.mode", EmptyMD);
+    }
+
     return Builder.CreateBitCast(RMW, OrigTy);
   }
   case AMDGPU::BI__builtin_amdgcn_s_sendmsg_rtn:
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx11.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx11.cl
index f44465746356878..138616ccca7182a 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx11.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx11.cl
@@ -49,7 +49,7 @@ void test_s_wait_event_export_ready() {
 }
 
 // CHECK-LABEL: @test_global_add_f32
-// CHECK: {{.*}}call{{.*}} float @llvm.amdgcn.global.atomic.fadd.f32.p1.f32(ptr addrspace(1) %{{.*}}, float %{{.*}})
+// CHECK: = atomicrmw fadd ptr addrspace(1) %addr, float %x syncscope("agent") monotonic, align 4, !amdgpu.no.fine.grained.memory !{{[0-9]+}}, !amdgpu.ignore.denormal.mode !{{[0-9]+$}}
 void test_global_add_f32(float *rtn, global float *addr, float x) {
   *rtn = __builtin_amdgcn_global_atomic_fadd_f32(addr, x);
 }
diff --git a/clang/test/CodeGenOpenCL/builtins-fp-atomics-gfx12.cl b/clang/test/CodeGenOpenCL/builtins-fp-atomics-gfx12.cl
index 63381942eaba574..6b8a6d14575db88 100644
--- a/clang/test/CodeGenOpenCL/builtins-fp-atomics-gfx12.cl
+++ b/clang/test/CodeGenOpenCL/builtins-fp-atomics-gfx12.cl
@@ -11,7 +11,7 @@ typedef short __attribute__((ext_vector_type(2))) short2;
 
 // CHECK-LABEL: test_local_add_2bf16
 // CHECK: [[BC0:%.+]] = bitcast <2 x i16> {{.+}} to <2 x bfloat>
-// CHECK: [[RMW:%.+]] = atomicrmw fadd ptr addrspace(3) %{{.+}}, <2 x bfloat> [[BC0]] seq_cst, align 4
+// CHECK: [[RMW:%.+]] = atomicrmw fadd ptr addrspace(3) %{{.+}}, <2 x bfloat> [[BC0]] syncscope("agent") monotonic, align 4
 // CHECK-NEXT: bitcast <2 x bfloat> [[RMW]] to <2 x i16>
 
 // GFX12-LABEL:  test_local_add_2bf16
@@ -22,7 +22,7 @@ short2 test_local_add_2bf16(__local short2 *addr, short2 x) {
 
 // CHECK-LABEL: test_local_add_2bf16_noret
 // CHECK: [[BC0:%.+]] = bitcast <2 x i16> {{.+}} to <2 x bfloat>
-// CHECK: [[RMW:%.+]] = atomicrmw fadd ptr addrspace(3) %{{.+}}, <2 x bfloat> [[BC0]] seq_cst, align 4
+// CHECK: [[RMW:%.+]] = atomicrmw fadd ptr addrspace(3) %{{.+}}, <2 x bfloat> [[BC0]] syncscope("agent") monotonic, align 4
 // CHECK-NEXT: bitcast <2 x bfloat> [[RMW]] to <2 x i16>
 
 // GFX12-LABEL:  test_local_add_2bf16_noret
@@ -32,7 +32,7 @@ void test_local_add_2bf16_noret(__local short2 *addr, short2 x) {
 }
 
 // CHECK-LABEL: test_local_add_2f16
-// CHECK: = atomicrmw fadd ptr addrspace(3) %{{.+}}, <2 x half> %{{.+}} seq_cst, align 4
+// CHECK: = atomicrmw fadd ptr addrspace(3) %{{.+}}, <2 x half> %{{.+}} monotonic, align 4
 // GFX12-LABEL:  test_local_add_2f16
 // GFX12: ds_pk_add_rtn_f16
 half2 test_local_add_2f16(__local half2 *addr, half2 x) {
@@ -40,7 +40,7 @@ half2 test_local_add_2f16(__local half2 *addr, half2 x) {
 }
 
 // CHECK-LABEL: test_local_add_2f16_noret
-// CHECK: = atomicrmw fadd ptr addrspace(3) %{{.+}}, <2 x half> %{{.+}} seq_cst, align 4
+// CHECK: = atomicrmw fadd ptr addrspace(3) %{{.+}}, <2 x half> %{{.+}} monotonic, align 4
 // GFX12-LABEL:  test_local_add_2f16_noret
 // GFX12: ds_pk_add_f16
 void test_local_add_2f16_noret(__local half2 *addr, half2 x) {
diff --git a/clang/test/CodeGenOpenCL/builtins-fp-atomics-gfx8.cl b/clang/test/CodeGenOpenCL/builtins-fp-atomics-gfx8.cl
index ad4d0b7af3d4be7..2f00977ec6014e1 100644
--- a/clang/test/CodeGenOpenCL/builtins-fp-atomics-gfx8.cl
+++ b/clang/test/CodeGenOpenCL/builtins-fp-atomics-gfx8.cl
@@ -6,7 +6,7 @@
 // REQUIRES: amdgpu-registered-target
 
 // CHECK-LABEL: test_fadd_local
-// CHECK: = atomicrmw fadd ptr addrspace(3) %{{.+}}, float %{{.+}} seq_cst, align 4
+// CHECK: = atomicrmw fadd ptr addrspace(3) %{{.+}}, float %{{.+}} monotonic, align 4
 // GFX8-LABEL: test_fadd_local$local:
 // GFX8: ds_add_rtn_f32 v2, v0, v1
 // GFX8: s_endpgm
@@ -16,7 +16,7 @@ kernel void test_fadd_local(__local float *ptr, float val){
 }
 
 // CHECK-LABEL: test_fadd_local_volatile
-// CHECK: = atomicrmw volatile fadd ptr addrspace(3) %{{.+}}, float %{{.+}} seq_cst, align 4
+// CHECK: = atomicrmw volatile fadd ptr addrspace(3) %{{.+}}, float %{{.+}} monotonic, align 4
 kernel void test_fadd_local_volatile(volatile __local float *ptr, float val){
     volatile float *res;
     *res = __builtin_amdgcn_ds_atomic_fadd_f32(ptr, val);
diff --git a/clang/test/CodeGenOpenCL/builtins-fp-atomics-gfx90a.cl b/clang/test/CodeGenOpenCL/builtins-fp-atomics-gfx90a.cl
index e2117f11858f7f8..c525c250c937cad 100644
--- a/clang/test/CodeGenOpenCL/builtins-fp-atomics-gfx90a.cl
+++ b/clang/test/CodeGenOpenCL/builtins-fp-atomics-gfx90a.cl
@@ -9,7 +9,7 @@
 typedef half __attribute__((ext_vector_type(2))) half2;
 
 // CHECK-LABEL: test_global_add_f64
-// CHECK: call double @llvm.amdgcn.global.atomic.fadd.f64.p1.f64(ptr addrspace(1) %{{.*}}, double %{{.*}})
+// CHECK: = atomicrmw fadd ptr addrspace(1) %{{.+}}, double %{{.+}} syncscope("agent") monotonic, align 8, !amdgpu.no.fine.grained.memory !{{[0-9]+$}}
 // GFX90A-LABEL:  test_global_add_f64$local:
 // GFX90A:  global_atomic_add_f64
 void test_global_add_f64(__global double *addr, double x) {
@@ -99,7 +99,7 @@ void test_flat_global_max_f64(__global double *addr, double x){
 }
 
 // CHECK-LABEL: test_ds_add_local_f64
-// CHECK: = atomicrmw fadd ptr addrspace(3) %{{.+}}, double %{{.+}} seq_cst, align 8
+// CHECK: = atomicrmw fadd ptr addrspace(3) %{{.+}}, double %{{.+}} monotonic, align 8
 // GFX90A:  test_ds_add_local_f64$local
 // GFX90A:  ds_add_rtn_f64
 void test_ds_add_local_f64(__local double *addr, double x){
@@ -108,7 +108,7 @@ void test_ds_add_local_f64(__local double *addr, double x){
 }
 
 // CHECK-LABEL: test_ds_addf_local_f32
-// CHECK: = atomicrmw fadd ptr addrspace(3) %{{.+}}, float %{{.+}} seq_cst, align 4
+// CHECK: = atomicrmw fadd ptr addrspace(3) %{{.+}}, float %{{.+}} monotonic, align 4
 // GFX90A-LABEL:  test_ds_addf_local_f32$local
 // GFX90A:  ds_add_rtn_f32
 void test_ds_addf_local_f32(__local float *addr, float x){
@@ -117,7 +117,7 @@ void test_ds_addf_local_f32(__local float *addr, float x){
 }
 
 // CHECK-LABEL: @test_global_add_f32
-// CHECK: call float @llvm.amdgcn.global.atomic.fadd.f32.p1.f32(ptr addrspace(1) %{{.*}}, float %{{.*}})
+// CHECK: = atomicrmw fadd ptr addrspace(1) %{{.+}}, float %{{.+}} syncscope("agent") monotonic, align 4, !amdgpu.no.fine.grained.memory !{{[0-9]+}}, !amdgpu.ignore.denormal.mode !{{[0-9]+$}}
 void test_global_add_f32(float *rtn, global float *addr, float x) {
   *rtn = __builtin_amdgcn_global_atomic_fadd_f32(addr, x);
 }
diff --git a/clang/test/CodeGenOpenCL/builtins-fp-atomics-gfx940.cl b/clang/test/CodeGenOpenCL/builtins-fp-atomics-gfx940.cl
index 92a33ceac2290f5..5481138b9fee43a 100644
--- a/clang/test/CodeGenOpenCL/builtins-fp-atomics-gfx940.cl
+++ b/clang/test/CodeGenOpenCL/builtins-fp-atomics-gfx940.cl
@@ -44,7 +44,7 @@ short2 test_global_add_2bf16(__global short2 *addr, short2 x) {
 // CHECK-LABEL: test_local_add_2bf16
 
 // CHECK: [[BC0:%.+]] = bitcast <2 x i16> {{.+}} to <2 x bfloat>
-// CHECK: [[RMW:%.+]] = atomicrmw fadd ptr addrspace(3) %{{.+}}, <2 x bfloat> [[BC0]] seq_cst, align 4
+// CHECK: [[RMW:%.+]] = atomicrmw fadd ptr addrspace(3) %{{.+}}, <2 x bfloat> [[BC0]] syncscope("agent") monotonic, align 4{{$}}
 // CHECK-NEXT: bitcast <2 x bfloat> [[RMW]] to <2 x i16>
 
 // GFX940-LABEL:  test_local_add_2bf16
@@ -54,7 +54,7 @@ short2 test_local_add_2bf16(__local short2 *addr, short2 x) {
 }
 
 // CHECK-LABEL: test_local_add_2f16
-// CHECK: = atomicrmw fadd ptr addrspace(3) %{{.+}}, <2 x half> %{{.+}} seq_cst, align 4
+// CHECK: = atomicrmw fadd ptr addrspace(3) %{{.+}}, <2 x half> %{{.+}} monotonic, align 4
 // GFX940-LABEL:  test_local_add_2f16
 // GFX940: ds_pk_add_rtn_f16
 half2 test_local_add_2f16(__local half2 *addr, half2 x) {
@@ -62,7 +62,7 @@ half2 test_local_add_2f16(__local half2 *addr, half2 x) {
 }
 
 // CHECK-LABEL: test_local_add_2f16_noret
-// CHECK: = atomicrmw fadd ptr addrspace(3) %{{.+}}, <2 x half> %{{.+}} seq_cst, align 4
+// CHECK: = atomicrmw fadd ptr addrspace(3) %{{.+}}, <2 x half> %{{.+}} monotonic, align 4
 // GFX940-LABEL:  test_local_add_2f16_noret
 // GFX940: ds_pk_add_f16
 void test_local_add_2f16_noret(__local half2 *addr, half2 x) {
@@ -70,7 +70,7 @@ void test_local_add_2f16_noret(__local half2 *addr, half2 x) {
 }
 
 // CHECK-LABEL: @test_global_add_f32
-// CHECK: call float @llvm.amdgcn.global.atomic.fadd.f32.p1.f32(ptr addrspace(1) %{{.*}}, float %{{.*}})
+// CHECK: = atomicrmw fadd ptr addrspace(1) %{{.+}}, float %{{.+}} syncscope("agent") monotonic, align 4, !amdgpu.no.fine.grained.memory !{{[0-9]+}}, !amdgpu.ignore.denormal.mode !{{[0-9]+$}}
 void test_global_add_f32(float *rtn, global float *addr, float x) {
   *rtn = __builtin_amdgcn_global_atomic_fadd_f32(addr, x);
 }

From 2b0a8fcf702fb63fca8ec6e11dca35baf70f058d Mon Sep 17 00:00:00 2001
From: Sirraide <aeternalmail@gmail.com>
Date: Thu, 15 Aug 2024 21:16:30 +0200
Subject: [PATCH 067/441] =?UTF-8?q?[Clang]=20Implement=20C++26=E2=80=99s?=
 =?UTF-8?q?=20P2893R3=20=E2=80=98Variadic=20friends=E2=80=99=20(#101448)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Implement P2893R3 ‘Variadic friends’ for C++26.

This closes #98587.

Co-authored-by: Younan Zhang <zyn7109@gmail.com>
---
 clang/docs/LanguageExtensions.rst             |   1 +
 clang/docs/ReleaseNotes.rst                   |   2 +
 clang/include/clang/AST/DeclFriend.h          |  39 +-
 .../clang/Basic/DiagnosticParseKinds.td       |   6 +
 .../clang/Basic/DiagnosticSemaKinds.td        |   4 +
 clang/include/clang/Sema/Sema.h               |   7 +-
 clang/lib/AST/ASTImporter.cpp                 |   7 +-
 clang/lib/AST/DeclFriend.cpp                  |  14 +-
 clang/lib/AST/DeclPrinter.cpp                 |   5 +-
 clang/lib/AST/JSONNodeDumper.cpp              |   1 +
 clang/lib/AST/ODRHash.cpp                     |   1 +
 clang/lib/AST/TextNodeDumper.cpp              |   2 +
 clang/lib/Frontend/InitPreprocessor.cpp       |   1 +
 clang/lib/Parse/ParseDeclCXX.cpp              |  98 +++-
 clang/lib/Sema/SemaDecl.cpp                   |   8 +-
 clang/lib/Sema/SemaDeclCXX.cpp                |  53 ++-
 .../lib/Sema/SemaTemplateInstantiateDecl.cpp  |  43 +-
 clang/lib/Serialization/ASTReaderDecl.cpp     |   1 +
 clang/lib/Serialization/ASTWriterDecl.cpp     |   1 +
 clang/test/AST/ast-dump-funcs-json.cpp        | 441 +++++++++++++-----
 clang/test/AST/cxx2c-variadic-friends.cpp     |  81 ++++
 clang/test/CXX/drs/cwg29xx.cpp                |  25 +
 clang/test/Lexer/cxx-features.cpp             |   4 +
 .../cxx2c-variadic-friends-ext-diags.cpp      |  16 +
 clang/test/Parser/cxx2c-variadic-friends.cpp  |  91 ++++
 clang/test/SemaCXX/cxx2c-variadic-friends.cpp | 156 +++++++
 clang/www/cxx_dr_status.html                  |   2 +-
 clang/www/cxx_status.html                     |   2 +-
 28 files changed, 942 insertions(+), 170 deletions(-)
 create mode 100644 clang/test/AST/cxx2c-variadic-friends.cpp
 create mode 100644 clang/test/CXX/drs/cwg29xx.cpp
 create mode 100644 clang/test/Parser/cxx2c-variadic-friends-ext-diags.cpp
 create mode 100644 clang/test/Parser/cxx2c-variadic-friends.cpp
 create mode 100644 clang/test/SemaCXX/cxx2c-variadic-friends.cpp

diff --git a/clang/docs/LanguageExtensions.rst b/clang/docs/LanguageExtensions.rst
index 4679dbb68b25e10..114e742f3561b75 100644
--- a/clang/docs/LanguageExtensions.rst
+++ b/clang/docs/LanguageExtensions.rst
@@ -1505,6 +1505,7 @@ Attributes on Lambda-Expressions                                              C+
 Attributes on Structured Bindings            __cpp_structured_bindings        C++26         C++03
 Pack Indexing                                __cpp_pack_indexing              C++26         C++03
 ``= delete ("should have a reason");``       __cpp_deleted_function           C++26         C++03
+Variadic Friends                             __cpp_variadic_friend            C++26         C++03
 -------------------------------------------- -------------------------------- ------------- -------------
 Designated initializers (N494)                                                C99           C89
 Array & element qualification (N2607)                                         C23           C89
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 68cf79928bda021..14f1eecc5748ed0 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -126,6 +126,8 @@ C++2c Feature Support
 - Add ``__builtin_is_virtual_base_of`` intrinsic, which supports
   `P2985R0 A type trait for detecting virtual base classes <https://wg21.link/p2985r0>`_
 
+- Implemented `P2893R3 Variadic Friends <https://wg21.link/P2893>`_
+
 Resolutions to C++ Defect Reports
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
diff --git a/clang/include/clang/AST/DeclFriend.h b/clang/include/clang/AST/DeclFriend.h
index 9789282f351a55c..095f14a81fd5747 100644
--- a/clang/include/clang/AST/DeclFriend.h
+++ b/clang/include/clang/AST/DeclFriend.h
@@ -70,6 +70,9 @@ class FriendDecl final
   // Location of the 'friend' specifier.
   SourceLocation FriendLoc;
 
+  // Location of the '...', if present.
+  SourceLocation EllipsisLoc;
+
   /// True if this 'friend' declaration is unsupported.  Eventually we
   /// will support every possible friend declaration, but for now we
   /// silently ignore some and set this flag to authorize all access.
@@ -82,10 +85,11 @@ class FriendDecl final
   unsigned NumTPLists : 31;
 
   FriendDecl(DeclContext *DC, SourceLocation L, FriendUnion Friend,
-             SourceLocation FriendL,
+             SourceLocation FriendL, SourceLocation EllipsisLoc,
              ArrayRef<TemplateParameterList *> FriendTypeTPLists)
       : Decl(Decl::Friend, DC, L), Friend(Friend), FriendLoc(FriendL),
-        UnsupportedFriend(false), NumTPLists(FriendTypeTPLists.size()) {
+        EllipsisLoc(EllipsisLoc), UnsupportedFriend(false),
+        NumTPLists(FriendTypeTPLists.size()) {
     for (unsigned i = 0; i < NumTPLists; ++i)
       getTrailingObjects<TemplateParameterList *>()[i] = FriendTypeTPLists[i];
   }
@@ -110,7 +114,7 @@ class FriendDecl final
 
   static FriendDecl *
   Create(ASTContext &C, DeclContext *DC, SourceLocation L, FriendUnion Friend_,
-         SourceLocation FriendL,
+         SourceLocation FriendL, SourceLocation EllipsisLoc = {},
          ArrayRef<TemplateParameterList *> FriendTypeTPLists = std::nullopt);
   static FriendDecl *CreateDeserialized(ASTContext &C, GlobalDeclID ID,
                                         unsigned FriendTypeNumTPLists);
@@ -143,8 +147,24 @@ class FriendDecl final
     return FriendLoc;
   }
 
+  /// Retrieves the location of the '...', if present.
+  SourceLocation getEllipsisLoc() const { return EllipsisLoc; }
+
   /// Retrieves the source range for the friend declaration.
   SourceRange getSourceRange() const override LLVM_READONLY {
+    if (TypeSourceInfo *TInfo = getFriendType()) {
+      SourceLocation StartL =
+          (NumTPLists == 0) ? getFriendLoc()
+                            : getTrailingObjects<TemplateParameterList *>()[0]
+                                  ->getTemplateLoc();
+      SourceLocation EndL = isPackExpansion() ? getEllipsisLoc()
+                                              : TInfo->getTypeLoc().getEndLoc();
+      return SourceRange(StartL, EndL);
+    }
+
+    if (isPackExpansion())
+      return SourceRange(getFriendLoc(), getEllipsisLoc());
+
     if (NamedDecl *ND = getFriendDecl()) {
       if (const auto *FD = dyn_cast<FunctionDecl>(ND))
         return FD->getSourceRange();
@@ -158,15 +178,8 @@ class FriendDecl final
       }
       return SourceRange(getFriendLoc(), ND->getEndLoc());
     }
-    else if (TypeSourceInfo *TInfo = getFriendType()) {
-      SourceLocation StartL =
-          (NumTPLists == 0) ? getFriendLoc()
-                            : getTrailingObjects<TemplateParameterList *>()[0]
-                                  ->getTemplateLoc();
-      return SourceRange(StartL, TInfo->getTypeLoc().getEndLoc());
-    }
-    else
-      return SourceRange(getFriendLoc(), getLocation());
+
+    return SourceRange(getFriendLoc(), getLocation());
   }
 
   /// Determines if this friend kind is unsupported.
@@ -177,6 +190,8 @@ class FriendDecl final
     UnsupportedFriend = Unsupported;
   }
 
+  bool isPackExpansion() const { return EllipsisLoc.isValid(); }
+
   // Implement isa/cast/dyncast/etc.
   static bool classof(const Decl *D) { return classofKind(D->getKind()); }
   static bool classofKind(Kind K) { return K == Decl::Friend; }
diff --git a/clang/include/clang/Basic/DiagnosticParseKinds.td b/clang/include/clang/Basic/DiagnosticParseKinds.td
index 12aab09f285567f..62a97b36737e72e 100644
--- a/clang/include/clang/Basic/DiagnosticParseKinds.td
+++ b/clang/include/clang/Basic/DiagnosticParseKinds.td
@@ -965,6 +965,12 @@ def warn_cxx23_delete_with_message : Warning<
   "'= delete' with a message is incompatible with C++ standards before C++2c">,
   DefaultIgnore, InGroup<CXXPre26Compat>;
 
+def ext_variadic_friends : ExtWarn<
+  "variadic 'friend' declarations are a C++2c extension">, InGroup<CXX26>;
+def warn_cxx23_variadic_friends : Warning<
+  "variadic 'friend' declarations are incompatible with C++ standards before C++2c">,
+  DefaultIgnore, InGroup<CXXPre26Compat>;
+
 // C++11 default member initialization
 def ext_nonstatic_member_init : ExtWarn<
   "default member initializer for non-static data member is a C++11 "
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 461eeb19f65e4a3..8a92973236ddbdb 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -1741,6 +1741,10 @@ def ext_friend_tag_redecl_outside_namespace : ExtWarn<
   "enclosing namespace is a Microsoft extension; add a nested name specifier">,
   InGroup<MicrosoftUnqualifiedFriend>;
 def err_pure_friend : Error<"friend declaration cannot have a pure-specifier">;
+def err_friend_template_decl_multiple_specifiers: Error<
+  "a friend declaration that befriends a template must contain exactly one type-specifier">;
+def friend_template_decl_malformed_pack_expansion : Error<
+  "friend declaration expands pack %0 that is declared it its own template parameter list">;
 
 def err_invalid_base_in_interface : Error<
   "interface type cannot inherit from "
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index a025ff6fc13f360..88e82dca007b8bd 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -3800,7 +3800,8 @@ class Sema final : public SemaBase {
                                    const ParsedAttributesView &DeclAttrs,
                                    MultiTemplateParamsArg TemplateParams,
                                    bool IsExplicitInstantiation,
-                                   RecordDecl *&AnonRecord);
+                                   RecordDecl *&AnonRecord,
+                                   SourceLocation EllipsisLoc = {});
 
   /// BuildAnonymousStructOrUnion - Handle the declaration of an
   /// anonymous structure or union. Anonymous unions are a C++ feature
@@ -5538,7 +5539,8 @@ class Sema final : public SemaBase {
   /// parameters present at all, require proper matching, i.e.
   ///   template <> template \<class T> friend class A<int>::B;
   Decl *ActOnFriendTypeDecl(Scope *S, const DeclSpec &DS,
-                            MultiTemplateParamsArg TemplateParams);
+                            MultiTemplateParamsArg TemplateParams,
+                            SourceLocation EllipsisLoc);
   NamedDecl *ActOnFriendFunctionDecl(Scope *S, Declarator &D,
                                      MultiTemplateParamsArg TemplateParams);
 
@@ -5852,6 +5854,7 @@ class Sema final : public SemaBase {
                                      unsigned TagSpec, SourceLocation TagLoc,
                                      CXXScopeSpec &SS, IdentifierInfo *Name,
                                      SourceLocation NameLoc,
+                                     SourceLocation EllipsisLoc,
                                      const ParsedAttributesView &Attr,
                                      MultiTemplateParamsArg TempParamLists);
 
diff --git a/clang/lib/AST/ASTImporter.cpp b/clang/lib/AST/ASTImporter.cpp
index 198bc34a9f031bb..3bc0a647ebf94ff 100644
--- a/clang/lib/AST/ASTImporter.cpp
+++ b/clang/lib/AST/ASTImporter.cpp
@@ -4429,11 +4429,14 @@ ExpectedDecl ASTNodeImporter::VisitFriendDecl(FriendDecl *D) {
   auto FriendLocOrErr = import(D->getFriendLoc());
   if (!FriendLocOrErr)
     return FriendLocOrErr.takeError();
+  auto EllipsisLocOrErr = import(D->getEllipsisLoc());
+  if (!EllipsisLocOrErr)
+    return EllipsisLocOrErr.takeError();
 
   FriendDecl *FrD;
   if (GetImportedOrCreateDecl(FrD, D, Importer.getToContext(), DC,
-                              *LocationOrErr, ToFU,
-                              *FriendLocOrErr, ToTPLists))
+                              *LocationOrErr, ToFU, *FriendLocOrErr,
+                              *EllipsisLocOrErr, ToTPLists))
     return FrD;
 
   FrD->setAccess(D->getAccess());
diff --git a/clang/lib/AST/DeclFriend.cpp b/clang/lib/AST/DeclFriend.cpp
index 04b9b93699f36ce..8b285bfce8d5221 100644
--- a/clang/lib/AST/DeclFriend.cpp
+++ b/clang/lib/AST/DeclFriend.cpp
@@ -31,11 +31,11 @@ FriendDecl *FriendDecl::getNextFriendSlowCase() {
                            NextFriend.get(getASTContext().getExternalSource()));
 }
 
-FriendDecl *FriendDecl::Create(ASTContext &C, DeclContext *DC,
-                               SourceLocation L,
-                               FriendUnion Friend,
-                               SourceLocation FriendL,
-                          ArrayRef<TemplateParameterList *> FriendTypeTPLists) {
+FriendDecl *
+FriendDecl::Create(ASTContext &C, DeclContext *DC, SourceLocation L,
+                   FriendUnion Friend, SourceLocation FriendL,
+                   SourceLocation EllipsisLoc,
+                   ArrayRef<TemplateParameterList *> FriendTypeTPLists) {
 #ifndef NDEBUG
   if (Friend.is<NamedDecl *>()) {
     const auto *D = Friend.get<NamedDecl*>();
@@ -56,8 +56,8 @@ FriendDecl *FriendDecl::Create(ASTContext &C, DeclContext *DC,
   std::size_t Extra =
       FriendDecl::additionalSizeToAlloc<TemplateParameterList *>(
           FriendTypeTPLists.size());
-  auto *FD = new (C, DC, Extra) FriendDecl(DC, L, Friend, FriendL,
-                                           FriendTypeTPLists);
+  auto *FD = new (C, DC, Extra)
+      FriendDecl(DC, L, Friend, FriendL, EllipsisLoc, FriendTypeTPLists);
   cast<CXXRecordDecl>(DC)->pushFriendDecl(FD);
   return FD;
 }
diff --git a/clang/lib/AST/DeclPrinter.cpp b/clang/lib/AST/DeclPrinter.cpp
index 26773a69ab9acff..07be813abd8adca 100644
--- a/clang/lib/AST/DeclPrinter.cpp
+++ b/clang/lib/AST/DeclPrinter.cpp
@@ -868,7 +868,7 @@ void DeclPrinter::VisitFriendDecl(FriendDecl *D) {
     for (unsigned i = 0; i < NumTPLists; ++i)
       printTemplateParameters(D->getFriendTypeTemplateParameterList(i));
     Out << "friend ";
-    Out << " " << TSI->getType().getAsString(Policy);
+    Out << TSI->getType().getAsString(Policy);
   }
   else if (FunctionDecl *FD =
       dyn_cast<FunctionDecl>(D->getFriendDecl())) {
@@ -885,6 +885,9 @@ void DeclPrinter::VisitFriendDecl(FriendDecl *D) {
     Out << "friend ";
     VisitRedeclarableTemplateDecl(CTD);
   }
+
+  if (D->isPackExpansion())
+    Out << "...";
 }
 
 void DeclPrinter::VisitFieldDecl(FieldDecl *D) {
diff --git a/clang/lib/AST/JSONNodeDumper.cpp b/clang/lib/AST/JSONNodeDumper.cpp
index f8f80c8c2515755..565f1e05710c879 100644
--- a/clang/lib/AST/JSONNodeDumper.cpp
+++ b/clang/lib/AST/JSONNodeDumper.cpp
@@ -1090,6 +1090,7 @@ void JSONNodeDumper::VisitAccessSpecDecl(const AccessSpecDecl *ASD) {
 void JSONNodeDumper::VisitFriendDecl(const FriendDecl *FD) {
   if (const TypeSourceInfo *T = FD->getFriendType())
     JOS.attribute("type", createQualType(T->getType()));
+  attributeOnlyIfTrue("isPackExpansion", FD->isPackExpansion());
 }
 
 void JSONNodeDumper::VisitObjCIvarDecl(const ObjCIvarDecl *D) {
diff --git a/clang/lib/AST/ODRHash.cpp b/clang/lib/AST/ODRHash.cpp
index fbfe92318dc5eee..b748093831e3f50 100644
--- a/clang/lib/AST/ODRHash.cpp
+++ b/clang/lib/AST/ODRHash.cpp
@@ -461,6 +461,7 @@ class ODRDeclVisitor : public ConstDeclVisitor<ODRDeclVisitor> {
     } else {
       AddDecl(D->getFriendDecl());
     }
+    Hash.AddBoolean(D->isPackExpansion());
   }
 
   void VisitTemplateTypeParmDecl(const TemplateTypeParmDecl *D) {
diff --git a/clang/lib/AST/TextNodeDumper.cpp b/clang/lib/AST/TextNodeDumper.cpp
index d50d4c7028c6972..2c962253c8bea42 100644
--- a/clang/lib/AST/TextNodeDumper.cpp
+++ b/clang/lib/AST/TextNodeDumper.cpp
@@ -2697,6 +2697,8 @@ void TextNodeDumper::VisitAccessSpecDecl(const AccessSpecDecl *D) {
 void TextNodeDumper::VisitFriendDecl(const FriendDecl *D) {
   if (TypeSourceInfo *T = D->getFriendType())
     dumpType(T->getType());
+  if (D->isPackExpansion())
+    OS << "...";
 }
 
 void TextNodeDumper::VisitObjCIvarDecl(const ObjCIvarDecl *D) {
diff --git a/clang/lib/Frontend/InitPreprocessor.cpp b/clang/lib/Frontend/InitPreprocessor.cpp
index 8e62461d8a1818b..4f2856dd2247f8f 100644
--- a/clang/lib/Frontend/InitPreprocessor.cpp
+++ b/clang/lib/Frontend/InitPreprocessor.cpp
@@ -765,6 +765,7 @@ static void InitializeCPlusPlusFeatureTestMacros(const LangOptions &LangOpts,
   // C++26 features supported in earlier language modes.
   Builder.defineMacro("__cpp_pack_indexing", "202311L");
   Builder.defineMacro("__cpp_deleted_function", "202403L");
+  Builder.defineMacro("__cpp_variadic_friend", "202403L");
 
   if (LangOpts.Char8)
     Builder.defineMacro("__cpp_char8_t", "202207L");
diff --git a/clang/lib/Parse/ParseDeclCXX.cpp b/clang/lib/Parse/ParseDeclCXX.cpp
index d45a738fe4c5969..18c5fe6056b4726 100644
--- a/clang/lib/Parse/ParseDeclCXX.cpp
+++ b/clang/lib/Parse/ParseDeclCXX.cpp
@@ -452,7 +452,7 @@ Decl *Parser::ParseLinkage(ParsingDeclSpec &DS, DeclaratorContext Context) {
 ///
 ///      export-function-declaration:
 ///         'export' function-declaration
-/// 
+///
 ///      export-declaration-group:
 ///         'export' '{' function-declaration-seq[opt] '}'
 ///
@@ -2007,9 +2007,16 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind,
 
   const PrintingPolicy &Policy = Actions.getASTContext().getPrintingPolicy();
   TagUseKind TUK;
-  if (isDefiningTypeSpecifierContext(DSC, getLangOpts().CPlusPlus) ==
-          AllowDefiningTypeSpec::No ||
-      (getLangOpts().OpenMP && OpenMPDirectiveParsing))
+
+  // C++26 [class.mem.general]p10: If a name-declaration matches the
+  // syntactic requirements of friend-type-declaration, it is a
+  // friend-type-declaration.
+  if (getLangOpts().CPlusPlus && DS.isFriendSpecifiedFirst() &&
+      Tok.isOneOf(tok::comma, tok::ellipsis))
+    TUK = TagUseKind::Friend;
+  else if (isDefiningTypeSpecifierContext(DSC, getLangOpts().CPlusPlus) ==
+               AllowDefiningTypeSpec::No ||
+           (getLangOpts().OpenMP && OpenMPDirectiveParsing))
     TUK = TagUseKind::Reference;
   else if (Tok.is(tok::l_brace) ||
            (DSC != DeclSpecContext::DSC_association &&
@@ -2241,9 +2248,28 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind,
                             diag::err_keyword_not_allowed,
                             /*DiagnoseEmptyAttrs=*/true);
 
+    // Consume '...' first so we error on the ',' after it if there is one.
+    SourceLocation EllipsisLoc;
+    TryConsumeToken(tok::ellipsis, EllipsisLoc);
+
+    // CWG 2917: In a template-declaration whose declaration is a
+    // friend-type-declaration, the friend-type-specifier-list shall
+    // consist of exactly one friend-type-specifier.
+    //
+    // Essentially, the following is obviously nonsense, so disallow it:
+    //
+    //   template <typename>
+    //   friend class S, int;
+    //
+    if (Tok.is(tok::comma)) {
+      Diag(Tok.getLocation(),
+           diag::err_friend_template_decl_multiple_specifiers);
+      SkipUntil(tok::semi, StopBeforeMatch);
+    }
+
     TagOrTempResult = Actions.ActOnTemplatedFriendTag(
         getCurScope(), DS.getFriendSpecLoc(), TagType, StartLoc, SS, Name,
-        NameLoc, attrs,
+        NameLoc, EllipsisLoc, attrs,
         MultiTemplateParamsArg(TemplateParams ? &(*TemplateParams)[0] : nullptr,
                                TemplateParams ? TemplateParams->size() : 0));
   } else {
@@ -2818,6 +2844,7 @@ void Parser::MaybeParseAndDiagnoseDeclSpecAfterCXX11VirtSpecifierSeq(
 ///       member-declaration:
 ///         decl-specifier-seq[opt] member-declarator-list[opt] ';'
 ///         function-definition ';'[opt]
+/// [C++26] friend-type-declaration
 ///         ::[opt] nested-name-specifier template[opt] unqualified-id ';'[TODO]
 ///         using-declaration                                            [TODO]
 /// [C++0x] static_assert-declaration
@@ -2850,6 +2877,18 @@ void Parser::MaybeParseAndDiagnoseDeclSpecAfterCXX11VirtSpecifierSeq(
 ///       constant-initializer:
 ///         '=' constant-expression
 ///
+///       friend-type-declaration:
+///         'friend' friend-type-specifier-list ;
+///
+///       friend-type-specifier-list:
+///         friend-type-specifier ...[opt]
+///         friend-type-specifier-list , friend-type-specifier ...[opt]
+///
+///       friend-type-specifier:
+///         simple-type-specifier
+///         elaborated-type-specifier
+///         typename-specifier
+///
 Parser::DeclGroupPtrTy Parser::ParseCXXClassMemberDeclaration(
     AccessSpecifier AS, ParsedAttributes &AccessAttrs,
     ParsedTemplateInfo &TemplateInfo, ParsingDeclRAIIObject *TemplateDiags) {
@@ -3051,6 +3090,55 @@ Parser::DeclGroupPtrTy Parser::ParseCXXClassMemberDeclaration(
   if (DS.hasTagDefinition())
     Actions.ActOnDefinedDeclarationSpecifier(DS.getRepAsDecl());
 
+  // Handle C++26's variadic friend declarations. These don't even have
+  // declarators, so we get them out of the way early here.
+  if (DS.isFriendSpecifiedFirst() && Tok.isOneOf(tok::comma, tok::ellipsis)) {
+    Diag(Tok.getLocation(), getLangOpts().CPlusPlus26
+                                ? diag::warn_cxx23_variadic_friends
+                                : diag::ext_variadic_friends);
+
+    SourceLocation FriendLoc = DS.getFriendSpecLoc();
+    SmallVector<Decl *> Decls;
+
+    // Handles a single friend-type-specifier.
+    auto ParsedFriendDecl = [&](ParsingDeclSpec &DeclSpec) {
+      SourceLocation VariadicLoc;
+      TryConsumeToken(tok::ellipsis, VariadicLoc);
+
+      RecordDecl *AnonRecord = nullptr;
+      Decl *D = Actions.ParsedFreeStandingDeclSpec(
+          getCurScope(), AS, DeclSpec, DeclAttrs, TemplateParams, false,
+          AnonRecord, VariadicLoc);
+      DeclSpec.complete(D);
+      if (!D) {
+        SkipUntil(tok::semi, tok::r_brace);
+        return true;
+      }
+
+      Decls.push_back(D);
+      return false;
+    };
+
+    if (ParsedFriendDecl(DS))
+      return nullptr;
+
+    while (TryConsumeToken(tok::comma)) {
+      ParsingDeclSpec DeclSpec(*this, TemplateDiags);
+      const char *PrevSpec = nullptr;
+      unsigned DiagId = 0;
+      DeclSpec.SetFriendSpec(FriendLoc, PrevSpec, DiagId);
+      ParseDeclarationSpecifiers(DeclSpec, TemplateInfo, AS,
+                                 DeclSpecContext::DSC_class, nullptr);
+      if (ParsedFriendDecl(DeclSpec))
+        return nullptr;
+    }
+
+    ExpectAndConsume(tok::semi, diag::err_expected_semi_after_stmt,
+                     "friend declaration");
+
+    return Actions.BuildDeclaratorGroup(Decls);
+  }
+
   ParsingDeclarator DeclaratorInfo(*this, DS, DeclAttrs,
                                    DeclaratorContext::Member);
   if (TemplateInfo.TemplateParams)
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index 641b180527da558..83a652691e2e034 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -5000,7 +5000,8 @@ Decl *Sema::ParsedFreeStandingDeclSpec(Scope *S, AccessSpecifier AS,
                                        const ParsedAttributesView &DeclAttrs,
                                        MultiTemplateParamsArg TemplateParams,
                                        bool IsExplicitInstantiation,
-                                       RecordDecl *&AnonRecord) {
+                                       RecordDecl *&AnonRecord,
+                                       SourceLocation EllipsisLoc) {
   Decl *TagD = nullptr;
   TagDecl *Tag = nullptr;
   if (DS.getTypeSpecType() == DeclSpec::TST_class ||
@@ -5067,9 +5068,12 @@ Decl *Sema::ParsedFreeStandingDeclSpec(Scope *S, AccessSpecifier AS,
     // whatever routines created it handled the friendship aspect.
     if (TagD && !Tag)
       return nullptr;
-    return ActOnFriendTypeDecl(S, DS, TemplateParams);
+    return ActOnFriendTypeDecl(S, DS, TemplateParams, EllipsisLoc);
   }
 
+  assert(EllipsisLoc.isInvalid() &&
+         "Friend ellipsis but not friend-specified?");
+
   // Track whether this decl-specifier declares anything.
   bool DeclaresAnything = true;
 
diff --git a/clang/lib/Sema/SemaDeclCXX.cpp b/clang/lib/Sema/SemaDeclCXX.cpp
index 9ca91a2def39f5c..e05595e565d54a4 100644
--- a/clang/lib/Sema/SemaDeclCXX.cpp
+++ b/clang/lib/Sema/SemaDeclCXX.cpp
@@ -17386,7 +17386,8 @@ Decl *Sema::BuildStaticAssertDeclaration(SourceLocation StaticAssertLoc,
 DeclResult Sema::ActOnTemplatedFriendTag(
     Scope *S, SourceLocation FriendLoc, unsigned TagSpec, SourceLocation TagLoc,
     CXXScopeSpec &SS, IdentifierInfo *Name, SourceLocation NameLoc,
-    const ParsedAttributesView &Attr, MultiTemplateParamsArg TempParamLists) {
+    SourceLocation EllipsisLoc, const ParsedAttributesView &Attr,
+    MultiTemplateParamsArg TempParamLists) {
   TagTypeKind Kind = TypeWithKeyword::getTagTypeKindForTypeSpec(TagSpec);
 
   bool IsMemberSpecialization = false;
@@ -17430,6 +17431,7 @@ DeclResult Sema::ActOnTemplatedFriendTag(
   // If it's explicit specializations all the way down, just forget
   // about the template header and build an appropriate non-templated
   // friend.  TODO: for source fidelity, remember the headers.
+  NestedNameSpecifierLoc QualifierLoc = SS.getWithLocInContext(Context);
   if (isAllExplicitSpecializations) {
     if (SS.isEmpty()) {
       bool Owned = false;
@@ -17445,7 +17447,6 @@ DeclResult Sema::ActOnTemplatedFriendTag(
                       /*IsTemplateParamOrArg=*/false, /*OOK=*/OOK_Outside);
     }
 
-    NestedNameSpecifierLoc QualifierLoc = SS.getWithLocInContext(Context);
     ElaboratedTypeKeyword Keyword
       = TypeWithKeyword::getKeywordForTagTypeKind(Kind);
     QualType T = CheckTypenameType(Keyword, TagLoc, QualifierLoc,
@@ -17467,8 +17468,9 @@ DeclResult Sema::ActOnTemplatedFriendTag(
       TL.getNamedTypeLoc().castAs<TypeSpecTypeLoc>().setNameLoc(NameLoc);
     }
 
-    FriendDecl *Friend = FriendDecl::Create(Context, CurContext, NameLoc,
-                                            TSI, FriendLoc, TempParamLists);
+    FriendDecl *Friend =
+        FriendDecl::Create(Context, CurContext, NameLoc, TSI, FriendLoc,
+                           EllipsisLoc, TempParamLists);
     Friend->setAccess(AS_public);
     CurContext->addDecl(Friend);
     return Friend;
@@ -17476,7 +17478,22 @@ DeclResult Sema::ActOnTemplatedFriendTag(
 
   assert(SS.isNotEmpty() && "valid templated tag with no SS and no direct?");
 
-
+  // CWG 2917: if it (= the friend-type-specifier) is a pack expansion
+  // (13.7.4 [temp.variadic]), any packs expanded by that pack expansion
+  // shall not have been introduced by the template-declaration.
+  SmallVector<UnexpandedParameterPack, 1> Unexpanded;
+  collectUnexpandedParameterPacks(QualifierLoc, Unexpanded);
+  unsigned FriendDeclDepth = TempParamLists.front()->getDepth();
+  for (UnexpandedParameterPack &U : Unexpanded) {
+    if (getDepthAndIndex(U).first >= FriendDeclDepth) {
+      auto *ND = U.first.dyn_cast<NamedDecl *>();
+      if (!ND)
+        ND = U.first.get<const TemplateTypeParmType *>()->getDecl();
+      Diag(U.second, diag::friend_template_decl_malformed_pack_expansion)
+          << ND->getDeclName() << SourceRange(SS.getBeginLoc(), EllipsisLoc);
+      return true;
+    }
+  }
 
   // Handle the case of a templated-scope friend class.  e.g.
   //   template <class T> class A<T>::B;
@@ -17491,8 +17508,9 @@ DeclResult Sema::ActOnTemplatedFriendTag(
   TL.setQualifierLoc(SS.getWithLocInContext(Context));
   TL.setNameLoc(NameLoc);
 
-  FriendDecl *Friend = FriendDecl::Create(Context, CurContext, NameLoc,
-                                          TSI, FriendLoc, TempParamLists);
+  FriendDecl *Friend =
+      FriendDecl::Create(Context, CurContext, NameLoc, TSI, FriendLoc,
+                         EllipsisLoc, TempParamLists);
   Friend->setAccess(AS_public);
   Friend->setUnsupportedFriend(true);
   CurContext->addDecl(Friend);
@@ -17500,7 +17518,8 @@ DeclResult Sema::ActOnTemplatedFriendTag(
 }
 
 Decl *Sema::ActOnFriendTypeDecl(Scope *S, const DeclSpec &DS,
-                                MultiTemplateParamsArg TempParams) {
+                                MultiTemplateParamsArg TempParams,
+                                SourceLocation EllipsisLoc) {
   SourceLocation Loc = DS.getBeginLoc();
   SourceLocation FriendLoc = DS.getFriendSpecLoc();
 
@@ -17541,8 +17560,18 @@ Decl *Sema::ActOnFriendTypeDecl(Scope *S, const DeclSpec &DS,
   if (TheDeclarator.isInvalidType())
     return nullptr;
 
-  if (DiagnoseUnexpandedParameterPack(Loc, TSI, UPPC_FriendDeclaration))
+  // If '...' is present, the type must contain an unexpanded parameter
+  // pack, and vice versa.
+  bool Invalid = false;
+  if (EllipsisLoc.isInvalid() &&
+      DiagnoseUnexpandedParameterPack(Loc, TSI, UPPC_FriendDeclaration))
     return nullptr;
+  if (EllipsisLoc.isValid() &&
+      !TSI->getType()->containsUnexpandedParameterPack()) {
+    Diag(EllipsisLoc, diag::err_pack_expansion_without_parameter_packs)
+        << TSI->getTypeLoc().getSourceRange();
+    Invalid = true;
+  }
 
   if (!T->isElaboratedTypeSpecifier()) {
     if (TempParams.size()) {
@@ -17588,11 +17617,12 @@ Decl *Sema::ActOnFriendTypeDecl(Scope *S, const DeclSpec &DS,
 
   Decl *D;
   if (!TempParams.empty())
+    // TODO: Support variadic friend template decls?
     D = FriendTemplateDecl::Create(Context, CurContext, Loc, TempParams, TSI,
                                    FriendLoc);
   else
     D = FriendDecl::Create(Context, CurContext, TSI->getTypeLoc().getBeginLoc(),
-                           TSI, FriendLoc);
+                           TSI, FriendLoc, EllipsisLoc);
 
   if (!D)
     return nullptr;
@@ -17600,6 +17630,9 @@ Decl *Sema::ActOnFriendTypeDecl(Scope *S, const DeclSpec &DS,
   D->setAccess(AS_public);
   CurContext->addDecl(D);
 
+  if (Invalid)
+    D->setInvalidDecl();
+
   return D;
 }
 
diff --git a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
index f93cd113988ae42..14ca29f1bc3f1aa 100644
--- a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
@@ -1442,8 +1442,47 @@ Decl *TemplateDeclInstantiator::VisitFriendDecl(FriendDecl *D) {
     if (D->isUnsupportedFriend()) {
       InstTy = Ty;
     } else {
-      InstTy = SemaRef.SubstType(Ty, TemplateArgs,
-                                 D->getLocation(), DeclarationName());
+      if (D->isPackExpansion()) {
+        SmallVector<UnexpandedParameterPack, 2> Unexpanded;
+        SemaRef.collectUnexpandedParameterPacks(Ty->getTypeLoc(), Unexpanded);
+        assert(!Unexpanded.empty() && "Pack expansion without packs");
+
+        bool ShouldExpand = true;
+        bool RetainExpansion = false;
+        std::optional<unsigned> NumExpansions;
+        if (SemaRef.CheckParameterPacksForExpansion(
+                D->getEllipsisLoc(), D->getSourceRange(), Unexpanded,
+                TemplateArgs, ShouldExpand, RetainExpansion, NumExpansions))
+          return nullptr;
+
+        assert(!RetainExpansion &&
+               "should never retain an expansion for a variadic friend decl");
+
+        if (ShouldExpand) {
+          SmallVector<FriendDecl *> Decls;
+          for (unsigned I = 0; I != *NumExpansions; I++) {
+            Sema::ArgumentPackSubstitutionIndexRAII SubstIndex(SemaRef, I);
+            TypeSourceInfo *TSI = SemaRef.SubstType(
+                Ty, TemplateArgs, D->getEllipsisLoc(), DeclarationName());
+            if (!TSI)
+              return nullptr;
+
+            auto FD =
+                FriendDecl::Create(SemaRef.Context, Owner, D->getLocation(),
+                                   TSI, D->getFriendLoc());
+
+            FD->setAccess(AS_public);
+            Owner->addDecl(FD);
+            Decls.push_back(FD);
+          }
+
+          // Just drop this node; we have no use for it anymore.
+          return nullptr;
+        }
+      }
+
+      InstTy = SemaRef.SubstType(Ty, TemplateArgs, D->getLocation(),
+                                 DeclarationName());
     }
     if (!InstTy)
       return nullptr;
diff --git a/clang/lib/Serialization/ASTReaderDecl.cpp b/clang/lib/Serialization/ASTReaderDecl.cpp
index a9199f7e50f5dc8..ef160228933c598 100644
--- a/clang/lib/Serialization/ASTReaderDecl.cpp
+++ b/clang/lib/Serialization/ASTReaderDecl.cpp
@@ -2361,6 +2361,7 @@ void ASTDeclReader::VisitFriendDecl(FriendDecl *D) {
   D->NextFriend = readDeclID().getRawValue();
   D->UnsupportedFriend = (Record.readInt() != 0);
   D->FriendLoc = readSourceLocation();
+  D->EllipsisLoc = readSourceLocation();
 }
 
 void ASTDeclReader::VisitFriendTemplateDecl(FriendTemplateDecl *D) {
diff --git a/clang/lib/Serialization/ASTWriterDecl.cpp b/clang/lib/Serialization/ASTWriterDecl.cpp
index 8a4ca54349e38f1..555f6325da646bf 100644
--- a/clang/lib/Serialization/ASTWriterDecl.cpp
+++ b/clang/lib/Serialization/ASTWriterDecl.cpp
@@ -1660,6 +1660,7 @@ void ASTDeclWriter::VisitFriendDecl(FriendDecl *D) {
   Record.AddDeclRef(D->getNextFriend());
   Record.push_back(D->UnsupportedFriend);
   Record.AddSourceLocation(D->FriendLoc);
+  Record.AddSourceLocation(D->EllipsisLoc);
   Code = serialization::DECL_FRIEND;
 }
 
diff --git a/clang/test/AST/ast-dump-funcs-json.cpp b/clang/test/AST/ast-dump-funcs-json.cpp
index 041d98f2713d32e..957df5cea6ec534 100644
--- a/clang/test/AST/ast-dump-funcs-json.cpp
+++ b/clang/test/AST/ast-dump-funcs-json.cpp
@@ -41,13 +41,18 @@ int main() {
   Test1(); // Causes this to be marked 'used'
 }
 
+template <typename ...Ts>
+struct TestFriends {
+    friend Ts...;
+};
+
 // NOTE: CHECK lines have been autogenerated by gen_ast_dump_json_test.py
 
 
 // CHECK-NOT: {{^}}Dumping
 // CHECK:  "kind": "CXXMethodDecl",
 // CHECK-NEXT:  "loc": {
-// CHECK-NEXT:   "offset": {{[0-9]+}},
+// CHECK-NEXT:   "offset": 124,
 // CHECK-NEXT:   "file": "{{.*}}",
 // CHECK-NEXT:   "line": 4,
 // CHECK-NEXT:   "col": 8,
@@ -55,12 +60,12 @@ int main() {
 // CHECK-NEXT:  },
 // CHECK-NEXT:  "range": {
 // CHECK-NEXT:   "begin": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 119,
 // CHECK-NEXT:    "col": 3,
 // CHECK-NEXT:    "tokLen": 4
 // CHECK-NEXT:   },
 // CHECK-NEXT:   "end": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 130,
 // CHECK-NEXT:    "col": 14,
 // CHECK-NEXT:    "tokLen": 1
 // CHECK-NEXT:   }
@@ -76,7 +81,7 @@ int main() {
 // CHECK-NOT: {{^}}Dumping
 // CHECK:  "kind": "CXXMethodDecl",
 // CHECK-NEXT:  "loc": {
-// CHECK-NEXT:   "offset": {{[0-9]+}},
+// CHECK-NEXT:   "offset": 140,
 // CHECK-NEXT:   "file": "{{.*}}",
 // CHECK-NEXT:   "line": 5,
 // CHECK-NEXT:   "col": 8,
@@ -84,12 +89,12 @@ int main() {
 // CHECK-NEXT:  },
 // CHECK-NEXT:  "range": {
 // CHECK-NEXT:   "begin": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 135,
 // CHECK-NEXT:    "col": 3,
 // CHECK-NEXT:    "tokLen": 4
 // CHECK-NEXT:   },
 // CHECK-NEXT:   "end": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 148,
 // CHECK-NEXT:    "col": 16,
 // CHECK-NEXT:    "tokLen": 5
 // CHECK-NEXT:   }
@@ -105,7 +110,7 @@ int main() {
 // CHECK-NOT: {{^}}Dumping
 // CHECK:  "kind": "CXXMethodDecl",
 // CHECK-NEXT:  "loc": {
-// CHECK-NEXT:   "offset": {{[0-9]+}},
+// CHECK-NEXT:   "offset": 162,
 // CHECK-NEXT:   "file": "{{.*}}",
 // CHECK-NEXT:   "line": 6,
 // CHECK-NEXT:   "col": 8,
@@ -113,12 +118,12 @@ int main() {
 // CHECK-NEXT:  },
 // CHECK-NEXT:  "range": {
 // CHECK-NEXT:   "begin": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 157,
 // CHECK-NEXT:    "col": 3,
 // CHECK-NEXT:    "tokLen": 4
 // CHECK-NEXT:   },
 // CHECK-NEXT:   "end": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 170,
 // CHECK-NEXT:    "col": 16,
 // CHECK-NEXT:    "tokLen": 8
 // CHECK-NEXT:   }
@@ -134,7 +139,7 @@ int main() {
 // CHECK-NOT: {{^}}Dumping
 // CHECK:  "kind": "CXXMethodDecl",
 // CHECK-NEXT:  "loc": {
-// CHECK-NEXT:   "offset": {{[0-9]+}},
+// CHECK-NEXT:   "offset": 187,
 // CHECK-NEXT:   "file": "{{.*}}",
 // CHECK-NEXT:   "line": 7,
 // CHECK-NEXT:   "col": 8,
@@ -142,12 +147,12 @@ int main() {
 // CHECK-NEXT:  },
 // CHECK-NEXT:  "range": {
 // CHECK-NEXT:   "begin": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 182,
 // CHECK-NEXT:    "col": 3,
 // CHECK-NEXT:    "tokLen": 4
 // CHECK-NEXT:   },
 // CHECK-NEXT:   "end": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 195,
 // CHECK-NEXT:    "col": 16,
 // CHECK-NEXT:    "tokLen": 1
 // CHECK-NEXT:   }
@@ -163,7 +168,7 @@ int main() {
 // CHECK-NOT: {{^}}Dumping
 // CHECK:  "kind": "CXXMethodDecl",
 // CHECK-NEXT:  "loc": {
-// CHECK-NEXT:   "offset": {{[0-9]+}},
+// CHECK-NEXT:   "offset": 205,
 // CHECK-NEXT:   "file": "{{.*}}",
 // CHECK-NEXT:   "line": 8,
 // CHECK-NEXT:   "col": 8,
@@ -171,12 +176,12 @@ int main() {
 // CHECK-NEXT:  },
 // CHECK-NEXT:  "range": {
 // CHECK-NEXT:   "begin": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 200,
 // CHECK-NEXT:    "col": 3,
 // CHECK-NEXT:    "tokLen": 4
 // CHECK-NEXT:   },
 // CHECK-NEXT:   "end": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 213,
 // CHECK-NEXT:    "col": 16,
 // CHECK-NEXT:    "tokLen": 2
 // CHECK-NEXT:   }
@@ -192,7 +197,7 @@ int main() {
 // CHECK-NOT: {{^}}Dumping
 // CHECK:  "kind": "CXXMethodDecl",
 // CHECK-NEXT:  "loc": {
-// CHECK-NEXT:   "offset": {{[0-9]+}},
+// CHECK-NEXT:   "offset": 232,
 // CHECK-NEXT:   "file": "{{.*}}",
 // CHECK-NEXT:   "line": 9,
 // CHECK-NEXT:   "col": 16,
@@ -200,12 +205,12 @@ int main() {
 // CHECK-NEXT:  },
 // CHECK-NEXT:  "range": {
 // CHECK-NEXT:   "begin": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 219,
 // CHECK-NEXT:    "col": 3,
 // CHECK-NEXT:    "tokLen": 7
 // CHECK-NEXT:   },
 // CHECK-NEXT:   "end": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 253,
 // CHECK-NEXT:    "col": 37,
 // CHECK-NEXT:    "tokLen": 1
 // CHECK-NEXT:   }
@@ -221,18 +226,18 @@ int main() {
 // CHECK-NEXT:    "id": "0x{{.*}}",
 // CHECK-NEXT:    "kind": "ParmVarDecl",
 // CHECK-NEXT:    "loc": {
-// CHECK-NEXT:     "offset": {{[0-9]+}},
+// CHECK-NEXT:     "offset": 243,
 // CHECK-NEXT:     "col": 27,
 // CHECK-NEXT:     "tokLen": 1
 // CHECK-NEXT:    },
 // CHECK-NEXT:    "range": {
 // CHECK-NEXT:     "begin": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 238,
 // CHECK-NEXT:      "col": 22,
 // CHECK-NEXT:      "tokLen": 5
 // CHECK-NEXT:     },
 // CHECK-NEXT:     "end": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 238,
 // CHECK-NEXT:      "col": 22,
 // CHECK-NEXT:      "tokLen": 5
 // CHECK-NEXT:     }
@@ -245,18 +250,18 @@ int main() {
 // CHECK-NEXT:    "id": "0x{{.*}}",
 // CHECK-NEXT:    "kind": "ParmVarDecl",
 // CHECK-NEXT:    "loc": {
-// CHECK-NEXT:     "offset": {{[0-9]+}},
+// CHECK-NEXT:     "offset": 249,
 // CHECK-NEXT:     "col": 33,
 // CHECK-NEXT:     "tokLen": 1
 // CHECK-NEXT:    },
 // CHECK-NEXT:    "range": {
 // CHECK-NEXT:     "begin": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 245,
 // CHECK-NEXT:      "col": 29,
 // CHECK-NEXT:      "tokLen": 3
 // CHECK-NEXT:     },
 // CHECK-NEXT:     "end": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 251,
 // CHECK-NEXT:      "col": 35,
 // CHECK-NEXT:      "tokLen": 2
 // CHECK-NEXT:     }
@@ -271,12 +276,12 @@ int main() {
 // CHECK-NEXT:      "kind": "IntegerLiteral",
 // CHECK-NEXT:      "range": {
 // CHECK-NEXT:       "begin": {
-// CHECK-NEXT:        "offset": {{[0-9]+}},
+// CHECK-NEXT:        "offset": 251,
 // CHECK-NEXT:        "col": 35,
 // CHECK-NEXT:        "tokLen": 2
 // CHECK-NEXT:       },
 // CHECK-NEXT:       "end": {
-// CHECK-NEXT:        "offset": {{[0-9]+}},
+// CHECK-NEXT:        "offset": 251,
 // CHECK-NEXT:        "col": 35,
 // CHECK-NEXT:        "tokLen": 2
 // CHECK-NEXT:       }
@@ -296,7 +301,7 @@ int main() {
 // CHECK-NOT: {{^}}Dumping
 // CHECK:  "kind": "CXXMethodDecl",
 // CHECK-NEXT:  "loc": {
-// CHECK-NEXT:   "offset": {{[0-9]+}},
+// CHECK-NEXT:   "offset": 271,
 // CHECK-NEXT:   "file": "{{.*}}",
 // CHECK-NEXT:   "line": 10,
 // CHECK-NEXT:   "col": 16,
@@ -304,12 +309,12 @@ int main() {
 // CHECK-NEXT:  },
 // CHECK-NEXT:  "range": {
 // CHECK-NEXT:   "begin": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 258,
 // CHECK-NEXT:    "col": 3,
 // CHECK-NEXT:    "tokLen": 7
 // CHECK-NEXT:   },
 // CHECK-NEXT:   "end": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 281,
 // CHECK-NEXT:    "col": 26,
 // CHECK-NEXT:    "tokLen": 1
 // CHECK-NEXT:   }
@@ -327,7 +332,7 @@ int main() {
 // CHECK-NOT: {{^}}Dumping
 // CHECK:  "kind": "CXXMethodDecl",
 // CHECK-NEXT:  "loc": {
-// CHECK-NEXT:   "offset": {{[0-9]+}},
+// CHECK-NEXT:   "offset": 343,
 // CHECK-NEXT:   "file": "{{.*}}",
 // CHECK-NEXT:   "line": 14,
 // CHECK-NEXT:   "col": 8,
@@ -335,12 +340,12 @@ int main() {
 // CHECK-NEXT:  },
 // CHECK-NEXT:  "range": {
 // CHECK-NEXT:   "begin": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 338,
 // CHECK-NEXT:    "col": 3,
 // CHECK-NEXT:    "tokLen": 4
 // CHECK-NEXT:   },
 // CHECK-NEXT:   "end": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 367,
 // CHECK-NEXT:    "col": 32,
 // CHECK-NEXT:    "tokLen": 8
 // CHECK-NEXT:   }
@@ -355,18 +360,18 @@ int main() {
 // CHECK-NEXT:    "id": "0x{{.*}}",
 // CHECK-NEXT:    "kind": "ParmVarDecl",
 // CHECK-NEXT:    "loc": {
-// CHECK-NEXT:     "offset": {{[0-9]+}},
+// CHECK-NEXT:     "offset": 354,
 // CHECK-NEXT:     "col": 19,
 // CHECK-NEXT:     "tokLen": 1
 // CHECK-NEXT:    },
 // CHECK-NEXT:    "range": {
 // CHECK-NEXT:     "begin": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 349,
 // CHECK-NEXT:      "col": 14,
 // CHECK-NEXT:      "tokLen": 5
 // CHECK-NEXT:     },
 // CHECK-NEXT:     "end": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 349,
 // CHECK-NEXT:      "col": 14,
 // CHECK-NEXT:      "tokLen": 5
 // CHECK-NEXT:     }
@@ -379,18 +384,18 @@ int main() {
 // CHECK-NEXT:    "id": "0x{{.*}}",
 // CHECK-NEXT:    "kind": "ParmVarDecl",
 // CHECK-NEXT:    "loc": {
-// CHECK-NEXT:     "offset": {{[0-9]+}},
+// CHECK-NEXT:     "offset": 360,
 // CHECK-NEXT:     "col": 25,
 // CHECK-NEXT:     "tokLen": 1
 // CHECK-NEXT:    },
 // CHECK-NEXT:    "range": {
 // CHECK-NEXT:     "begin": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 356,
 // CHECK-NEXT:      "col": 21,
 // CHECK-NEXT:      "tokLen": 3
 // CHECK-NEXT:     },
 // CHECK-NEXT:     "end": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 362,
 // CHECK-NEXT:      "col": 27,
 // CHECK-NEXT:      "tokLen": 3
 // CHECK-NEXT:     }
@@ -405,12 +410,12 @@ int main() {
 // CHECK-NEXT:      "kind": "IntegerLiteral",
 // CHECK-NEXT:      "range": {
 // CHECK-NEXT:       "begin": {
-// CHECK-NEXT:        "offset": {{[0-9]+}},
+// CHECK-NEXT:        "offset": 362,
 // CHECK-NEXT:        "col": 27,
 // CHECK-NEXT:        "tokLen": 3
 // CHECK-NEXT:       },
 // CHECK-NEXT:       "end": {
-// CHECK-NEXT:        "offset": {{[0-9]+}},
+// CHECK-NEXT:        "offset": 362,
 // CHECK-NEXT:        "col": 27,
 // CHECK-NEXT:        "tokLen": 3
 // CHECK-NEXT:       }
@@ -428,12 +433,12 @@ int main() {
 // CHECK-NEXT:    "kind": "OverrideAttr",
 // CHECK-NEXT:    "range": {
 // CHECK-NEXT:     "begin": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 367,
 // CHECK-NEXT:      "col": 32,
 // CHECK-NEXT:      "tokLen": 8
 // CHECK-NEXT:     },
 // CHECK-NEXT:     "end": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 367,
 // CHECK-NEXT:      "col": 32,
 // CHECK-NEXT:      "tokLen": 8
 // CHECK-NEXT:     }
@@ -446,7 +451,7 @@ int main() {
 // CHECK-NOT: {{^}}Dumping
 // CHECK:  "kind": "CXXMethodDecl",
 // CHECK-NEXT:  "loc": {
-// CHECK-NEXT:   "offset": {{[0-9]+}},
+// CHECK-NEXT:   "offset": 399,
 // CHECK-NEXT:   "file": "{{.*}}",
 // CHECK-NEXT:   "line": 18,
 // CHECK-NEXT:   "col": 8,
@@ -454,12 +459,12 @@ int main() {
 // CHECK-NEXT:  },
 // CHECK-NEXT:  "range": {
 // CHECK-NEXT:   "begin": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 394,
 // CHECK-NEXT:    "col": 3,
 // CHECK-NEXT:    "tokLen": 4
 // CHECK-NEXT:   },
 // CHECK-NEXT:   "end": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 405,
 // CHECK-NEXT:    "col": 14,
 // CHECK-NEXT:    "tokLen": 1
 // CHECK-NEXT:   }
@@ -475,7 +480,7 @@ int main() {
 // CHECK-NOT: {{^}}Dumping
 // CHECK:  "kind": "CXXMethodDecl",
 // CHECK-NEXT:  "loc": {
-// CHECK-NEXT:   "offset": {{[0-9]+}},
+// CHECK-NEXT:   "offset": 419,
 // CHECK-NEXT:   "file": "{{.*}}",
 // CHECK-NEXT:   "line": 20,
 // CHECK-NEXT:   "col": 9,
@@ -483,12 +488,12 @@ int main() {
 // CHECK-NEXT:  },
 // CHECK-NEXT:  "range": {
 // CHECK-NEXT:   "begin": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 411,
 // CHECK-NEXT:    "col": 1,
 // CHECK-NEXT:    "tokLen": 4
 // CHECK-NEXT:   },
 // CHECK-NEXT:   "end": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 428,
 // CHECK-NEXT:    "col": 18,
 // CHECK-NEXT:    "tokLen": 1
 // CHECK-NEXT:   }
@@ -506,12 +511,12 @@ int main() {
 // CHECK-NEXT:    "kind": "CompoundStmt",
 // CHECK-NEXT:    "range": {
 // CHECK-NEXT:     "begin": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 427,
 // CHECK-NEXT:      "col": 17,
 // CHECK-NEXT:      "tokLen": 1
 // CHECK-NEXT:     },
 // CHECK-NEXT:     "end": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 428,
 // CHECK-NEXT:      "col": 18,
 // CHECK-NEXT:      "tokLen": 1
 // CHECK-NEXT:     }
@@ -524,7 +529,7 @@ int main() {
 // CHECK-NOT: {{^}}Dumping
 // CHECK:  "kind": "FunctionDecl",
 // CHECK-NEXT:  "loc": {
-// CHECK-NEXT:   "offset": {{[0-9]+}},
+// CHECK-NEXT:   "offset": 446,
 // CHECK-NEXT:   "file": "{{.*}}",
 // CHECK-NEXT:   "line": 22,
 // CHECK-NEXT:   "col": 6,
@@ -532,12 +537,12 @@ int main() {
 // CHECK-NEXT:  },
 // CHECK-NEXT:  "range": {
 // CHECK-NEXT:   "begin": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 441,
 // CHECK-NEXT:    "col": 1,
 // CHECK-NEXT:    "tokLen": 4
 // CHECK-NEXT:   },
 // CHECK-NEXT:   "end": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 452,
 // CHECK-NEXT:    "col": 12,
 // CHECK-NEXT:    "tokLen": 1
 // CHECK-NEXT:   }
@@ -554,7 +559,7 @@ int main() {
 // CHECK-NOT: {{^}}Dumping
 // CHECK:  "kind": "FunctionDecl",
 // CHECK-NEXT:  "loc": {
-// CHECK-NEXT:   "offset": {{[0-9]+}},
+// CHECK-NEXT:   "offset": 460,
 // CHECK-NEXT:   "file": "{{.*}}",
 // CHECK-NEXT:   "line": 23,
 // CHECK-NEXT:   "col": 6,
@@ -562,12 +567,12 @@ int main() {
 // CHECK-NEXT:  },
 // CHECK-NEXT:  "range": {
 // CHECK-NEXT:   "begin": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 455,
 // CHECK-NEXT:    "col": 1,
 // CHECK-NEXT:    "tokLen": 4
 // CHECK-NEXT:   },
 // CHECK-NEXT:   "end": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 470,
 // CHECK-NEXT:    "col": 16,
 // CHECK-NEXT:    "tokLen": 1
 // CHECK-NEXT:   }
@@ -583,7 +588,7 @@ int main() {
 // CHECK-NOT: {{^}}Dumping
 // CHECK:  "kind": "FunctionDecl",
 // CHECK-NEXT:  "loc": {
-// CHECK-NEXT:   "offset": {{[0-9]+}},
+// CHECK-NEXT:   "offset": 478,
 // CHECK-NEXT:   "file": "{{.*}}",
 // CHECK-NEXT:   "line": 24,
 // CHECK-NEXT:   "col": 6,
@@ -591,12 +596,12 @@ int main() {
 // CHECK-NEXT:  },
 // CHECK-NEXT:  "range": {
 // CHECK-NEXT:   "begin": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 473,
 // CHECK-NEXT:    "col": 1,
 // CHECK-NEXT:    "tokLen": 4
 // CHECK-NEXT:   },
 // CHECK-NEXT:   "end": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 496,
 // CHECK-NEXT:    "col": 24,
 // CHECK-NEXT:    "tokLen": 1
 // CHECK-NEXT:   }
@@ -611,18 +616,18 @@ int main() {
 // CHECK-NEXT:    "id": "0x{{.*}}",
 // CHECK-NEXT:    "kind": "ParmVarDecl",
 // CHECK-NEXT:    "loc": {
-// CHECK-NEXT:     "offset": {{[0-9]+}},
+// CHECK-NEXT:     "offset": 488,
 // CHECK-NEXT:     "col": 16,
 // CHECK-NEXT:     "tokLen": 1
 // CHECK-NEXT:    },
 // CHECK-NEXT:    "range": {
 // CHECK-NEXT:     "begin": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 484,
 // CHECK-NEXT:      "col": 12,
 // CHECK-NEXT:      "tokLen": 3
 // CHECK-NEXT:     },
 // CHECK-NEXT:     "end": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 488,
 // CHECK-NEXT:      "col": 16,
 // CHECK-NEXT:      "tokLen": 1
 // CHECK-NEXT:     }
@@ -636,18 +641,18 @@ int main() {
 // CHECK-NEXT:    "id": "0x{{.*}}",
 // CHECK-NEXT:    "kind": "ParmVarDecl",
 // CHECK-NEXT:    "loc": {
-// CHECK-NEXT:     "offset": {{[0-9]+}},
+// CHECK-NEXT:     "offset": 495,
 // CHECK-NEXT:     "col": 23,
 // CHECK-NEXT:     "tokLen": 1
 // CHECK-NEXT:    },
 // CHECK-NEXT:    "range": {
 // CHECK-NEXT:     "begin": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 491,
 // CHECK-NEXT:      "col": 19,
 // CHECK-NEXT:      "tokLen": 3
 // CHECK-NEXT:     },
 // CHECK-NEXT:     "end": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 495,
 // CHECK-NEXT:      "col": 23,
 // CHECK-NEXT:      "tokLen": 1
 // CHECK-NEXT:     }
@@ -664,7 +669,7 @@ int main() {
 // CHECK-NOT: {{^}}Dumping
 // CHECK:  "kind": "FunctionDecl",
 // CHECK-NEXT:  "loc": {
-// CHECK-NEXT:   "offset": {{[0-9]+}},
+// CHECK-NEXT:   "offset": 504,
 // CHECK-NEXT:   "file": "{{.*}}",
 // CHECK-NEXT:   "line": 25,
 // CHECK-NEXT:   "col": 6,
@@ -672,12 +677,12 @@ int main() {
 // CHECK-NEXT:  },
 // CHECK-NEXT:  "range": {
 // CHECK-NEXT:   "begin": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 499,
 // CHECK-NEXT:    "col": 1,
 // CHECK-NEXT:    "tokLen": 4
 // CHECK-NEXT:   },
 // CHECK-NEXT:   "end": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 527,
 // CHECK-NEXT:    "col": 29,
 // CHECK-NEXT:    "tokLen": 1
 // CHECK-NEXT:   }
@@ -692,18 +697,18 @@ int main() {
 // CHECK-NEXT:    "id": "0x{{.*}}",
 // CHECK-NEXT:    "kind": "ParmVarDecl",
 // CHECK-NEXT:    "loc": {
-// CHECK-NEXT:     "offset": {{[0-9]+}},
+// CHECK-NEXT:     "offset": 514,
 // CHECK-NEXT:     "col": 16,
 // CHECK-NEXT:     "tokLen": 1
 // CHECK-NEXT:    },
 // CHECK-NEXT:    "range": {
 // CHECK-NEXT:     "begin": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 510,
 // CHECK-NEXT:      "col": 12,
 // CHECK-NEXT:      "tokLen": 3
 // CHECK-NEXT:     },
 // CHECK-NEXT:     "end": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 514,
 // CHECK-NEXT:      "col": 16,
 // CHECK-NEXT:      "tokLen": 1
 // CHECK-NEXT:     }
@@ -717,18 +722,18 @@ int main() {
 // CHECK-NEXT:    "id": "0x{{.*}}",
 // CHECK-NEXT:    "kind": "ParmVarDecl",
 // CHECK-NEXT:    "loc": {
-// CHECK-NEXT:     "offset": {{[0-9]+}},
+// CHECK-NEXT:     "offset": 521,
 // CHECK-NEXT:     "col": 23,
 // CHECK-NEXT:     "tokLen": 1
 // CHECK-NEXT:    },
 // CHECK-NEXT:    "range": {
 // CHECK-NEXT:     "begin": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 517,
 // CHECK-NEXT:      "col": 19,
 // CHECK-NEXT:      "tokLen": 3
 // CHECK-NEXT:     },
 // CHECK-NEXT:     "end": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 525,
 // CHECK-NEXT:      "col": 27,
 // CHECK-NEXT:      "tokLen": 2
 // CHECK-NEXT:     }
@@ -744,12 +749,12 @@ int main() {
 // CHECK-NEXT:      "kind": "IntegerLiteral",
 // CHECK-NEXT:      "range": {
 // CHECK-NEXT:       "begin": {
-// CHECK-NEXT:        "offset": {{[0-9]+}},
+// CHECK-NEXT:        "offset": 525,
 // CHECK-NEXT:        "col": 27,
 // CHECK-NEXT:        "tokLen": 2
 // CHECK-NEXT:       },
 // CHECK-NEXT:       "end": {
-// CHECK-NEXT:        "offset": {{[0-9]+}},
+// CHECK-NEXT:        "offset": 525,
 // CHECK-NEXT:        "col": 27,
 // CHECK-NEXT:        "tokLen": 2
 // CHECK-NEXT:       }
@@ -769,7 +774,7 @@ int main() {
 // CHECK-NOT: {{^}}Dumping
 // CHECK:  "kind": "FunctionDecl",
 // CHECK-NEXT:  "loc": {
-// CHECK-NEXT:   "offset": {{[0-9]+}},
+// CHECK-NEXT:   "offset": 545,
 // CHECK-NEXT:   "file": "{{.*}}",
 // CHECK-NEXT:   "line": 26,
 // CHECK-NEXT:   "col": 16,
@@ -777,12 +782,12 @@ int main() {
 // CHECK-NEXT:  },
 // CHECK-NEXT:  "range": {
 // CHECK-NEXT:   "begin": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 530,
 // CHECK-NEXT:    "col": 1,
 // CHECK-NEXT:    "tokLen": 9
 // CHECK-NEXT:   },
 // CHECK-NEXT:   "end": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 555,
 // CHECK-NEXT:    "col": 26,
 // CHECK-NEXT:    "tokLen": 1
 // CHECK-NEXT:   }
@@ -799,7 +804,7 @@ int main() {
 // CHECK-NOT: {{^}}Dumping
 // CHECK:  "kind": "FunctionDecl",
 // CHECK-NEXT:  "loc": {
-// CHECK-NEXT:   "offset": {{[0-9]+}},
+// CHECK-NEXT:   "offset": 570,
 // CHECK-NEXT:   "file": "{{.*}}",
 // CHECK-NEXT:   "line": 27,
 // CHECK-NEXT:   "col": 13,
@@ -807,12 +812,12 @@ int main() {
 // CHECK-NEXT:  },
 // CHECK-NEXT:  "range": {
 // CHECK-NEXT:   "begin": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 558,
 // CHECK-NEXT:    "col": 1,
 // CHECK-NEXT:    "tokLen": 6
 // CHECK-NEXT:   },
 // CHECK-NEXT:   "end": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 580,
 // CHECK-NEXT:    "col": 23,
 // CHECK-NEXT:    "tokLen": 1
 // CHECK-NEXT:   }
@@ -829,7 +834,7 @@ int main() {
 // CHECK-NOT: {{^}}Dumping
 // CHECK:  "kind": "FunctionDecl",
 // CHECK-NEXT:  "loc": {
-// CHECK-NEXT:   "offset": {{[0-9]+}},
+// CHECK-NEXT:   "offset": 595,
 // CHECK-NEXT:   "file": "{{.*}}",
 // CHECK-NEXT:   "line": 28,
 // CHECK-NEXT:   "col": 13,
@@ -837,12 +842,12 @@ int main() {
 // CHECK-NEXT:  },
 // CHECK-NEXT:  "range": {
 // CHECK-NEXT:   "begin": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 583,
 // CHECK-NEXT:    "col": 1,
 // CHECK-NEXT:    "tokLen": 6
 // CHECK-NEXT:   },
 // CHECK-NEXT:   "end": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 605,
 // CHECK-NEXT:    "col": 23,
 // CHECK-NEXT:    "tokLen": 1
 // CHECK-NEXT:   }
@@ -859,7 +864,7 @@ int main() {
 // CHECK-NOT: {{^}}Dumping
 // CHECK:  "kind": "FunctionDecl",
 // CHECK-NEXT:  "loc": {
-// CHECK-NEXT:   "offset": {{[0-9]+}},
+// CHECK-NEXT:   "offset": 620,
 // CHECK-NEXT:   "file": "{{.*}}",
 // CHECK-NEXT:   "line": 29,
 // CHECK-NEXT:   "col": 13,
@@ -867,12 +872,12 @@ int main() {
 // CHECK-NEXT:  },
 // CHECK-NEXT:  "range": {
 // CHECK-NEXT:   "begin": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 608,
 // CHECK-NEXT:    "col": 1,
 // CHECK-NEXT:    "tokLen": 6
 // CHECK-NEXT:   },
 // CHECK-NEXT:   "end": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 630,
 // CHECK-NEXT:    "col": 23,
 // CHECK-NEXT:    "tokLen": 1
 // CHECK-NEXT:   }
@@ -889,7 +894,7 @@ int main() {
 // CHECK-NOT: {{^}}Dumping
 // CHECK:  "kind": "FunctionDecl",
 // CHECK-NEXT:  "loc": {
-// CHECK-NEXT:   "offset": {{[0-9]+}},
+// CHECK-NEXT:   "offset": 638,
 // CHECK-NEXT:   "file": "{{.*}}",
 // CHECK-NEXT:   "line": 30,
 // CHECK-NEXT:   "col": 6,
@@ -897,12 +902,12 @@ int main() {
 // CHECK-NEXT:  },
 // CHECK-NEXT:  "range": {
 // CHECK-NEXT:   "begin": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 633,
 // CHECK-NEXT:    "col": 1,
 // CHECK-NEXT:    "tokLen": 4
 // CHECK-NEXT:   },
 // CHECK-NEXT:   "end": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 650,
 // CHECK-NEXT:    "col": 18,
 // CHECK-NEXT:    "tokLen": 8
 // CHECK-NEXT:   }
@@ -918,7 +923,7 @@ int main() {
 // CHECK-NOT: {{^}}Dumping
 // CHECK:  "kind": "FunctionDecl",
 // CHECK-NEXT:  "loc": {
-// CHECK-NEXT:   "offset": {{[0-9]+}},
+// CHECK-NEXT:   "offset": 665,
 // CHECK-NEXT:   "file": "{{.*}}",
 // CHECK-NEXT:   "line": 31,
 // CHECK-NEXT:   "col": 6,
@@ -926,12 +931,12 @@ int main() {
 // CHECK-NEXT:  },
 // CHECK-NEXT:  "range": {
 // CHECK-NEXT:   "begin": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 660,
 // CHECK-NEXT:    "col": 1,
 // CHECK-NEXT:    "tokLen": 4
 // CHECK-NEXT:   },
 // CHECK-NEXT:   "end": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 692,
 // CHECK-NEXT:    "col": 33,
 // CHECK-NEXT:    "tokLen": 1
 // CHECK-NEXT:   }
@@ -947,7 +952,7 @@ int main() {
 // CHECK-NOT: {{^}}Dumping
 // CHECK:  "kind": "FunctionDecl",
 // CHECK-NEXT:  "loc": {
-// CHECK-NEXT:   "offset": {{[0-9]+}},
+// CHECK-NEXT:   "offset": 700,
 // CHECK-NEXT:   "file": "{{.*}}",
 // CHECK-NEXT:   "line": 32,
 // CHECK-NEXT:   "col": 6,
@@ -955,12 +960,12 @@ int main() {
 // CHECK-NEXT:  },
 // CHECK-NEXT:  "range": {
 // CHECK-NEXT:   "begin": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 695,
 // CHECK-NEXT:    "col": 1,
 // CHECK-NEXT:    "tokLen": 4
 // CHECK-NEXT:   },
 // CHECK-NEXT:   "end": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 723,
 // CHECK-NEXT:    "col": 29,
 // CHECK-NEXT:    "tokLen": 1
 // CHECK-NEXT:   }
@@ -976,7 +981,7 @@ int main() {
 // CHECK-NOT: {{^}}Dumping
 // CHECK:  "kind": "FunctionTemplateDecl",
 // CHECK-NEXT:  "loc": {
-// CHECK-NEXT:   "offset": {{[0-9]+}},
+// CHECK-NEXT:   "offset": 751,
 // CHECK-NEXT:   "file": "{{.*}}",
 // CHECK-NEXT:   "line": 35,
 // CHECK-NEXT:   "col": 3,
@@ -984,13 +989,13 @@ int main() {
 // CHECK-NEXT:  },
 // CHECK-NEXT:  "range": {
 // CHECK-NEXT:   "begin": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 727,
 // CHECK-NEXT:    "line": 34,
 // CHECK-NEXT:    "col": 1,
 // CHECK-NEXT:    "tokLen": 8
 // CHECK-NEXT:   },
 // CHECK-NEXT:   "end": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 760,
 // CHECK-NEXT:    "line": 35,
 // CHECK-NEXT:    "col": 12,
 // CHECK-NEXT:    "tokLen": 1
@@ -1002,19 +1007,19 @@ int main() {
 // CHECK-NEXT:    "id": "0x{{.*}}",
 // CHECK-NEXT:    "kind": "TemplateTypeParmDecl",
 // CHECK-NEXT:    "loc": {
-// CHECK-NEXT:     "offset": {{[0-9]+}},
+// CHECK-NEXT:     "offset": 746,
 // CHECK-NEXT:     "line": 34,
 // CHECK-NEXT:     "col": 20,
 // CHECK-NEXT:     "tokLen": 1
 // CHECK-NEXT:    },
 // CHECK-NEXT:    "range": {
 // CHECK-NEXT:     "begin": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 737,
 // CHECK-NEXT:      "col": 11,
 // CHECK-NEXT:      "tokLen": 8
 // CHECK-NEXT:     },
 // CHECK-NEXT:     "end": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 746,
 // CHECK-NEXT:      "col": 20,
 // CHECK-NEXT:      "tokLen": 1
 // CHECK-NEXT:     }
@@ -1029,19 +1034,19 @@ int main() {
 // CHECK-NEXT:    "id": "0x{{.*}}",
 // CHECK-NEXT:    "kind": "FunctionDecl",
 // CHECK-NEXT:    "loc": {
-// CHECK-NEXT:     "offset": {{[0-9]+}},
+// CHECK-NEXT:     "offset": 751,
 // CHECK-NEXT:     "line": 35,
 // CHECK-NEXT:     "col": 3,
 // CHECK-NEXT:     "tokLen": 6
 // CHECK-NEXT:    },
 // CHECK-NEXT:    "range": {
 // CHECK-NEXT:     "begin": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 749,
 // CHECK-NEXT:      "col": 1,
 // CHECK-NEXT:      "tokLen": 1
 // CHECK-NEXT:     },
 // CHECK-NEXT:     "end": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 760,
 // CHECK-NEXT:      "col": 12,
 // CHECK-NEXT:      "tokLen": 1
 // CHECK-NEXT:     }
@@ -1055,18 +1060,18 @@ int main() {
 // CHECK-NEXT:      "id": "0x{{.*}}",
 // CHECK-NEXT:      "kind": "ParmVarDecl",
 // CHECK-NEXT:      "loc": {
-// CHECK-NEXT:       "offset": {{[0-9]+}},
+// CHECK-NEXT:       "offset": 760,
 // CHECK-NEXT:       "col": 12,
 // CHECK-NEXT:       "tokLen": 1
 // CHECK-NEXT:      },
 // CHECK-NEXT:      "range": {
 // CHECK-NEXT:       "begin": {
-// CHECK-NEXT:        "offset": {{[0-9]+}},
+// CHECK-NEXT:        "offset": 758,
 // CHECK-NEXT:        "col": 10,
 // CHECK-NEXT:        "tokLen": 1
 // CHECK-NEXT:       },
 // CHECK-NEXT:       "end": {
-// CHECK-NEXT:        "offset": {{[0-9]+}},
+// CHECK-NEXT:        "offset": 759,
 // CHECK-NEXT:        "col": 11,
 // CHECK-NEXT:        "tokLen": 1
 // CHECK-NEXT:       }
@@ -1084,7 +1089,7 @@ int main() {
 // CHECK-NOT: {{^}}Dumping
 // CHECK:  "kind": "FunctionDecl",
 // CHECK-NEXT:  "loc": {
-// CHECK-NEXT:   "offset": {{[0-9]+}},
+// CHECK-NEXT:   "offset": 769,
 // CHECK-NEXT:   "file": "{{.*}}",
 // CHECK-NEXT:   "line": 37,
 // CHECK-NEXT:   "col": 6,
@@ -1092,12 +1097,12 @@ int main() {
 // CHECK-NEXT:  },
 // CHECK-NEXT:  "range": {
 // CHECK-NEXT:   "begin": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 764,
 // CHECK-NEXT:    "col": 1,
 // CHECK-NEXT:    "tokLen": 4
 // CHECK-NEXT:   },
 // CHECK-NEXT:   "end": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 782,
 // CHECK-NEXT:    "col": 19,
 // CHECK-NEXT:    "tokLen": 1
 // CHECK-NEXT:   }
@@ -1112,18 +1117,18 @@ int main() {
 // CHECK-NEXT:    "id": "0x{{.*}}",
 // CHECK-NEXT:    "kind": "ParmVarDecl",
 // CHECK-NEXT:    "loc": {
-// CHECK-NEXT:     "offset": {{[0-9]+}},
+// CHECK-NEXT:     "offset": 779,
 // CHECK-NEXT:     "col": 16,
 // CHECK-NEXT:     "tokLen": 1
 // CHECK-NEXT:    },
 // CHECK-NEXT:    "range": {
 // CHECK-NEXT:     "begin": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 776,
 // CHECK-NEXT:      "col": 13,
 // CHECK-NEXT:      "tokLen": 3
 // CHECK-NEXT:     },
 // CHECK-NEXT:     "end": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 776,
 // CHECK-NEXT:      "col": 13,
 // CHECK-NEXT:      "tokLen": 3
 // CHECK-NEXT:     }
@@ -1137,12 +1142,12 @@ int main() {
 // CHECK-NEXT:    "kind": "CompoundStmt",
 // CHECK-NEXT:    "range": {
 // CHECK-NEXT:     "begin": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 781,
 // CHECK-NEXT:      "col": 18,
 // CHECK-NEXT:      "tokLen": 1
 // CHECK-NEXT:     },
 // CHECK-NEXT:     "end": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 782,
 // CHECK-NEXT:      "col": 19,
 // CHECK-NEXT:      "tokLen": 1
 // CHECK-NEXT:     }
@@ -1155,7 +1160,7 @@ int main() {
 // CHECK-NOT: {{^}}Dumping
 // CHECK:  "kind": "FunctionDecl",
 // CHECK-NEXT:  "loc": {
-// CHECK-NEXT:   "offset": {{[0-9]+}},
+// CHECK-NEXT:   "offset": 789,
 // CHECK-NEXT:   "file": "{{.*}}",
 // CHECK-NEXT:   "line": 38,
 // CHECK-NEXT:   "col": 6,
@@ -1163,12 +1168,12 @@ int main() {
 // CHECK-NEXT:  },
 // CHECK-NEXT:  "range": {
 // CHECK-NEXT:   "begin": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 784,
 // CHECK-NEXT:    "col": 1,
 // CHECK-NEXT:    "tokLen": 4
 // CHECK-NEXT:   },
 // CHECK-NEXT:   "end": {
-// CHECK-NEXT:    "offset": {{[0-9]+}},
+// CHECK-NEXT:    "offset": 807,
 // CHECK-NEXT:    "col": 24,
 // CHECK-NEXT:    "tokLen": 1
 // CHECK-NEXT:   }
@@ -1184,18 +1189,18 @@ int main() {
 // CHECK-NEXT:    "id": "0x{{.*}}",
 // CHECK-NEXT:    "kind": "ParmVarDecl",
 // CHECK-NEXT:    "loc": {
-// CHECK-NEXT:     "offset": {{[0-9]+}},
+// CHECK-NEXT:     "offset": 799,
 // CHECK-NEXT:     "col": 16,
 // CHECK-NEXT:     "tokLen": 1
 // CHECK-NEXT:    },
 // CHECK-NEXT:    "range": {
 // CHECK-NEXT:     "begin": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 796,
 // CHECK-NEXT:      "col": 13,
 // CHECK-NEXT:      "tokLen": 3
 // CHECK-NEXT:     },
 // CHECK-NEXT:     "end": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 796,
 // CHECK-NEXT:      "col": 13,
 // CHECK-NEXT:      "tokLen": 3
 // CHECK-NEXT:     }
@@ -1209,12 +1214,12 @@ int main() {
 // CHECK-NEXT:    "kind": "CompoundStmt",
 // CHECK-NEXT:    "range": {
 // CHECK-NEXT:     "begin": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 806,
 // CHECK-NEXT:      "col": 23,
 // CHECK-NEXT:      "tokLen": 1
 // CHECK-NEXT:     },
 // CHECK-NEXT:     "end": {
-// CHECK-NEXT:      "offset": {{[0-9]+}},
+// CHECK-NEXT:      "offset": 807,
 // CHECK-NEXT:      "col": 24,
 // CHECK-NEXT:      "tokLen": 1
 // CHECK-NEXT:     }
@@ -1222,3 +1227,189 @@ int main() {
 // CHECK-NEXT:   }
 // CHECK-NEXT:  ]
 // CHECK-NEXT: }
+
+
+// CHECK-NOT: {{^}}Dumping
+// CHECK:  "kind": "ClassTemplateDecl",
+// CHECK-NEXT:  "loc": {
+// CHECK-NEXT:   "offset": 905,
+// CHECK-NEXT:   "file": "{{.*}}",
+// CHECK-NEXT:   "line": 45,
+// CHECK-NEXT:   "col": 8,
+// CHECK-NEXT:   "tokLen": 11
+// CHECK-NEXT:  },
+// CHECK-NEXT:  "range": {
+// CHECK-NEXT:   "begin": {
+// CHECK-NEXT:    "offset": 872,
+// CHECK-NEXT:    "line": 44,
+// CHECK-NEXT:    "col": 1,
+// CHECK-NEXT:    "tokLen": 8
+// CHECK-NEXT:   },
+// CHECK-NEXT:   "end": {
+// CHECK-NEXT:    "offset": 937,
+// CHECK-NEXT:    "line": 47,
+// CHECK-NEXT:    "col": 1,
+// CHECK-NEXT:    "tokLen": 1
+// CHECK-NEXT:   }
+// CHECK-NEXT:  },
+// CHECK-NEXT:  "name": "TestFriends",
+// CHECK-NEXT:  "inner": [
+// CHECK-NEXT:   {
+// CHECK-NEXT:    "id": "0x{{.*}}",
+// CHECK-NEXT:    "kind": "TemplateTypeParmDecl",
+// CHECK-NEXT:    "loc": {
+// CHECK-NEXT:     "offset": 894,
+// CHECK-NEXT:     "line": 44,
+// CHECK-NEXT:     "col": 23,
+// CHECK-NEXT:     "tokLen": 2
+// CHECK-NEXT:    },
+// CHECK-NEXT:    "range": {
+// CHECK-NEXT:     "begin": {
+// CHECK-NEXT:      "offset": 882,
+// CHECK-NEXT:      "col": 11,
+// CHECK-NEXT:      "tokLen": 8
+// CHECK-NEXT:     },
+// CHECK-NEXT:     "end": {
+// CHECK-NEXT:      "offset": 894,
+// CHECK-NEXT:      "col": 23,
+// CHECK-NEXT:      "tokLen": 2
+// CHECK-NEXT:     }
+// CHECK-NEXT:    },
+// CHECK-NEXT:    "isReferenced": true,
+// CHECK-NEXT:    "name": "Ts",
+// CHECK-NEXT:    "tagUsed": "typename",
+// CHECK-NEXT:    "depth": 0,
+// CHECK-NEXT:    "index": 0,
+// CHECK-NEXT:    "isParameterPack": true
+// CHECK-NEXT:   },
+// CHECK-NEXT:   {
+// CHECK-NEXT:    "id": "0x{{.*}}",
+// CHECK-NEXT:    "kind": "CXXRecordDecl",
+// CHECK-NEXT:    "loc": {
+// CHECK-NEXT:     "offset": 905,
+// CHECK-NEXT:     "line": 45,
+// CHECK-NEXT:     "col": 8,
+// CHECK-NEXT:     "tokLen": 11
+// CHECK-NEXT:    },
+// CHECK-NEXT:    "range": {
+// CHECK-NEXT:     "begin": {
+// CHECK-NEXT:      "offset": 898,
+// CHECK-NEXT:      "col": 1,
+// CHECK-NEXT:      "tokLen": 6
+// CHECK-NEXT:     },
+// CHECK-NEXT:     "end": {
+// CHECK-NEXT:      "offset": 937,
+// CHECK-NEXT:      "line": 47,
+// CHECK-NEXT:      "col": 1,
+// CHECK-NEXT:      "tokLen": 1
+// CHECK-NEXT:     }
+// CHECK-NEXT:    },
+// CHECK-NEXT:    "name": "TestFriends",
+// CHECK-NEXT:    "tagUsed": "struct",
+// CHECK-NEXT:    "completeDefinition": true,
+// CHECK-NEXT:    "definitionData": {
+// CHECK-NEXT:     "canConstDefaultInit": true,
+// CHECK-NEXT:     "copyAssign": {
+// CHECK-NEXT:      "hasConstParam": true,
+// CHECK-NEXT:      "implicitHasConstParam": true,
+// CHECK-NEXT:      "needsImplicit": true,
+// CHECK-NEXT:      "simple": true,
+// CHECK-NEXT:      "trivial": true
+// CHECK-NEXT:     },
+// CHECK-NEXT:     "copyCtor": {
+// CHECK-NEXT:      "hasConstParam": true,
+// CHECK-NEXT:      "implicitHasConstParam": true,
+// CHECK-NEXT:      "needsImplicit": true,
+// CHECK-NEXT:      "simple": true,
+// CHECK-NEXT:      "trivial": true
+// CHECK-NEXT:     },
+// CHECK-NEXT:     "defaultCtor": {
+// CHECK-NEXT:      "defaultedIsConstexpr": true,
+// CHECK-NEXT:      "exists": true,
+// CHECK-NEXT:      "isConstexpr": true,
+// CHECK-NEXT:      "needsImplicit": true,
+// CHECK-NEXT:      "trivial": true
+// CHECK-NEXT:     },
+// CHECK-NEXT:     "dtor": {
+// CHECK-NEXT:      "irrelevant": true,
+// CHECK-NEXT:      "needsImplicit": true,
+// CHECK-NEXT:      "simple": true,
+// CHECK-NEXT:      "trivial": true
+// CHECK-NEXT:     },
+// CHECK-NEXT:     "hasConstexprNonCopyMoveConstructor": true,
+// CHECK-NEXT:     "isAggregate": true,
+// CHECK-NEXT:     "isEmpty": true,
+// CHECK-NEXT:     "isLiteral": true,
+// CHECK-NEXT:     "isPOD": true,
+// CHECK-NEXT:     "isStandardLayout": true,
+// CHECK-NEXT:     "isTrivial": true,
+// CHECK-NEXT:     "isTriviallyCopyable": true,
+// CHECK-NEXT:     "moveAssign": {
+// CHECK-NEXT:      "exists": true,
+// CHECK-NEXT:      "needsImplicit": true,
+// CHECK-NEXT:      "simple": true,
+// CHECK-NEXT:      "trivial": true
+// CHECK-NEXT:     },
+// CHECK-NEXT:     "moveCtor": {
+// CHECK-NEXT:      "exists": true,
+// CHECK-NEXT:      "needsImplicit": true,
+// CHECK-NEXT:      "simple": true,
+// CHECK-NEXT:      "trivial": true
+// CHECK-NEXT:     }
+// CHECK-NEXT:    },
+// CHECK-NEXT:    "inner": [
+// CHECK-NEXT:     {
+// CHECK-NEXT:      "id": "0x{{.*}}",
+// CHECK-NEXT:      "kind": "CXXRecordDecl",
+// CHECK-NEXT:      "loc": {
+// CHECK-NEXT:       "offset": 905,
+// CHECK-NEXT:       "line": 45,
+// CHECK-NEXT:       "col": 8,
+// CHECK-NEXT:       "tokLen": 11
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "range": {
+// CHECK-NEXT:       "begin": {
+// CHECK-NEXT:        "offset": 898,
+// CHECK-NEXT:        "col": 1,
+// CHECK-NEXT:        "tokLen": 6
+// CHECK-NEXT:       },
+// CHECK-NEXT:       "end": {
+// CHECK-NEXT:        "offset": 905,
+// CHECK-NEXT:        "col": 8,
+// CHECK-NEXT:        "tokLen": 11
+// CHECK-NEXT:       }
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "isImplicit": true,
+// CHECK-NEXT:      "name": "TestFriends",
+// CHECK-NEXT:      "tagUsed": "struct"
+// CHECK-NEXT:     },
+// CHECK-NEXT:     {
+// CHECK-NEXT:      "id": "0x{{.*}}",
+// CHECK-NEXT:      "kind": "FriendDecl",
+// CHECK-NEXT:      "loc": {
+// CHECK-NEXT:       "offset": 930,
+// CHECK-NEXT:       "line": 46,
+// CHECK-NEXT:       "col": 12,
+// CHECK-NEXT:       "tokLen": 2
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "range": {
+// CHECK-NEXT:       "begin": {
+// CHECK-NEXT:        "offset": 923,
+// CHECK-NEXT:        "col": 5,
+// CHECK-NEXT:        "tokLen": 6
+// CHECK-NEXT:       },
+// CHECK-NEXT:       "end": {
+// CHECK-NEXT:        "offset": 932,
+// CHECK-NEXT:        "col": 14,
+// CHECK-NEXT:        "tokLen": 3
+// CHECK-NEXT:       }
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "type": {
+// CHECK-NEXT:       "qualType": "Ts"
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "isPackExpansion": true
+// CHECK-NEXT:     }
+// CHECK-NEXT:    ]
+// CHECK-NEXT:   }
+// CHECK-NEXT:  ]
+// CHECK-NEXT: }
diff --git a/clang/test/AST/cxx2c-variadic-friends.cpp b/clang/test/AST/cxx2c-variadic-friends.cpp
new file mode 100644
index 000000000000000..fc84e7346fe0364
--- /dev/null
+++ b/clang/test/AST/cxx2c-variadic-friends.cpp
@@ -0,0 +1,81 @@
+// RUN: %clang_cc1 -fsyntax-only -ast-dump -std=c++2c %s | FileCheck %s
+// RUN: %clang_cc1 -ast-print -std=c++2c %s | FileCheck %s --check-prefix=PRINT
+// RUN: %clang_cc1 -emit-pch -std=c++2c -o %t %s
+// RUN: %clang_cc1 -x c++ -std=c++2c -include-pch %t -ast-dump-all /dev/null
+
+struct S;
+template <typename> struct TS; // #template
+
+// CHECK-LABEL: CXXRecordDecl {{.*}} struct Friends
+// PRINT-LABEL: struct Friends {
+struct Friends {
+  // CHECK: FriendDecl {{.*}} 'int'
+  // CHECK-NEXT: FriendDecl {{.*}} 'long'
+  // PRINT-NEXT: friend int;
+  // PRINT-NEXT: friend long;
+  friend int, long;
+
+  // CHECK-NEXT: FriendDecl {{.*}} 'int'
+  // CHECK-NEXT: FriendDecl {{.*}} 'long'
+  // CHECK-NEXT: FriendDecl {{.*}} 'char'
+  // PRINT-NEXT: friend int;
+  // PRINT-NEXT: friend long;
+  // PRINT-NEXT: friend char;
+  friend int, long, char;
+
+  // CHECK-NEXT: FriendDecl {{.*}} 'S'
+  // PRINT-NEXT: friend S;
+  friend S;
+
+  // CHECK-NEXT: FriendDecl {{.*}} 'S'
+  // CHECK-NEXT: FriendDecl {{.*}} 'S'
+  // CHECK-NEXT: FriendDecl {{.*}} 'S'
+  // PRINT-NEXT: friend S;
+  // PRINT-NEXT: friend S;
+  // PRINT-NEXT: friend S;
+  friend S, S, S;
+
+  // CHECK-NEXT: FriendDecl
+  // CHECK-NEXT: ClassTemplateDecl {{.*}} friend TS
+  // PRINT-NEXT: friend template <typename> struct TS;
+  template <typename> friend struct TS;
+};
+
+namespace specialisations {
+template<class T>
+struct C {
+  template<class U> struct Nested;
+};
+
+struct N {
+  template<class U> class C;
+};
+
+// CHECK-LABEL: ClassTemplateDecl {{.*}} Variadic
+// PRINT-LABEL: template <typename ...Pack> struct Variadic {
+template <typename ...Pack> struct Variadic {
+  // CHECK: FriendDecl {{.*}} 'Pack'...
+  // CHECK-NEXT: FriendDecl {{.*}} 'long'
+  // CHECK-NEXT: FriendDecl {{.*}} 'Pack'...
+  // PRINT-NEXT: friend Pack...;
+  // PRINT-NEXT: friend long;
+  // PRINT-NEXT: friend Pack...;
+  friend Pack..., long, Pack...;
+
+  // CHECK-NEXT: FriendDecl {{.*}} 'TS<Pack>'...
+  // PRINT-NEXT: friend TS<Pack>...;
+  friend TS<Pack>...;
+};
+
+// CHECK-LABEL: ClassTemplateDecl {{.*}} S2
+// PRINT-LABEL: template <class ...Ts> struct S2 {
+template<class ...Ts> struct S2 {
+  // CHECK: FriendDecl {{.*}} 'class C<Ts>':'C<Ts>'...
+  // PRINT-NEXT: friend class C<Ts>...;
+  friend class C<Ts>...;
+
+  // CHECK-NEXT: FriendDecl {{.*}} 'class N::C<Ts>':'C<Ts>'...
+  // PRINT-NEXT: friend class N::C<Ts>...
+  friend class N::C<Ts>...;
+};
+}
diff --git a/clang/test/CXX/drs/cwg29xx.cpp b/clang/test/CXX/drs/cwg29xx.cpp
new file mode 100644
index 000000000000000..8cac9f283980b6b
--- /dev/null
+++ b/clang/test/CXX/drs/cwg29xx.cpp
@@ -0,0 +1,25 @@
+// RUN: %clang_cc1 -std=c++98 -pedantic-errors -verify=expected,cxx98 %s
+// RUN: %clang_cc1 -std=c++11 -pedantic-errors -verify=expected %s
+// RUN: %clang_cc1 -std=c++14 -pedantic-errors -verify=expected %s
+// RUN: %clang_cc1 -std=c++17 -pedantic-errors -verify=expected %s
+// RUN: %clang_cc1 -std=c++20 -pedantic-errors -verify=expected %s
+// RUN: %clang_cc1 -std=c++23 -pedantic-errors -verify=expected %s
+// RUN: %clang_cc1 -std=c++2c -pedantic-errors -verify=expected %s
+
+namespace cwg2917 { // cwg2917: 20 open 2024-07-30
+template <typename>
+class Foo;
+
+template<class ...> // cxx98-error {{variadic templates are a C++11 extension}}
+struct C {
+  struct Nested { };
+};
+
+struct S {
+  template <typename>
+  friend class Foo, int; // expected-error {{a friend declaration that befriends a template must contain exactly one type-specifier}}
+
+  template <typename ...Ts> // cxx98-error {{variadic templates are a C++11 extension}}
+  friend class C<Ts>::Nested...; // expected-error {{friend declaration expands pack 'Ts' that is declared it its own template parameter list}}
+};
+} // namespace cwg2917
diff --git a/clang/test/Lexer/cxx-features.cpp b/clang/test/Lexer/cxx-features.cpp
index 08b732132228bac..1c51013ca06f774 100644
--- a/clang/test/Lexer/cxx-features.cpp
+++ b/clang/test/Lexer/cxx-features.cpp
@@ -34,6 +34,10 @@
 
 // --- C++26 features ---
 
+#if check(variadic_friend, 202403, 202403, 202403, 202403, 202403, 202403, 202403)
+#error "wrong value for __cpp_variadic_friend"
+#endif
+
 #if check(deleted_function, 202403, 202403, 202403, 202403, 202403, 202403, 202403)
 #error "wrong value for __cpp_deleted_function"
 #endif
diff --git a/clang/test/Parser/cxx2c-variadic-friends-ext-diags.cpp b/clang/test/Parser/cxx2c-variadic-friends-ext-diags.cpp
new file mode 100644
index 000000000000000..ffcc97ffd635298
--- /dev/null
+++ b/clang/test/Parser/cxx2c-variadic-friends-ext-diags.cpp
@@ -0,0 +1,16 @@
+// RUN: %clang_cc1 -std=c++2c -verify=compat -fsyntax-only -Wpre-c++26-compat %s
+// RUN: %clang_cc1 -std=c++11 -verify=pre2c -fsyntax-only -Wc++26-extensions %s
+
+struct S {
+  friend int, long, char; // compat-warning {{variadic 'friend' declarations are incompatible with C++ standards before C++2c}} \
+                          // pre2c-warning {{variadic 'friend' declarations are a C++2c extension}}
+};
+
+template <typename ...Types>
+struct TS {
+  friend Types...; // compat-warning {{variadic 'friend' declarations are incompatible with C++ standards before C++2c}} \
+                   // pre2c-warning {{variadic 'friend' declarations are a C++2c extension}}
+
+  friend int, Types..., Types...; // compat-warning {{variadic 'friend' declarations are incompatible with C++ standards before C++2c}} \
+                                  // pre2c-warning {{variadic 'friend' declarations are a C++2c extension}}
+};
diff --git a/clang/test/Parser/cxx2c-variadic-friends.cpp b/clang/test/Parser/cxx2c-variadic-friends.cpp
new file mode 100644
index 000000000000000..b7da3e611048129
--- /dev/null
+++ b/clang/test/Parser/cxx2c-variadic-friends.cpp
@@ -0,0 +1,91 @@
+// RUN: %clang_cc1 -fsyntax-only -verify -std=c++2c %s
+
+template <typename> struct TS; // #template
+
+struct Errors {
+  friend int, int;
+  friend int, long, char;
+
+  // We simply diagnose and ignore the '...' here.
+  friend float...; // expected-error {{pack expansion does not contain any unexpanded parameter packs}}
+
+  friend short..., unsigned, unsigned short...; // expected-error 2 {{pack expansion does not contain any unexpanded parameter packs}}
+
+  template <typename>
+  friend struct TS, int; // expected-error {{a friend declaration that befriends a template must contain exactly one type-specifier}}
+
+  double friend; // expected-error {{'friend' must appear first in a non-function declaration}}
+  double friend, double; // expected-error {{expected member name or ';' after declaration specifiers}}
+};
+
+template <typename>
+struct C { template<class T> class Nested; };
+
+template <typename, typename>
+struct D { template<class T> class Nested; };
+
+template <bool>
+struct E { template<class T> class Nested; };
+
+template<class... Ts> // expected-note {{template parameter is declared here}}
+struct VS {
+  friend Ts...;
+
+  friend class Ts...; // expected-error {{declaration of 'Ts' shadows template parameter}}
+  // expected-error@-1 {{pack expansion does not contain any unexpanded parameter packs}}
+
+  // TODO: Fix-it hint to insert '...'.
+  friend Ts; // expected-error {{friend declaration contains unexpanded parameter pack}}
+
+  template<class... Us>
+  friend Us...; // expected-error {{friend type templates must use an elaborated type}}
+
+  template<class... Us> // expected-note {{is declared here}}
+  friend class Us...; // expected-error {{declaration of 'Us' shadows template parameter}}
+
+  template<class U>
+  friend class C<Ts>::template Nested<U>...; // expected-error {{cannot specialize a dependent template}}
+
+  template<class... Us>
+  friend class C<Ts...>::template Nested<Us>...; // expected-error {{cannot specialize a dependent template}}
+
+  // Nonsense (see CWG 2917).
+  template<class... Us>
+  friend class C<Us>::Nested...; // expected-error {{friend declaration expands pack 'Us' that is declared it its own template parameter list}}
+
+  template<bool... Bs>
+  friend class E<Bs>::Nested...; // expected-error {{friend declaration expands pack 'Bs' that is declared it its own template parameter list}}
+
+  // FIXME: Both of these should be valid, but we can't handle these at
+  // the moment because the NNS is dependent.
+  template<class ...T>
+  friend class TS<Ts>::Nested...; // expected-warning {{dependent nested name specifier 'TS<Ts>::' for friend template declaration is not supported; ignoring this friend declaration}}
+
+  template<class T>
+  friend class D<T, Ts>::Nested...; // expected-warning {{dependent nested name specifier 'D<T, Ts>::' for friend class declaration is not supported; turning off access control for 'VS'}}
+};
+
+namespace length_mismatch {
+struct A {
+  template <typename...>
+  struct Nested {
+    struct Foo{};
+  };
+};
+template <typename ...Ts>
+struct S {
+  template <typename ...Us>
+  struct T {
+    // expected-error@+2 {{pack expansion contains parameter packs 'Ts' and 'Us' that have different lengths (1 vs. 2)}}
+    // expected-error@+1 {{pack expansion contains parameter packs 'Ts' and 'Us' that have different lengths (2 vs. 1)}}
+    friend class Ts::template Nested<Us>::Foo...;
+  };
+};
+
+void f() {
+  S<A>::T<int> s;
+  S<A, A>::T<int, long> s2;
+  S<A>::T<int, long> s3; // expected-note {{in instantiation of}}
+  S<A, A>::T<int> s4; // expected-note {{in instantiation of}}
+}
+}
diff --git a/clang/test/SemaCXX/cxx2c-variadic-friends.cpp b/clang/test/SemaCXX/cxx2c-variadic-friends.cpp
new file mode 100644
index 000000000000000..a4d7c8078338d2a
--- /dev/null
+++ b/clang/test/SemaCXX/cxx2c-variadic-friends.cpp
@@ -0,0 +1,156 @@
+// RUN: %clang_cc1 -fsyntax-only -verify -std=c++2c %s
+
+struct A;
+struct B;
+struct C;
+
+struct S {};
+template <typename> struct TS {};
+
+template <typename ...Pack>
+class X {
+  friend Pack...;
+  static void f() { } // expected-note {{declared private here}}
+};
+
+class Y {
+  friend A, B, C;
+  static void g() { } // expected-note {{declared private here}}
+};
+
+struct A {
+  A() {
+    X<A>::f();
+    Y::g();
+  };
+};
+
+struct B {
+  B() {
+    X<B, C>::f();
+    Y::g();
+  };
+};
+
+struct C {
+  C() {
+    X<A, B, C>::f();
+    Y::g();
+  };
+};
+
+struct D {
+  D() {
+    X<A, B, C>::f(); // expected-error {{'f' is a private member of 'X<A, B, C>'}}
+    Y::g(); // expected-error {{'g' is a private member of 'Y'}}
+  };
+};
+
+void f1() {
+  A a;
+  B b;
+  C c;
+  D d;
+}
+
+template <typename ...Pack>
+struct Z {
+  template <template <typename> class Template>
+  struct Inner {
+    friend Template<Pack>...;
+  };
+};
+
+void f2() {
+  Z<int, long, char> z;
+  Z<int, long, char>::Inner<TS> inner;
+}
+
+namespace p2893r3_examples {
+template<class... Ts>
+class Passkey {
+  friend Ts...;
+  Passkey() {} // expected-note {{declared private here}}
+};
+
+class Foo;
+class Bar;
+class Baz;
+
+class C {
+public:
+  void f(Passkey<Foo, Bar, Baz>);
+};
+
+class Foo {
+  Foo() { C c; c.f({}); }
+};
+
+class Bar {
+  Bar() { C c; c.f({}); }
+};
+
+class Baz {
+  Baz() { C c; c.f({}); }
+};
+
+class Quux {
+  Quux() { C c; c.f({}); } // expected-error {{calling a private constructor of class 'p2893r3_examples::Passkey<p2893r3_examples::Foo, p2893r3_examples::Bar, p2893r3_examples::Baz>'}}
+};
+
+template<class Derived, class MsgT>
+struct Receiver {
+  void receive(MsgT) {
+    static_cast<Derived*>(this)->private_ += 1;
+  }
+};
+
+template<class... MsgTs>
+struct Dispatcher : Receiver<Dispatcher<MsgTs...>, MsgTs>... {
+  using Receiver<Dispatcher, MsgTs>::receive...;
+  friend Receiver<Dispatcher, MsgTs>...;
+
+private:
+  int private_;
+};
+
+void f() {
+  Dispatcher<int, float> d;
+  d.receive(0);
+  d.receive(0.0f);
+}
+} // namespace p2893r3_examples
+
+namespace p2893r3_note {
+template <class... Ts> class R {
+  friend Ts...;
+};
+
+template <class... Ts, class... Us>
+class R<R<Ts...>, R<Us...>> {
+  friend Ts::Nested..., Us...;
+};
+
+struct E { struct Nested; };
+R<R<E>, R<C, int>> rr;
+} // namespace p2893r3_note
+
+namespace template_template {
+template <typename U, template <typename> typename... Friend>
+class S {
+  friend class Friend<U>...;
+  static constexpr int a = 42;
+};
+
+template <typename U>
+struct T {
+  static_assert(S<U, T>::a == 42);
+  static_assert(S<U, T>::a == 43); // expected-error {{static assertion failed due to requirement 'S<int, template_template::T>::a == 43'}} \
+                                   // expected-note {{expression evaluates to '42 == 43'}}
+};
+
+void f() {
+  T<int> t; // expected-note {{in instantiation of}}
+}
+}
+
diff --git a/clang/www/cxx_dr_status.html b/clang/www/cxx_dr_status.html
index 9b0de55483d2755..34ca6f76bb2ce62 100755
--- a/clang/www/cxx_dr_status.html
+++ b/clang/www/cxx_dr_status.html
@@ -17318,7 +17318,7 @@ <h2 id="cxxdr">C++ defect report implementation status</h2>
     <td><a href="https://cplusplus.github.io/CWG/issues/2917.html">2917</a></td>
     <td>open</td>
     <td>Disallow multiple <I>friend-type-specifier</I>s for a friend template</td>
-    <td align="center">Not resolved</td>
+    <td title="Clang 20 implements 2024-07-30 resolution" align="center">Not Resolved*</td>
   </tr>
   <tr class="open" id="2918">
     <td><a href="https://cplusplus.github.io/CWG/issues/2918.html">2918</a></td>
diff --git a/clang/www/cxx_status.html b/clang/www/cxx_status.html
index a6ded8be3ae9e59..faee8b578b62423 100755
--- a/clang/www/cxx_status.html
+++ b/clang/www/cxx_status.html
@@ -202,7 +202,7 @@ <h2 id="cxx26">C++2c implementation status</h2>
  <tr>
   <td>Variadic friends</td>
   <td><a href="https://wg21.link/P2893R3">P2893R3</a></td>
-  <td class="none" align="center">No</td>
+  <td class="unreleased" align="center">Clang 20</td>
  </tr>
  <!-- Summer 2024 papers (St Louis) -->
  <tr>

From 932538199818554cee7347c17de3f4e004b75257 Mon Sep 17 00:00:00 2001
From: Amy Kwan <amy.kwan1@ibm.com>
Date: Thu, 15 Aug 2024 15:25:54 -0400
Subject: [PATCH 068/441] [PowerPC][GlobalMerge] Enable GlobalMerge by default
 on AIX (#101226)

This patch turns on the GlobalMerge pass by default on AIX and updates
LIT tests accordingly.
---
 llvm/lib/Target/PowerPC/PPCTargetMachine.cpp               | 7 ++++++-
 llvm/test/CodeGen/PowerPC/merge-private.ll                 | 6 ++++++
 llvm/test/CodeGen/PowerPC/mergeable-string-pool.ll         | 4 ++--
 .../test/DebugInfo/Symbolize/XCOFF/xcoff-symbolize-data.ll | 2 +-
 4 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp b/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp
index 763b6edb1c09fba..6ce345dd44138dc 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp
+++ b/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp
@@ -500,7 +500,12 @@ void PPCPassConfig::addIRPasses() {
 }
 
 bool PPCPassConfig::addPreISel() {
-  if (EnableGlobalMerge)
+  // The GlobalMerge pass is intended to be on by default on AIX.
+  // Specifying the command line option overrides the AIX default.
+  if ((EnableGlobalMerge.getNumOccurrences() > 0)
+          ? EnableGlobalMerge
+          : (TM->getTargetTriple().isOSAIX() &&
+             getOptLevel() != CodeGenOptLevel::None))
     addPass(
         createGlobalMergePass(TM, GlobalMergeMaxOffset, false, false, true));
 
diff --git a/llvm/test/CodeGen/PowerPC/merge-private.ll b/llvm/test/CodeGen/PowerPC/merge-private.ll
index 6ed2d6dfc542b76..0ca706abb275fcc 100644
--- a/llvm/test/CodeGen/PowerPC/merge-private.ll
+++ b/llvm/test/CodeGen/PowerPC/merge-private.ll
@@ -11,6 +11,12 @@
 ; RUN: llc -verify-machineinstrs -mtriple powerpc64-unknown-linux -mcpu=pwr8 \
 ; RUN:     -ppc-asm-full-reg-names -ppc-global-merge=true < %s | FileCheck %s \
 ; RUN:     --check-prefix=LINUX64BE
+; The below run line is added to ensure that the assembly corresponding to
+; the following check-prefix is generated by default on AIX (without any
+; options).
+; RUN: llc -verify-machineinstrs -mtriple powerpc64-ibm-aix-xcoff -mcpu=pwr8 \
+; RUN:     -ppc-asm-full-reg-names < %s | FileCheck %s \
+; RUN:     --check-prefix=AIX64
 
 @.str = private unnamed_addr constant [15 x i8] c"Private global\00", align 1
 @str = internal constant [16 x i8] c"Internal global\00", align 1
diff --git a/llvm/test/CodeGen/PowerPC/mergeable-string-pool.ll b/llvm/test/CodeGen/PowerPC/mergeable-string-pool.ll
index 81147d10cde6e7d..833ed9fa65acf18 100644
--- a/llvm/test/CodeGen/PowerPC/mergeable-string-pool.ll
+++ b/llvm/test/CodeGen/PowerPC/mergeable-string-pool.ll
@@ -1,6 +1,6 @@
-; RUN: llc -verify-machineinstrs -mtriple powerpc-ibm-aix-xcoff -mcpu=pwr8 \
+; RUN: llc -verify-machineinstrs -mtriple powerpc-ibm-aix-xcoff -mcpu=pwr8 -enable-global-merge=false \
 ; RUN:   -ppc-asm-full-reg-names < %s | FileCheck %s --check-prefixes=AIX32,AIXDATA
-; RUN: llc -verify-machineinstrs -mtriple powerpc64-ibm-aix-xcoff -mcpu=pwr8 \
+; RUN: llc -verify-machineinstrs -mtriple powerpc64-ibm-aix-xcoff -mcpu=pwr8 -enable-global-merge=false \
 ; RUN:   -ppc-asm-full-reg-names < %s | FileCheck %s --check-prefixes=AIX64,AIXDATA
 ; RUN: llc -verify-machineinstrs -mtriple powerpc64-unknown-linux -mcpu=pwr8 \
 ; RUN:   -ppc-asm-full-reg-names < %s | FileCheck %s --check-prefixes=LINUX64BE,LINUXDATA
diff --git a/llvm/test/DebugInfo/Symbolize/XCOFF/xcoff-symbolize-data.ll b/llvm/test/DebugInfo/Symbolize/XCOFF/xcoff-symbolize-data.ll
index 5432b59d583bacf..1a467ec72a75da2 100644
--- a/llvm/test/DebugInfo/Symbolize/XCOFF/xcoff-symbolize-data.ll
+++ b/llvm/test/DebugInfo/Symbolize/XCOFF/xcoff-symbolize-data.ll
@@ -5,7 +5,7 @@
 ;; AIX assembly syntax.
 
 ; REQUIRES: powerpc-registered-target
-; RUN: llc -filetype=obj -o %t -mtriple=powerpc-aix-ibm-xcoff < %s
+; RUN: llc -filetype=obj -o %t -mtriple=powerpc-aix-ibm-xcoff -ppc-global-merge=false < %s
 ; RUN: llvm-symbolizer --obj=%t 'DATA 0x60' 'DATA 0x61' 'DATA 0x64' 'DATA 0X68' \
 ; RUN:   'DATA 0x90' 'DATA 0x94' 'DATA 0X98' | \
 ; RUN:   FileCheck %s

From 3dea42f3e5edc5a571081b7c5d58cce2fc4b2671 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Thu, 15 Aug 2024 12:33:12 -0700
Subject: [PATCH 069/441] [TargetLowering] Don't call
 SelectionDAG::getTargetLoweringInfo() from TargetLowering methods. NFC
 (#104197)

If we are inside a TargetLowering method,
`SelectionDAG::getTargetLoweringInfo()` should be the same as `this`.
---
 .../CodeGen/SelectionDAG/TargetLowering.cpp   | 53 ++++++++-----------
 1 file changed, 21 insertions(+), 32 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 2c939967a5e1d9b..c1a87de93e73594 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -598,12 +598,11 @@ bool TargetLowering::ShrinkDemandedOp(SDValue Op, unsigned BitWidth,
 
   // Search for the smallest integer type with free casts to and from
   // Op's type. For expedience, just check power-of-2 integer types.
-  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   unsigned DemandedSize = DemandedBits.getActiveBits();
   for (unsigned SmallVTBits = llvm::bit_ceil(DemandedSize);
        SmallVTBits < BitWidth; SmallVTBits = NextPowerOf2(SmallVTBits)) {
     EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), SmallVTBits);
-    if (TLI.isTruncateFree(VT, SmallVT) && TLI.isZExtFree(SmallVT, VT)) {
+    if (isTruncateFree(VT, SmallVT) && isZExtFree(SmallVT, VT)) {
       // We found a type with free casts.
       SDValue X = DAG.getNode(
           Op.getOpcode(), dl, SmallVT,
@@ -1622,9 +1621,8 @@ bool TargetLowering::SimplifyDemandedBits(
             APInt Ones = APInt::getAllOnes(BitWidth);
             Ones = Op0Opcode == ISD::SHL ? Ones.shl(ShiftAmt)
                                          : Ones.lshr(ShiftAmt);
-            const TargetLowering &TLI = TLO.DAG.getTargetLoweringInfo();
             if ((DemandedBits & C->getAPIntValue()) == (DemandedBits & Ones) &&
-                TLI.isDesirableToCommuteXorWithShift(Op.getNode())) {
+                isDesirableToCommuteXorWithShift(Op.getNode())) {
               // If the xor constant is a demanded mask, do a 'not' before the
               // shift:
               // xor (X << ShiftC), XorC --> (not X) << ShiftC
@@ -3065,8 +3063,7 @@ bool TargetLowering::SimplifyDemandedVectorElts(
 
   KnownUndef = KnownZero = APInt::getZero(NumElts);
 
-  const TargetLowering &TLI = TLO.DAG.getTargetLoweringInfo();
-  if (!TLI.shouldSimplifyDemandedVectorElts(Op, TLO))
+  if (!shouldSimplifyDemandedVectorElts(Op, TLO))
     return false;
 
   // TODO: For now we assume we know nothing about scalable vectors.
@@ -4162,8 +4159,7 @@ SDValue TargetLowering::optimizeSetCCOfSignedTruncationCheck(
 
   // We don't want to do this in every single case.
   SelectionDAG &DAG = DCI.DAG;
-  if (!DAG.getTargetLoweringInfo().shouldTransformSignedTruncationCheck(
-          XVT, KeptBits))
+  if (!shouldTransformSignedTruncationCheck(XVT, KeptBits))
     return SDValue();
 
   // Unfold into:  sext_inreg(%x) cond %x
@@ -4187,10 +4183,9 @@ SDValue TargetLowering::optimizeSetCCByHoistingAndByConstFromLogicalShift(
   SDValue X, C, Y;
 
   SelectionDAG &DAG = DCI.DAG;
-  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
 
   // Look for '(C l>>/<< Y)'.
-  auto Match = [&NewShiftOpcode, &X, &C, &Y, &TLI, &DAG](SDValue V) {
+  auto Match = [&NewShiftOpcode, &X, &C, &Y, &DAG, this](SDValue V) {
     // The shift should be one-use.
     if (!V.hasOneUse())
       return false;
@@ -4216,7 +4211,7 @@ SDValue TargetLowering::optimizeSetCCByHoistingAndByConstFromLogicalShift(
 
     ConstantSDNode *XC =
         isConstOrConstSplat(X, /*AllowUndefs=*/true, /*AllowTruncation=*/true);
-    return TLI.shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
+    return shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
         X, XC, CC, Y, OldShiftOpcode, NewShiftOpcode, DAG);
   };
 
@@ -5114,7 +5109,6 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
   // Back to non-vector simplifications.
   // TODO: Can we do these for vector splats?
   if (auto *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
-    const TargetLowering &TLI = DAG.getTargetLoweringInfo();
     const APInt &C1 = N1C->getAPIntValue();
     EVT ShValTy = N0.getValueType();
 
@@ -5132,7 +5126,7 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
           // Perform the xform if the AND RHS is a single bit.
           unsigned ShCt = AndRHS->getAPIntValue().logBase2();
           if (AndRHS->getAPIntValue().isPowerOf2() &&
-              !TLI.shouldAvoidTransformToShift(ShValTy, ShCt)) {
+              !shouldAvoidTransformToShift(ShValTy, ShCt)) {
             return DAG.getNode(
                 ISD::TRUNCATE, dl, VT,
                 DAG.getNode(ISD::SRL, dl, ShValTy, N0,
@@ -5142,8 +5136,7 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
           // (X & 8) == 8  -->  (X & 8) >> 3
           // Perform the xform if C1 is a single bit.
           unsigned ShCt = C1.logBase2();
-          if (C1.isPowerOf2() &&
-              !TLI.shouldAvoidTransformToShift(ShValTy, ShCt)) {
+          if (C1.isPowerOf2() && !shouldAvoidTransformToShift(ShValTy, ShCt)) {
             return DAG.getNode(
                 ISD::TRUNCATE, dl, VT,
                 DAG.getNode(ISD::SRL, dl, ShValTy, N0,
@@ -5162,7 +5155,7 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
           const APInt &AndRHSC = AndRHS->getAPIntValue();
           if (AndRHSC.isNegatedPowerOf2() && C1.isSubsetOf(AndRHSC)) {
             unsigned ShiftBits = AndRHSC.countr_zero();
-            if (!TLI.shouldAvoidTransformToShift(ShValTy, ShiftBits)) {
+            if (!shouldAvoidTransformToShift(ShValTy, ShiftBits)) {
               SDValue Shift = DAG.getNode(
                   ISD::SRL, dl, ShValTy, N0.getOperand(0),
                   DAG.getShiftAmountConstant(ShiftBits, ShValTy, dl));
@@ -5191,7 +5184,7 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
         NewC.lshrInPlace(ShiftBits);
         if (ShiftBits && NewC.getSignificantBits() <= 64 &&
             isLegalICmpImmediate(NewC.getSExtValue()) &&
-            !TLI.shouldAvoidTransformToShift(ShValTy, ShiftBits)) {
+            !shouldAvoidTransformToShift(ShValTy, ShiftBits)) {
           SDValue Shift =
               DAG.getNode(ISD::SRL, dl, ShValTy, N0,
                           DAG.getShiftAmountConstant(ShiftBits, ShValTy, dl));
@@ -6234,8 +6227,7 @@ SDValue TargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
                               SelectionDAG &DAG,
                               SmallVectorImpl<SDNode *> &Created) const {
   AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
-  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
-  if (TLI.isIntDivCheap(N->getValueType(0), Attr))
+  if (isIntDivCheap(N->getValueType(0), Attr))
     return SDValue(N, 0); // Lower SDIV as SDIV
   return SDValue();
 }
@@ -6245,8 +6237,7 @@ TargetLowering::BuildSREMPow2(SDNode *N, const APInt &Divisor,
                               SelectionDAG &DAG,
                               SmallVectorImpl<SDNode *> &Created) const {
   AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
-  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
-  if (TLI.isIntDivCheap(N->getValueType(0), Attr))
+  if (isIntDivCheap(N->getValueType(0), Attr))
     return SDValue(N, 0); // Lower SREM as SREM
   return SDValue();
 }
@@ -11746,13 +11737,12 @@ bool TargetLowering::LegalizeSetCCCondCode(SelectionDAG &DAG, EVT VT,
                                            SDValue EVL, bool &NeedInvert,
                                            const SDLoc &dl, SDValue &Chain,
                                            bool IsSignaling) const {
-  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   MVT OpVT = LHS.getSimpleValueType();
   ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get();
   NeedInvert = false;
   assert(!EVL == !Mask && "VP Mask and EVL must either both be set or unset");
   bool IsNonVP = !EVL;
-  switch (TLI.getCondCodeAction(CCCode, OpVT)) {
+  switch (getCondCodeAction(CCCode, OpVT)) {
   default:
     llvm_unreachable("Unknown condition code action!");
   case TargetLowering::Legal:
@@ -11760,7 +11750,7 @@ bool TargetLowering::LegalizeSetCCCondCode(SelectionDAG &DAG, EVT VT,
     break;
   case TargetLowering::Expand: {
     ISD::CondCode InvCC = ISD::getSetCCSwappedOperands(CCCode);
-    if (TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) {
+    if (isCondCodeLegalOrCustom(InvCC, OpVT)) {
       std::swap(LHS, RHS);
       CC = DAG.getCondCode(InvCC);
       return true;
@@ -11768,13 +11758,13 @@ bool TargetLowering::LegalizeSetCCCondCode(SelectionDAG &DAG, EVT VT,
     // Swapping operands didn't work. Try inverting the condition.
     bool NeedSwap = false;
     InvCC = getSetCCInverse(CCCode, OpVT);
-    if (!TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) {
+    if (!isCondCodeLegalOrCustom(InvCC, OpVT)) {
       // If inverting the condition is not enough, try swapping operands
       // on top of it.
       InvCC = ISD::getSetCCSwappedOperands(InvCC);
       NeedSwap = true;
     }
-    if (TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) {
+    if (isCondCodeLegalOrCustom(InvCC, OpVT)) {
       CC = DAG.getCondCode(InvCC);
       NeedInvert = true;
       if (NeedSwap)
@@ -11788,18 +11778,18 @@ bool TargetLowering::LegalizeSetCCCondCode(SelectionDAG &DAG, EVT VT,
     default:
       llvm_unreachable("Don't know how to expand this condition!");
     case ISD::SETUO:
-      if (TLI.isCondCodeLegal(ISD::SETUNE, OpVT)) {
+      if (isCondCodeLegal(ISD::SETUNE, OpVT)) {
         CC1 = ISD::SETUNE;
         CC2 = ISD::SETUNE;
         Opc = ISD::OR;
         break;
       }
-      assert(TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) &&
+      assert(isCondCodeLegal(ISD::SETOEQ, OpVT) &&
              "If SETUE is expanded, SETOEQ or SETUNE must be legal!");
       NeedInvert = true;
       [[fallthrough]];
     case ISD::SETO:
-      assert(TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) &&
+      assert(isCondCodeLegal(ISD::SETOEQ, OpVT) &&
              "If SETO is expanded, SETOEQ must be legal!");
       CC1 = ISD::SETOEQ;
       CC2 = ISD::SETOEQ;
@@ -11812,9 +11802,8 @@ bool TargetLowering::LegalizeSetCCCondCode(SelectionDAG &DAG, EVT VT,
       // of SETOGT/SETOLT to be legal, the other can be emulated by swapping
       // the operands.
       CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO;
-      if (!TLI.isCondCodeLegal(CC2, OpVT) &&
-          (TLI.isCondCodeLegal(ISD::SETOGT, OpVT) ||
-           TLI.isCondCodeLegal(ISD::SETOLT, OpVT))) {
+      if (!isCondCodeLegal(CC2, OpVT) && (isCondCodeLegal(ISD::SETOGT, OpVT) ||
+                                          isCondCodeLegal(ISD::SETOLT, OpVT))) {
         CC1 = ISD::SETOGT;
         CC2 = ISD::SETOLT;
         Opc = ISD::OR;

From 9aa3b53ac52167aba6253a52805874948ce40c8f Mon Sep 17 00:00:00 2001
From: Ziqing Luo <ziqing@udel.edu>
Date: Thu, 15 Aug 2024 12:34:15 -0700
Subject: [PATCH 070/441] [-Wunsafe-buffer-usage] Fix a small bug recently
 found (#102953)

`QualType::isConstantArrayType()` checks canonical type. So a following
cast should be applied to canonical type as well:

```
if (Ty->isConstantArrayType())
  cast<ConstantArrayType>(Ty.getCanonicalType());  //  cast<ConstantArrayType>(Ty) is incorrect
```
---
 clang/lib/Analysis/UnsafeBufferUsage.cpp           | 14 +++++++-------
 ...fe-buffer-usage-in-container-span-construct.cpp |  3 +++
 2 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/clang/lib/Analysis/UnsafeBufferUsage.cpp b/clang/lib/Analysis/UnsafeBufferUsage.cpp
index 051381edabf0b25..95c288bfe3fd2f7 100644
--- a/clang/lib/Analysis/UnsafeBufferUsage.cpp
+++ b/clang/lib/Analysis/UnsafeBufferUsage.cpp
@@ -402,9 +402,9 @@ AST_MATCHER(CXXConstructExpr, isSafeSpanTwoParamConstruct) {
 
   QualType Arg0Ty = Arg0->IgnoreImplicit()->getType();
 
-  if (Arg0Ty->isConstantArrayType()) {
-    const APSInt ConstArrSize =
-        APSInt(cast<ConstantArrayType>(Arg0Ty)->getSize());
+  if (auto *ConstArrTy =
+          Finder->getASTContext().getAsConstantArrayType(Arg0Ty)) {
+    const APSInt ConstArrSize = APSInt(ConstArrTy->getSize());
 
     // Check form 4:
     return Arg1CV && APSInt::compareValues(ConstArrSize, *Arg1CV) == 0;
@@ -2664,7 +2664,7 @@ static FixItList fixVarDeclWithArray(const VarDecl *D, const ASTContext &Ctx,
 
   // Note: the code below expects the declaration to not use any type sugar like
   // typedef.
-  if (auto CAT = dyn_cast<clang::ConstantArrayType>(D->getType())) {
+  if (auto CAT = Ctx.getAsConstantArrayType(D->getType())) {
     const QualType &ArrayEltT = CAT->getElementType();
     assert(!ArrayEltT.isNull() && "Trying to fix a non-array type variable!");
     // FIXME: support multi-dimensional arrays
@@ -2797,9 +2797,9 @@ fixVariable(const VarDecl *VD, FixitStrategy::Kind K,
     return {};
   }
   case FixitStrategy::Kind::Array: {
-    if (VD->isLocalVarDecl() &&
-        isa<clang::ConstantArrayType>(VD->getType().getCanonicalType()))
-      return fixVariableWithArray(VD, Tracker, Ctx, Handler);
+    if (VD->isLocalVarDecl())
+      if (auto CAT = Ctx.getAsConstantArrayType(VD->getType()))
+        return fixVariableWithArray(VD, Tracker, Ctx, Handler);
 
     DEBUG_NOTE_DECL_FAIL(VD, " : not a local const-size array");
     return {};
diff --git a/clang/test/SemaCXX/warn-unsafe-buffer-usage-in-container-span-construct.cpp b/clang/test/SemaCXX/warn-unsafe-buffer-usage-in-container-span-construct.cpp
index a1ddc384e0d9b8c..e97511593bbd815 100644
--- a/clang/test/SemaCXX/warn-unsafe-buffer-usage-in-container-span-construct.cpp
+++ b/clang/test/SemaCXX/warn-unsafe-buffer-usage-in-container-span-construct.cpp
@@ -79,6 +79,8 @@ namespace construct_wt_ptr_size {
     unsigned Y = 10;
     std::span<int> S = std::span{&X, 1}; // no-warning
     int Arr[10];
+    typedef int TenInts_t[10];
+    TenInts_t Arr2;
 
     S = std::span{&X, 2};                // expected-warning{{the two-parameter std::span construction is unsafe as it can introduce mismatch between buffer size and the bound information}}
     S = std::span{new int[10], 10};      // no-warning
@@ -90,6 +92,7 @@ namespace construct_wt_ptr_size {
     S = std::span{new int[10], 9};       // expected-warning{{the two-parameter std::span construction is unsafe as it can introduce mismatch between buffer size and the bound information}}  // not smart enough to tell its safe
     S = std::span{new int[10], Y};       // expected-warning{{the two-parameter std::span construction is unsafe as it can introduce mismatch between buffer size and the bound information}}  // not smart enough to tell its safe
     S = std::span{Arr, 10};              // no-warning
+    S = std::span{Arr2, 10};             // no-warning
     S = std::span{Arr, Y};               // expected-warning{{the two-parameter std::span construction is unsafe as it can introduce mismatch between buffer size and the bound information}}  // not smart enough to tell its safe
     S = std::span{p, 0};                 // no-warning
   }

From 1d5c2ad4f15f250a0b71495543c5626581a58a7a Mon Sep 17 00:00:00 2001
From: Kirill Stoimenov <87100199+kstoimenov@users.noreply.github.com>
Date: Thu, 15 Aug 2024 12:41:06 -0700
Subject: [PATCH 071/441] Reapply "Fix prctl to handle PR_GET_PDEATHSIG.
 (#101749)" (#104469)

Reapply with a fix. The fix is changing the arg2 type from u64 to int.

This reverts commit 046524e8fe425cbc86c3371f2bf29fbb39d98435.
---
 .../sanitizer_common/sanitizer_common_interceptors.inc    | 7 +++++--
 .../test/sanitizer_common/TestCases/Linux/prctl.cpp       | 8 ++++++++
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc b/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc
index 49c9dcbef358ff2..e09a4a8ae25fd8b 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc
@@ -1255,6 +1255,7 @@ INTERCEPTOR(int, prctl, int option, unsigned long arg2, unsigned long arg3,
   static const int PR_SET_VMA = 0x53564d41;
   static const int PR_SCHED_CORE = 62;
   static const int PR_SCHED_CORE_GET = 0;
+  static const int PR_GET_PDEATHSIG = 2;
   if (option == PR_SET_VMA && arg2 == 0UL) {
     char *name = (char *)arg5;
     COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1);
@@ -1270,7 +1271,9 @@ INTERCEPTOR(int, prctl, int option, unsigned long arg2, unsigned long arg3,
     COMMON_INTERCEPTOR_WRITE_RANGE(ctx, name, internal_strlen(name) + 1);
   } else if (res != -1 && option == PR_SCHED_CORE &&
              arg2 == PR_SCHED_CORE_GET) {
-    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, (u64*)(arg5), sizeof(u64));
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, (u64 *)(arg5), sizeof(u64));
+  } else if (res != -1 && option == PR_GET_PDEATHSIG) {
+    COMMON_INTERCEPTOR_WRITE_RANGE(ctx, (u64 *)(arg2), sizeof(int));
   }
   return res;
 }
@@ -9980,7 +9983,7 @@ INTERCEPTOR(SSIZE_T, getrandom, void *buf, SIZE_T buflen, unsigned int flags) {
   void *ctx;
   COMMON_INTERCEPTOR_ENTER(ctx, getrandom, buf, buflen, flags);
   // If GRND_NONBLOCK is set in the flags, it is non blocking.
-  static const int grnd_nonblock = 1; 
+  static const int grnd_nonblock = 1;
   SSIZE_T n;
   if ((flags & grnd_nonblock))
     n = REAL(getrandom)(buf, buflen, flags);
diff --git a/compiler-rt/test/sanitizer_common/TestCases/Linux/prctl.cpp b/compiler-rt/test/sanitizer_common/TestCases/Linux/prctl.cpp
index d0be7f4fa87899d..cbff02d66efa78b 100644
--- a/compiler-rt/test/sanitizer_common/TestCases/Linux/prctl.cpp
+++ b/compiler-rt/test/sanitizer_common/TestCases/Linux/prctl.cpp
@@ -41,6 +41,14 @@ int main() {
     }
   }
 
+  int signum;
+  res = prctl(PR_GET_PDEATHSIG, reinterpret_cast<unsigned long>(&signum));
+  if (res < 0) {
+    assert(errno == EINVAL);
+  } else {
+    assert(signum == 0);
+  }
+
   char invname[81], vlname[] = "prctl";
   for (auto i = 0; i < sizeof(invname); i++) {
     invname[i] = 0x1e;

From 16f4e85860efcccbadca5d0a00a3872244efae08 Mon Sep 17 00:00:00 2001
From: Steven Wu <stevenwu@apple.com>
Date: Thu, 15 Aug 2024 12:50:03 -0700
Subject: [PATCH 072/441] Revert "[sanitizer] Remove GetCurrentThread nullness
 checks from Allocate"

This reverts commit 4411d1e3926d67c393e6a7bdb910bbe77507ff26 for
breaking Darwin bots:
  AddressSanitizer-Unit :: ./Asan-x86_64-calls-Noinst-Test/10/16
  AddressSanitizer-Unit :: ./Asan-x86_64-calls-Noinst-Test/12/16
  AddressSanitizer-Unit :: ./Asan-x86_64-calls-Noinst-Test/13/16
  AddressSanitizer-Unit :: ./Asan-x86_64-inline-Noinst-Test/10/16
  AddressSanitizer-Unit :: ./Asan-x86_64-inline-Noinst-Test/12/16
  AddressSanitizer-Unit :: ./Asan-x86_64-inline-Noinst-Test/13/16
  AddressSanitizer-Unit :: ./Asan-x86_64h-calls-Noinst-Test/10/16
  AddressSanitizer-Unit :: ./Asan-x86_64h-calls-Noinst-Test/12/16
  AddressSanitizer-Unit :: ./Asan-x86_64h-calls-Noinst-Test/13/16
  AddressSanitizer-Unit :: ./Asan-x86_64h-inline-Noinst-Test/10/16
  AddressSanitizer-Unit :: ./Asan-x86_64h-inline-Noinst-Test/12/16
  AddressSanitizer-Unit :: ./Asan-x86_64h-inline-Noinst-Test/13/16
---
 compiler-rt/lib/asan/asan_allocator.cpp | 11 +++++++++--
 compiler-rt/lib/msan/msan_allocator.cpp | 11 +++++++++--
 2 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/compiler-rt/lib/asan/asan_allocator.cpp b/compiler-rt/lib/asan/asan_allocator.cpp
index e041861edaf0b7b..9e66f77217ec6b9 100644
--- a/compiler-rt/lib/asan/asan_allocator.cpp
+++ b/compiler-rt/lib/asan/asan_allocator.cpp
@@ -576,8 +576,15 @@ struct Allocator {
     }
 
     AsanThread *t = GetCurrentThread();
-    void *allocated = allocator.Allocate(
-        GetAllocatorCache(&t->malloc_storage()), needed_size, 8);
+    void *allocated;
+    if (t) {
+      AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage());
+      allocated = allocator.Allocate(cache, needed_size, 8);
+    } else {
+      SpinMutexLock l(&fallback_mutex);
+      AllocatorCache *cache = &fallback_allocator_cache;
+      allocated = allocator.Allocate(cache, needed_size, 8);
+    }
     if (UNLIKELY(!allocated)) {
       SetAllocatorOutOfMemory();
       if (AllocatorMayReturnNull())
diff --git a/compiler-rt/lib/msan/msan_allocator.cpp b/compiler-rt/lib/msan/msan_allocator.cpp
index f478b9979f2daad..d7d4967c949859f 100644
--- a/compiler-rt/lib/msan/msan_allocator.cpp
+++ b/compiler-rt/lib/msan/msan_allocator.cpp
@@ -199,8 +199,15 @@ static void *MsanAllocate(BufferedStackTrace *stack, uptr size, uptr alignment,
     ReportRssLimitExceeded(stack);
   }
   MsanThread *t = GetCurrentThread();
-  void *allocated = allocator.Allocate(GetAllocatorCache(&t->malloc_storage()),
-                                       size, alignment);
+  void *allocated;
+  if (t) {
+    AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage());
+    allocated = allocator.Allocate(cache, size, alignment);
+  } else {
+    SpinMutexLock l(&fallback_mutex);
+    AllocatorCache *cache = &fallback_allocator_cache;
+    allocated = allocator.Allocate(cache, size, alignment);
+  }
   if (UNLIKELY(!allocated)) {
     SetAllocatorOutOfMemory();
     if (AllocatorMayReturnNull())

From 55aa4ea1c7ff99aea69e07c9db6471f9ac31167a Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Thu, 15 Aug 2024 14:59:28 -0500
Subject: [PATCH 073/441] [libc] Add definition for `atan2l` on 64-bit long
 double platforms (#104489)

Summary:
This just adds `atan2l` for platforms that can implement it as an alias
to `atan2`.
---
 libc/config/gpu/entrypoints.txt      |  1 +
 libc/newhdrgen/yaml/math.yaml        | 14 ++++++++++++++
 libc/spec/stdc.td                    |  1 +
 libc/src/math/CMakeLists.txt         |  1 +
 libc/src/math/atan2l.h               | 20 ++++++++++++++++++++
 libc/src/math/generic/CMakeLists.txt | 11 +++++++++++
 libc/src/math/generic/atan2l.cpp     | 26 ++++++++++++++++++++++++++
 7 files changed, 74 insertions(+)
 create mode 100644 libc/src/math/atan2l.h
 create mode 100644 libc/src/math/generic/atan2l.cpp

diff --git a/libc/config/gpu/entrypoints.txt b/libc/config/gpu/entrypoints.txt
index b2644d2ebf386cf..9ead5aa92d5a92b 100644
--- a/libc/config/gpu/entrypoints.txt
+++ b/libc/config/gpu/entrypoints.txt
@@ -248,6 +248,7 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.atan
     libc.src.math.atan2
     libc.src.math.atan2f
+    libc.src.math.atan2l
     libc.src.math.atanf
     libc.src.math.atanh
     libc.src.math.atanhf
diff --git a/libc/newhdrgen/yaml/math.yaml b/libc/newhdrgen/yaml/math.yaml
index f8b105514271c70..04b6a073deace04 100644
--- a/libc/newhdrgen/yaml/math.yaml
+++ b/libc/newhdrgen/yaml/math.yaml
@@ -37,6 +37,13 @@ functions:
     return_type: float
     arguments:
       - type: float
+  - name: atan2
+    standards:
+      - stdc
+    return_type: double
+    arguments:
+      - type: double
+      - type: double
   - name: atan2f
     standards:
       - stdc
@@ -44,6 +51,13 @@ functions:
     arguments:
       - type: float
       - type: float
+  - name: atan2l
+    standards:
+      - stdc
+    return_type: long double
+    arguments:
+      - type: long double
+      - type: long double
   - name: atanf
     standards:
       - stdc
diff --git a/libc/spec/stdc.td b/libc/spec/stdc.td
index 8497acbfd62f0d3..118dcce829be232 100644
--- a/libc/spec/stdc.td
+++ b/libc/spec/stdc.td
@@ -715,6 +715,7 @@ def StdC : StandardSpec<"stdc"> {
 
           FunctionSpec<"atan2", RetValSpec<DoubleType>, [ArgSpec<DoubleType>, ArgSpec<DoubleType>]>,
           FunctionSpec<"atan2f", RetValSpec<FloatType>, [ArgSpec<FloatType>, ArgSpec<FloatType>]>,
+          FunctionSpec<"atan2l", RetValSpec<LongDoubleType>, [ArgSpec<LongDoubleType>, ArgSpec<LongDoubleType>]>,
 
           FunctionSpec<"acoshf", RetValSpec<FloatType>, [ArgSpec<FloatType>]>,
           FunctionSpec<"asinhf", RetValSpec<FloatType>, [ArgSpec<FloatType>]>,
diff --git a/libc/src/math/CMakeLists.txt b/libc/src/math/CMakeLists.txt
index 132511a536366cc..3cba34fc2493223 100644
--- a/libc/src/math/CMakeLists.txt
+++ b/libc/src/math/CMakeLists.txt
@@ -55,6 +55,7 @@ add_math_entrypoint_object(atanf)
 
 add_math_entrypoint_object(atan2)
 add_math_entrypoint_object(atan2f)
+add_math_entrypoint_object(atan2l)
 
 add_math_entrypoint_object(atanh)
 add_math_entrypoint_object(atanhf)
diff --git a/libc/src/math/atan2l.h b/libc/src/math/atan2l.h
new file mode 100644
index 000000000000000..134d570d72e1f52
--- /dev/null
+++ b/libc/src/math/atan2l.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for atan2l ------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_ATAN2L_H
+#define LLVM_LIBC_SRC_MATH_ATAN2L_H
+
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+long double atan2l(long double x, long double y);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_ATAN2L_H
diff --git a/libc/src/math/generic/CMakeLists.txt b/libc/src/math/generic/CMakeLists.txt
index 745bd65e1d75b51..350072f4b9649d0 100644
--- a/libc/src/math/generic/CMakeLists.txt
+++ b/libc/src/math/generic/CMakeLists.txt
@@ -4230,6 +4230,17 @@ add_entrypoint_object(
     libc.src.__support.macros.optimization
 )
 
+add_entrypoint_object(
+  atan2l
+  SRCS
+    atan2l.cpp
+  HDRS
+    ../atan2l.h
+  COMPILE_OPTIONS
+    -O3
+  DEPENDS
+    .atan2
+)
 
 add_entrypoint_object(
   scalbln
diff --git a/libc/src/math/generic/atan2l.cpp b/libc/src/math/generic/atan2l.cpp
new file mode 100644
index 000000000000000..47a2e985f380c55
--- /dev/null
+++ b/libc/src/math/generic/atan2l.cpp
@@ -0,0 +1,26 @@
+//===-- Extended-precision atan2 function ---------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/atan2l.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/properties/types.h"
+#include "src/math/atan2.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+// TODO: Implement this for extended precision.
+LLVM_LIBC_FUNCTION(long double, atan2l, (long double y, long double x)) {
+#if defined(LIBC_TYPES_LONG_DOUBLE_IS_FLOAT64)
+  return static_cast<long double>(
+      atan2(static_cast<double>(y), static_cast<double>(x)));
+#else
+#error "Extended precision is not yet supported"
+#endif
+}
+
+} // namespace LIBC_NAMESPACE_DECL

From f377edb96abe9fd6fa458415f07b0f411e59cf24 Mon Sep 17 00:00:00 2001
From: royitaqi <royitaqi@users.noreply.github.com>
Date: Thu, 15 Aug 2024 13:14:26 -0700
Subject: [PATCH 074/441] (lldb) Fix PATH_MAX for Windows (#104493)

The build break was caused by a [previous
patch](https://github.com/llvm/llvm-project/pull/102223) using
`PATH_MAX` which is undefined in Windows.
---
 lldb/source/Utility/RealpathPrefixes.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lldb/source/Utility/RealpathPrefixes.cpp b/lldb/source/Utility/RealpathPrefixes.cpp
index 14c81ee6a1f5711..ee1c404c8996764 100644
--- a/lldb/source/Utility/RealpathPrefixes.cpp
+++ b/lldb/source/Utility/RealpathPrefixes.cpp
@@ -8,6 +8,7 @@
 
 #include "lldb/Utility/RealpathPrefixes.h"
 
+#include "lldb/Host/PosixApi.h"
 #include "lldb/Utility/FileSpec.h"
 #include "lldb/Utility/FileSpecList.h"
 #include "lldb/Utility/LLDBLog.h"

From 062ae0427a99b0399483572a04eb055784e7fb0f Mon Sep 17 00:00:00 2001
From: Amr Hesham <amr96@programmer.net>
Date: Thu, 15 Aug 2024 22:16:38 +0200
Subject: [PATCH 075/441] [Clang] [NFC] Rewrite constexpr vectors test to use
 element access (#102757)

Previously, this was using FileCheck to test against the emitted IR;
switch to using `static_assert`ions instead now that we can access
vector elements at compile-time.

Fixes #102463
---
 clang/test/SemaCXX/constexpr-vectors.cpp | 539 ++++++++++++++---------
 1 file changed, 331 insertions(+), 208 deletions(-)

diff --git a/clang/test/SemaCXX/constexpr-vectors.cpp b/clang/test/SemaCXX/constexpr-vectors.cpp
index 99b045f888d87c2..6cd305e57325773 100644
--- a/clang/test/SemaCXX/constexpr-vectors.cpp
+++ b/clang/test/SemaCXX/constexpr-vectors.cpp
@@ -1,10 +1,6 @@
-// RUN: %clang_cc1 -std=c++14 -Wno-unused-value %s -disable-llvm-passes -triple x86_64-linux-gnu -emit-llvm -o - | FileCheck %s
+// RUN: %clang_cc1 %s -Wno-uninitialized -std=c++14 -fsyntax-only -verify
 
-// FIXME: Unfortunately there is no good way to validate that our values are
-// correct since Vector types don't have operator [] implemented for constexpr.
-// Instead, we need to use filecheck to ensure the emitted IR is correct. Once
-// someone implements array subscript operator for these types as constexpr,
-// this test should modified to jsut use static asserts.
+// expected-no-diagnostics
 
 using FourCharsVecSize __attribute__((vector_size(4))) = char;
 using FourIntsVecSize __attribute__((vector_size(16))) = int;
@@ -150,564 +146,691 @@ constexpr auto CmpBinOr(T t, U u) {
   return t;
 }
 
+constexpr auto CmpF(float t, float u) {
+  return __builtin_fabs(t - u) < 0.0001;
+}
+
 // Only int vs float makes a difference here, so we only need to test 1 of each.
 // Test Char to make sure the mixed-nature of shifts around char is evident.
 void CharUsage() {
   constexpr auto a = FourCharsVecSize{6, 3, 2, 1} +
-                     FourCharsVecSize{12, 15, 5, 7};
-  // CHECK: store <4 x i8> <i8 18, i8 18, i8 7, i8 8>
+            FourCharsVecSize{12, 15, 5, 7};
+  static_assert(a[0] == 18 && a[1] == 18 && a[2] == 7 && a[3] == 8, "");
+  
   constexpr auto b = FourCharsVecSize{19, 15, 13, 12} -
                      FourCharsVecSize{13, 14, 5, 3};
-  // CHECK: store <4 x i8> <i8 6, i8 1, i8 8, i8 9>
+  static_assert(b[0] == 6 && b[1] == 1 && b[2] == 8 && b[3] == 9, "");
+  
   constexpr auto c = FourCharsVecSize{8, 4, 2, 1} *
                      FourCharsVecSize{3, 4, 5, 6};
-  // CHECK: store <4 x i8> <i8 24, i8 16, i8 10, i8 6>
+  static_assert(c[0] == 24 && c[1] == 16 && c[2] == 10 && c[3] == 6, "");
+
   constexpr auto d = FourCharsVecSize{12, 12, 10, 10} /
                      FourCharsVecSize{6, 4, 5, 2};
-  // CHECK: store <4 x i8> <i8 2, i8 3, i8 2, i8 5>
+  static_assert(d[0] == 2 && d[1] == 3 && d[2] == 2 && d[3] == 5, "");
+
   constexpr auto e = FourCharsVecSize{12, 12, 10, 10} %
                      FourCharsVecSize{6, 4, 4, 3};
-  // CHECK: store <4 x i8> <i8 0, i8 0, i8 2, i8 1>
+  static_assert(e[0] == 0 && e[1] == 0 && e[2] == 2 && e[3] == 1, "");
 
   constexpr auto f = FourCharsVecSize{6, 3, 2, 1} + 3;
-  // CHECK: store <4 x i8> <i8 9, i8 6, i8 5, i8 4>
+  static_assert(f[0] == 9 && f[1] == 6 && f[2] == 5 && f[3] == 4, "");
+  
   constexpr auto g = FourCharsVecSize{19, 15, 12, 10} - 3;
-  // CHECK: store <4 x i8> <i8 16, i8 12, i8 9, i8 7>
+  static_assert(g[0] == 16 && g[1] == 12 && g[2] == 9 && g[3] == 7, "");  
+
   constexpr auto h = FourCharsVecSize{8, 4, 2, 1} * 3;
-  // CHECK: store <4 x i8> <i8 24, i8 12, i8 6, i8 3>
+  static_assert(h[0] == 24 && h[1] == 12 && h[2] == 6 && h[3] == 3, "");
+ 
   constexpr auto j = FourCharsVecSize{12, 15, 18, 21} / 3;
-  // CHECK: store <4 x i8> <i8 4, i8 5, i8 6, i8 7>
+  static_assert(j[0] == 4 && j[1] == 5 && j[2] == 6 && j[3] == 7, "");
+
   constexpr auto k = FourCharsVecSize{12, 17, 19, 22} % 3;
-  // CHECK: store <4 x i8> <i8 0, i8 2, i8 1, i8 1>
+  static_assert(k[0] == 0 && k[1] == 2 && k[2] == 1 && k[3] == 1, "");
 
   constexpr auto l = 3 + FourCharsVecSize{6, 3, 2, 1};
-  // CHECK: store <4 x i8> <i8 9, i8 6, i8 5, i8 4>
+  static_assert(l[0] == 9 && l[1] == 6 && l[2] == 5 && l[3] == 4, "");
+
   constexpr auto m = 20 - FourCharsVecSize{19, 15, 12, 10};
-  // CHECK: store <4 x i8> <i8 1, i8 5, i8 8, i8 10>
+  static_assert(m[0] == 1 && m[1] == 5 && m[2] == 8 && m[3] == 10, "");
+ 
   constexpr auto n = 3 * FourCharsVecSize{8, 4, 2, 1};
-  // CHECK: store <4 x i8> <i8 24, i8 12, i8 6, i8 3>
+  static_assert(n[0] == 24 && n[1] == 12 && n[2] == 6 && n[3] == 3, "");
+
   constexpr auto o = 100 / FourCharsVecSize{12, 15, 18, 21};
-  // CHECK: store <4 x i8> <i8 8, i8 6, i8 5, i8 4>
+  static_assert(o[0] == 8 && o[1] == 6 && o[2] == 5 && o[3] == 4, "");
+
   constexpr auto p = 100 % FourCharsVecSize{12, 15, 18, 21};
-  // CHECK: store <4 x i8> <i8 4, i8 10, i8 10, i8 16>
+  static_assert(p[0] == 4 && p[1] == 10 && p[2] == 10 && p[3] == 16, "");
 
   constexpr auto q = FourCharsVecSize{6, 3, 2, 1} << FourCharsVecSize{1, 1, 2, 2};
-  // CHECK: store <4 x i8> <i8 12, i8 6, i8 8, i8 4>
+  static_assert(q[0] == 12 && q[1] == 6 && q[2] == 8 && q[3] == 4, "");
+
   constexpr auto r = FourCharsVecSize{19, 15, 12, 10} >>
                      FourCharsVecSize{1, 1, 2, 2};
-  // CHECK: store <4 x i8> <i8 9, i8 7, i8 3, i8 2>
+  static_assert(r[0] == 9 && r[1] == 7 && r[2] == 3 && r[3] == 2, "");
+
   constexpr auto s = FourCharsVecSize{6, 3, 5, 10} << 1;
-  // CHECK: store <4 x i8> <i8 12, i8 6, i8 10, i8 20>
+  static_assert(s[0] == 12 && s[1] == 6 && s[2] == 10 && s[3] == 20, "");
+
   constexpr auto t = FourCharsVecSize{19, 15, 10, 20} >> 1;
-  // CHECK: store <4 x i8> <i8 9, i8 7, i8 5, i8 10>
+  static_assert(t[0] == 9 && t[1] == 7 && t[2] == 5 && t[3] == 10, "");
+
   constexpr auto u = 12 << FourCharsVecSize{1, 2, 3, 3};
-  // CHECK: store <4 x i8> <i8 24, i8 48, i8 96, i8 96>
+  static_assert(u[0] == 24 && u[1] == 48 && u[2] == 96 && u[3] == 96, "");
+
   constexpr auto v = 12 >> FourCharsVecSize{1, 2, 2, 1};
-  // CHECK: store <4 x i8> <i8 6, i8 3, i8 3, i8 6>
+  static_assert(v[0] == 6 && v[1] == 3 && v[2] == 3 && v[3] == 6, "");
 
   constexpr auto w = FourCharsVecSize{1, 2, 3, 4} <
                      FourCharsVecSize{4, 3, 2, 1};
-  // CHECK: store <4 x i8> <i8 -1, i8 -1, i8 0, i8 0>
+  static_assert(w[0] == -1 && w[1] == -1 && w[2] == 0 && w[3] == 0, "");
+
   constexpr auto x = FourCharsVecSize{1, 2, 3, 4} >
                      FourCharsVecSize{4, 3, 2, 1};
-  // CHECK: store <4 x i8> <i8 0, i8 0, i8 -1, i8 -1>
+  static_assert(x[0] == 0 && x[1] == 0 && x[2] == -1 && x[3] == -1, "");
+
   constexpr auto y = FourCharsVecSize{1, 2, 3, 4} <=
                      FourCharsVecSize{4, 3, 3, 1};
-  // CHECK: store <4 x i8> <i8 -1, i8 -1, i8 -1, i8 0>
+  static_assert(y[0] == -1 && y[1] == -1 && y[2] == -1 && y[3] == 0, "");
+
   constexpr auto z = FourCharsVecSize{1, 2, 3, 4} >=
                      FourCharsVecSize{4, 3, 3, 1};
-  // CHECK: store <4 x i8> <i8 0, i8 0, i8 -1, i8 -1>
+  static_assert(z[0] == 0 && z[1] == 0 && z[2] == -1 && z[3] == -1, "");
+
   constexpr auto A = FourCharsVecSize{1, 2, 3, 4} ==
                      FourCharsVecSize{4, 3, 3, 1};
-  // CHECK: store <4 x i8> <i8 0, i8 0, i8 -1, i8 0>
+  static_assert(A[0] == 0 && A[1] == 0 && A[2] == -1 && A[3] == 0, "");
+
   constexpr auto B = FourCharsVecSize{1, 2, 3, 4} !=
                      FourCharsVecSize{4, 3, 3, 1};
-  // CHECK: store <4 x i8> <i8 -1, i8 -1, i8 0, i8 -1>
+  static_assert(B[0] == -1 && B[1] == -1 && B[2] == 0 && B[3] == -1, "");
 
   constexpr auto C = FourCharsVecSize{1, 2, 3, 4} < 3;
-  // CHECK: store <4 x i8> <i8 -1, i8 -1, i8 0, i8 0>
+  static_assert(C[0] == -1 && C[1] == -1 && C[2] == 0 && C[3] == 0, "");
+
   constexpr auto D = FourCharsVecSize{1, 2, 3, 4} > 3;
-  // CHECK: store <4 x i8> <i8 0, i8 0, i8 0, i8 -1>
+  static_assert(D[0] == 0 && D[1] == 0 && D[2] == 0 && D[3] == -1, "");
+
   constexpr auto E = FourCharsVecSize{1, 2, 3, 4} <= 3;
-  // CHECK: store <4 x i8> <i8 -1, i8 -1, i8 -1, i8 0>
+  static_assert(E[0] == -1 && E[1] == -1 && E[2] == -1 && E[3] == 0, "");
+    
   constexpr auto F = FourCharsVecSize{1, 2, 3, 4} >= 3;
-  // CHECK: store <4 x i8> <i8 0, i8 0, i8 -1, i8 -1>
+  static_assert(F[0] == 0 && F[1] == 0 && F[2] == -1 && F[3] == -1, "");
+
   constexpr auto G = FourCharsVecSize{1, 2, 3, 4} == 3;
-  // CHECK: store <4 x i8> <i8 0, i8 0, i8 -1, i8 0>
+  static_assert(G[0] == 0 && G[1] == 0 && G[2] == -1 && G[3] == 0, "");
+
   constexpr auto H = FourCharsVecSize{1, 2, 3, 4} != 3;
-  // CHECK: store <4 x i8> <i8 -1, i8 -1, i8 0, i8 -1>
+  static_assert(H[0] == -1 && H[1] == -1 && H[2] == 0 && H[3] == -1, "");
 
   constexpr auto I = FourCharsVecSize{1, 2, 3, 4} &
                      FourCharsVecSize{4, 3, 2, 1};
-  // CHECK: store <4 x i8> <i8 0, i8 2, i8 2, i8 0>
+  static_assert(I[0] == 0 && I[1] == 2 && I[2] == 2 && I[3] == 0, "");
+ 
   constexpr auto J = FourCharsVecSize{1, 2, 3, 4} ^
                      FourCharsVecSize { 4, 3, 2, 1 };
-  // CHECK: store <4 x i8> <i8 5, i8 1, i8 1, i8 5>
+  static_assert(J[0] == 5 && J[1] == 1 && J[2] == 1 && J[3] == 5, "");
+
   constexpr auto K = FourCharsVecSize{1, 2, 3, 4} |
                      FourCharsVecSize{4, 3, 2, 1};
-  // CHECK: store <4 x i8> <i8 5, i8 3, i8 3, i8 5>
+  static_assert(K[0] == 5 && K[1] == 3 && K[2] == 3 && K[3] == 5, "");
+
   constexpr auto L = FourCharsVecSize{1, 2, 3, 4} & 3;
-  // CHECK: store <4 x i8> <i8 1, i8 2, i8 3, i8 0>
+  static_assert(L[0] == 1 && L[1] == 2 && L[2] == 3 && L[3] == 0, "");
+
   constexpr auto M = FourCharsVecSize{1, 2, 3, 4} ^ 3;
-  // CHECK: store <4 x i8> <i8 2, i8 1, i8 0, i8 7>
+  static_assert(M[0] == 2 && M[1] == 1 && M[2] == 0 && M[3] == 7, "");
+
   constexpr auto N = FourCharsVecSize{1, 2, 3, 4} | 3;
-  // CHECK: store <4 x i8> <i8 3, i8 3, i8 3, i8 7>
+  static_assert(N[0] == 3 && N[1] == 3 && N[2] == 3 && N[3] == 7, "");
 
   constexpr auto O = FourCharsVecSize{5, 0, 6, 0} &&
                      FourCharsVecSize{5, 5, 0, 0};
-  // CHECK: store <4 x i8> <i8 1, i8 0, i8 0, i8 0>
+  static_assert(O[0] == 1 && O[1] == 0 && O[2] == 0 && O[3] == 0, "");
+  
   constexpr auto P = FourCharsVecSize{5, 0, 6, 0} ||
                      FourCharsVecSize{5, 5, 0, 0};
-  // CHECK: store <4 x i8> <i8 1, i8 1, i8 1, i8 0>
+  static_assert(P[0] == 1 && P[1] == 1 && P[2] == 1 && P[3] == 0, "");
 
   constexpr auto Q = FourCharsVecSize{5, 0, 6, 0} && 3;
-  // CHECK: store <4 x i8> <i8 1, i8 0, i8 1, i8 0>
+  static_assert(Q[0] == 1 && Q[1] == 0 && Q[2] == 1 && Q[3] == 0, "");
+
   constexpr auto R = FourCharsVecSize{5, 0, 6, 0} || 3;
-  // CHECK: store <4 x i8> <i8 1, i8 1, i8 1, i8 1>
+  static_assert(R[0] == 1 && R[1] == 1 && R[2] == 1 && R[3] == 1, "");
 
   constexpr auto T = CmpMul(a, b);
-  // CHECK: store <4 x i8> <i8 108, i8 18, i8 56, i8 72>
+  static_assert(T[0] == 108 && T[1] == 18 && T[2] == 56 && T[3] == 72, "");
 
   constexpr auto U = CmpDiv(a, b);
-  // CHECK: store <4 x i8> <i8 3, i8 18, i8 0, i8 0>
+  static_assert(U[0] == 3 && U[1] == 18 && U[2] == 0 && U[3] == 0, "");
 
   constexpr auto V = CmpRem(a, b);
-  // CHECK: store <4 x i8> <i8 0, i8 0, i8 7, i8 8>
+  static_assert(V[0] == 0 && V[1] == 0 && V[2] == 7 && V[3] == 8, "");
 
   constexpr auto X = CmpAdd(a, b);
-  // CHECK: store <4 x i8> <i8 24, i8 19, i8 15, i8 17>
+  static_assert(X[0] == 24 && X[1] == 19 && X[2] == 15 && X[3] == 17, "");
 
   constexpr auto Y = CmpSub(a, b);
-  // CHECK: store <4 x i8> <i8 12, i8 17, i8 -1, i8 -1>
+  static_assert(Y[0] == 12 && Y[1] == 17 && Y[2] == -1 && Y[3] == -1, "");
 
   constexpr auto InvH = -H;
-  // CHECK: store <4 x i8> <i8 1, i8 1, i8 0, i8 1>
+  static_assert(InvH[0] == 1 && InvH[1] == 1 && InvH[2] == 0 && InvH[3] == 1, "");
+
   constexpr auto Z = CmpLSH(a, InvH);
-  // CHECK: store <4 x i8> <i8 36, i8 36, i8 7, i8 16>
+  static_assert(Z[0] == 36 && Z[1] == 36 && Z[2] == 7 && Z[3] == 16, "");
 
   constexpr auto aa = CmpRSH(a, InvH);
-  // CHECK: store <4 x i8> <i8 9, i8 9, i8 7, i8 4>
+  static_assert(aa[0] == 9 && aa[1] == 9 && aa[2] == 7 && aa[3] == 4, "");
 
   constexpr auto ab = CmpBinAnd(a, b);
-  // CHECK: store <4 x i8> <i8 2, i8 0, i8 0, i8 8>
+  static_assert(ab[0] == 2 && ab[1] == 0 && ab[2] == 0 && ab[3] == 8, "");
 
   constexpr auto ac = CmpBinXOr(a, b);
-  // CHECK: store <4 x i8> <i8 20, i8 19, i8 15, i8 1>
+  static_assert(ac[0] == 20 && ac[1] == 19 && ac[2] == 15 && ac[3] == 1, "");
 
   constexpr auto ad = CmpBinOr(a, b);
-  // CHECK: store <4 x i8> <i8 22, i8 19, i8 15, i8 9>
+  static_assert(ad[0] == 22 && ad[1] == 19 && ad[2] == 15 && ad[3] == 9, ""); 
 
   constexpr auto ae = ~FourCharsVecSize{1, 2, 10, 20};
-  // CHECK: store <4 x i8> <i8 -2, i8 -3, i8 -11, i8 -21>
+  static_assert(ae[0] == -2 && ae[1] == -3 && ae[2] == -11 && ae[3] == -21, "");
 
   constexpr auto af = !FourCharsVecSize{0, 1, 8, -1};
-  // CHECK: store <4 x i8> <i8 -1, i8 0, i8 0, i8 0>
+  static_assert(af[0] == -1 && af[1] == 0 && af[2] == 0 && af[3] == 0, "");
 }
 
 void CharExtVecUsage() {
   constexpr auto a = FourCharsExtVec{6, 3, 2, 1} +
                      FourCharsExtVec{12, 15, 5, 7};
-  // CHECK: store <4 x i8> <i8 18, i8 18, i8 7, i8 8>
+  static_assert(a[0] == 18 && a[1] == 18 && a[2] == 7 && a[3] == 8, "");
+
   constexpr auto b = FourCharsExtVec{19, 15, 13, 12} -
                      FourCharsExtVec{13, 14, 5, 3};
-  // CHECK: store <4 x i8> <i8 6, i8 1, i8 8, i8 9>
+  static_assert(b[0] == 6 && b[1] == 1 && b[2] == 8 && b[3] == 9, "");
+
   constexpr auto c = FourCharsExtVec{8, 4, 2, 1} *
                      FourCharsExtVec{3, 4, 5, 6};
-  // CHECK: store <4 x i8> <i8 24, i8 16, i8 10, i8 6>
+  static_assert(c[0] == 24 && c[1] == 16 && c[2] == 10 && c[3] == 6, "");
+
   constexpr auto d = FourCharsExtVec{12, 12, 10, 10} /
                      FourCharsExtVec{6, 4, 5, 2};
-  // CHECK: store <4 x i8> <i8 2, i8 3, i8 2, i8 5>
+  static_assert(d[0] == 2 && d[1] == 3 && d[2] == 2 && d[3] == 5, "");
+
   constexpr auto e = FourCharsExtVec{12, 12, 10, 10} %
                      FourCharsExtVec{6, 4, 4, 3};
-  // CHECK: store <4 x i8> <i8 0, i8 0, i8 2, i8 1>
+  static_assert(e[0] == 0 && e[1] == 0 && e[2] == 2 && e[3] == 1, "");
 
   constexpr auto f = FourCharsExtVec{6, 3, 2, 1} + 3;
-  // CHECK: store <4 x i8> <i8 9, i8 6, i8 5, i8 4>
+  static_assert(f[0] == 9 && f[1] == 6 && f[2] == 5 && f[3] == 4, "");
+
   constexpr auto g = FourCharsExtVec{19, 15, 12, 10} - 3;
-  // CHECK: store <4 x i8> <i8 16, i8 12, i8 9, i8 7>
+  static_assert(g[0] == 16 && g[1] == 12 && g[2] == 9 && g[3] == 7, "");
+
   constexpr auto h = FourCharsExtVec{8, 4, 2, 1} * 3;
-  // CHECK: store <4 x i8> <i8 24, i8 12, i8 6, i8 3>
+  static_assert(h[0] == 24 && h[1] == 12 && h[2] == 6 && h[3] == 3, "");
+
   constexpr auto j = FourCharsExtVec{12, 15, 18, 21} / 3;
-  // CHECK: store <4 x i8> <i8 4, i8 5, i8 6, i8 7>
+  static_assert(j[0] == 4 && j[1] == 5 && j[2] == 6 && j[3] == 7, "");
+
   constexpr auto k = FourCharsExtVec{12, 17, 19, 22} % 3;
-  // CHECK: store <4 x i8> <i8 0, i8 2, i8 1, i8 1>
+  static_assert(k[0] == 0 && k[1] == 2 && k[2] == 1 && k[3] == 1, "");
 
   constexpr auto l = 3 + FourCharsExtVec{6, 3, 2, 1};
-  // CHECK: store <4 x i8> <i8 9, i8 6, i8 5, i8 4>
+  static_assert(l[0] == 9 && l[1] == 6 && l[2] == 5 && l[3] == 4, "");
+
   constexpr auto m = 20 - FourCharsExtVec{19, 15, 12, 10};
-  // CHECK: store <4 x i8> <i8 1, i8 5, i8 8, i8 10>
+  static_assert(m[0] == 1 && m[1] == 5 && m[2] == 8 && m[3] == 10, "");
+
   constexpr auto n = 3 * FourCharsExtVec{8, 4, 2, 1};
-  // CHECK: store <4 x i8> <i8 24, i8 12, i8 6, i8 3>
+  static_assert(n[0] == 24 && n[1] == 12 && n[2] == 6 && n[3] == 3, "");
+
   constexpr auto o = 100 / FourCharsExtVec{12, 15, 18, 21};
-  // CHECK: store <4 x i8> <i8 8, i8 6, i8 5, i8 4>
+  static_assert(o[0] == 8 && o[1] == 6 && o[2] == 5 && o[3] == 4, "");
+
   constexpr auto p = 100 % FourCharsExtVec{12, 15, 18, 21};
-  // CHECK: store <4 x i8> <i8 4, i8 10, i8 10, i8 16>
+  static_assert(p[0] == 4 && p[1] == 10 && p[2] == 10 && p[3] == 16, "");
 
   constexpr auto q = FourCharsExtVec{6, 3, 2, 1} << FourCharsVecSize{1, 1, 2, 2};
-  // CHECK: store <4 x i8> <i8 12, i8 6, i8 8, i8 4>
+  static_assert(q[0] == 12 && q[1] == 6 && q[2] == 8 && q[3] == 4, "");
+
   constexpr auto r = FourCharsExtVec{19, 15, 12, 10} >>
                      FourCharsExtVec{1, 1, 2, 2};
-  // CHECK: store <4 x i8> <i8 9, i8 7, i8 3, i8 2>
+  static_assert(r[0] == 9 && r[1] == 7 && r[2] == 3 && r[3] == 2, ""); 
+
   constexpr auto s = FourCharsExtVec{6, 3, 5, 10} << 1;
-  // CHECK: store <4 x i8> <i8 12, i8 6, i8 10, i8 20>
+  static_assert(s[0] == 12 && s[1] == 6 && s[2] == 10 && s[3] == 20, "");
+
   constexpr auto t = FourCharsExtVec{19, 15, 10, 20} >> 1;
-  // CHECK: store <4 x i8> <i8 9, i8 7, i8 5, i8 10>
+  static_assert(t[0] == 9 && t[1] == 7 && t[2] == 5 && t[3] == 10, "");
+
   constexpr auto u = 12 << FourCharsExtVec{1, 2, 3, 3};
-  // CHECK: store <4 x i8> <i8 24, i8 48, i8 96, i8 96>
+  static_assert(u[0] == 24 && u[1] == 48 && u[2] == 96 && u[3] == 96, "");
+
   constexpr auto v = 12 >> FourCharsExtVec{1, 2, 2, 1};
-  // CHECK: store <4 x i8> <i8 6, i8 3, i8 3, i8 6>
+  static_assert(v[0] == 6 && v[1] == 3 && v[2] == 3 && v[3] == 6, "");
 
   constexpr auto w = FourCharsExtVec{1, 2, 3, 4} <
                      FourCharsExtVec{4, 3, 2, 1};
-  // CHECK: store <4 x i8> <i8 -1, i8 -1, i8 0, i8 0>
+  static_assert(w[0] == -1 && w[1] == -1 && w[2] == 0 && w[3] == 0, "");
+
   constexpr auto x = FourCharsExtVec{1, 2, 3, 4} >
                      FourCharsExtVec{4, 3, 2, 1};
-  // CHECK: store <4 x i8> <i8 0, i8 0, i8 -1, i8 -1>
+  static_assert(x[0] == 0 && x[1] == 0 && x[2] == -1 && x[3] == -1, "");
+
   constexpr auto y = FourCharsExtVec{1, 2, 3, 4} <=
                      FourCharsExtVec{4, 3, 3, 1};
-  // CHECK: store <4 x i8> <i8 -1, i8 -1, i8 -1, i8 0>
+  static_assert(y[0] == -1 && y[1] == -1 && y[2] == -1 && y[3] == 0, "");
+
   constexpr auto z = FourCharsExtVec{1, 2, 3, 4} >=
                      FourCharsExtVec{4, 3, 3, 1};
-  // CHECK: store <4 x i8> <i8 0, i8 0, i8 -1, i8 -1>
+  static_assert(z[0] == 0 && z[1] == 0 && z[2] == -1 && z[3] == -1, "");
+
   constexpr auto A = FourCharsExtVec{1, 2, 3, 4} ==
                      FourCharsExtVec{4, 3, 3, 1};
-  // CHECK: store <4 x i8> <i8 0, i8 0, i8 -1, i8 0>
+  static_assert(A[0] == 0 && A[1] == 0 && A[2] == -1 && A[3] == 0, "");
+
   constexpr auto B = FourCharsExtVec{1, 2, 3, 4} !=
                      FourCharsExtVec{4, 3, 3, 1};
-  // CHECK: store <4 x i8> <i8 -1, i8 -1, i8 0, i8 -1>
+  static_assert(B[0] == -1 && B[1] == -1 && B[2] == 0 && B[3] == -1, "");
 
   constexpr auto C = FourCharsExtVec{1, 2, 3, 4} < 3;
-  // CHECK: store <4 x i8> <i8 -1, i8 -1, i8 0, i8 0>
+  static_assert(C[0] == -1 && C[1] == -1 && C[2] == 0 && C[3] == 0, "");
+
   constexpr auto D = FourCharsExtVec{1, 2, 3, 4} > 3;
-  // CHECK: store <4 x i8> <i8 0, i8 0, i8 0, i8 -1>
+  static_assert(D[0] == 0 && D[1] == 0 && D[2] == 0 && D[3] == -1, "");
+
   constexpr auto E = FourCharsExtVec{1, 2, 3, 4} <= 3;
-  // CHECK: store <4 x i8> <i8 -1, i8 -1, i8 -1, i8 0>
+  static_assert(E[0] == -1 && E[1] == -1 && E[2] == -1 && E[3] == 0, "");
+  
   constexpr auto F = FourCharsExtVec{1, 2, 3, 4} >= 3;
-  // CHECK: store <4 x i8> <i8 0, i8 0, i8 -1, i8 -1>
+  static_assert(F[0] == 0 && F[1] == 0 && F[2] == -1 && F[3] == -1, "");
+
   constexpr auto G = FourCharsExtVec{1, 2, 3, 4} == 3;
-  // CHECK: store <4 x i8> <i8 0, i8 0, i8 -1, i8 0>
+  static_assert(G[0] == 0 && G[1] == 0 && G[2] == -1 && G[3] == 0, "");
+
   constexpr auto H = FourCharsExtVec{1, 2, 3, 4} != 3;
-  // CHECK: store <4 x i8> <i8 -1, i8 -1, i8 0, i8 -1>
+  static_assert(H[0] == -1 && H[1] == -1 && H[2] == 0 && H[3] == -1, "");
 
   constexpr auto I = FourCharsExtVec{1, 2, 3, 4} &
                      FourCharsExtVec{4, 3, 2, 1};
-  // CHECK: store <4 x i8> <i8 0, i8 2, i8 2, i8 0>
+  static_assert(I[0] == 0 && I[1] == 2 && I[2] == 2 && I[3] == 0, "");
+
   constexpr auto J = FourCharsExtVec{1, 2, 3, 4} ^
                      FourCharsExtVec { 4, 3, 2, 1 };
-  // CHECK: store <4 x i8> <i8 5, i8 1, i8 1, i8 5>
+  static_assert(J[0] == 5 && J[1] == 1 && J[2] == 1 && J[3] == 5, "");
+
   constexpr auto K = FourCharsExtVec{1, 2, 3, 4} |
                      FourCharsExtVec{4, 3, 2, 1};
-  // CHECK: store <4 x i8> <i8 5, i8 3, i8 3, i8 5>
+  static_assert(K[0] == 5 && K[1] == 3 && K[2] == 3 && K[3] == 5, "");
+  
   constexpr auto L = FourCharsExtVec{1, 2, 3, 4} & 3;
-  // CHECK: store <4 x i8> <i8 1, i8 2, i8 3, i8 0>
+  static_assert(L[0] == 1 && L[1] == 2 && L[2] == 3 && L[3] == 0, "");
+
   constexpr auto M = FourCharsExtVec{1, 2, 3, 4} ^ 3;
-  // CHECK: store <4 x i8> <i8 2, i8 1, i8 0, i8 7>
+  static_assert(M[0] == 2 && M[1] == 1 && M[2] == 0 && M[3] == 7, "");
+
   constexpr auto N = FourCharsExtVec{1, 2, 3, 4} | 3;
-  // CHECK: store <4 x i8> <i8 3, i8 3, i8 3, i8 7>
+  static_assert(N[0] == 3 && N[1] == 3 && N[2] == 3 && N[3] == 7, "");
 
   constexpr auto O = FourCharsExtVec{5, 0, 6, 0} &&
                      FourCharsExtVec{5, 5, 0, 0};
-  // CHECK: store <4 x i8> <i8 1, i8 0, i8 0, i8 0>
+  static_assert(O[0] == 1 && O[1] == 0 && O[2] == 0 && O[3] == 0, "");
+
   constexpr auto P = FourCharsExtVec{5, 0, 6, 0} ||
                      FourCharsExtVec{5, 5, 0, 0};
-  // CHECK: store <4 x i8> <i8 1, i8 1, i8 1, i8 0>
+  static_assert(P[0] == 1 && P[1] == 1 && P[2] == 1 && P[3] == 0, "");
 
   constexpr auto Q = FourCharsExtVec{5, 0, 6, 0} && 3;
-  // CHECK: store <4 x i8> <i8 1, i8 0, i8 1, i8 0>
+  static_assert(Q[0] == 1 && Q[1] == 0 && Q[2] == 1 && Q[3] == 0, "");
+
   constexpr auto R = FourCharsExtVec{5, 0, 6, 0} || 3;
-  // CHECK: store <4 x i8> <i8 1, i8 1, i8 1, i8 1>
+  static_assert(R[0] == 1 && R[1] == 1 && R[2] == 1 && R[3] == 1, "");
 
   constexpr auto T = CmpMul(a, b);
-  // CHECK: store <4 x i8> <i8 108, i8 18, i8 56, i8 72>
+  static_assert(T[0] == 108 && T[1] == 18 && T[2] == 56 && T[3] == 72, "");
 
   constexpr auto U = CmpDiv(a, b);
-  // CHECK: store <4 x i8> <i8 3, i8 18, i8 0, i8 0>
+  static_assert(U[0] == 3 && U[1] == 18 && U[2] == 0 && U[3] == 0, "");
 
   constexpr auto V = CmpRem(a, b);
-  // CHECK: store <4 x i8> <i8 0, i8 0, i8 7, i8 8>
+  static_assert(V[0] == 0 && V[1] == 0 && V[2] == 7 && V[3] == 8, "");
 
   constexpr auto X = CmpAdd(a, b);
-  // CHECK: store <4 x i8> <i8 24, i8 19, i8 15, i8 17>
+  static_assert(X[0] == 24 && X[1] == 19 && X[2] == 15 && X[3] == 17, "");
 
   constexpr auto Y = CmpSub(a, b);
-  // CHECK: store <4 x i8> <i8 12, i8 17, i8 -1, i8 -1>
+  static_assert(Y[0] == 12 && Y[1] == 17 && Y[2] == -1 && Y[3] == -1, "");
 
   constexpr auto InvH = -H;
-  // CHECK: store <4 x i8> <i8 1, i8 1, i8 0, i8 1>
+  static_assert(InvH[0] == 1 && InvH[1] == 1 && InvH[2] == 0 && InvH[3] == 1, "");
 
   constexpr auto Z = CmpLSH(a, InvH);
-  // CHECK: store <4 x i8> <i8 36, i8 36, i8 7, i8 16>
+  static_assert(Z[0] == 36 && Z[1] == 36 && Z[2] == 7 && Z[3] == 16, "");
 
   constexpr auto aa = CmpRSH(a, InvH);
-  // CHECK: store <4 x i8> <i8 9, i8 9, i8 7, i8 4>
+  static_assert(aa[0] == 9 && aa[1] == 9 && aa[2] == 7 && aa[3] == 4, "");
 
   constexpr auto ab = CmpBinAnd(a, b);
-  // CHECK: store <4 x i8> <i8 2, i8 0, i8 0, i8 8>
+  static_assert(ab[0] == 2 && ab[1] == 0 && ab[2] == 0 && ab[3] == 8, "");
 
   constexpr auto ac = CmpBinXOr(a, b);
-  // CHECK: store <4 x i8> <i8 20, i8 19, i8 15, i8 1>
+  static_assert(ac[0] == 20 && ac[1] == 19 && ac[2] == 15 && ac[3] == 1, "");
 
   constexpr auto ad = CmpBinOr(a, b);
-  // CHECK: store <4 x i8> <i8 22, i8 19, i8 15, i8 9>
+  static_assert(ad[0] == 22 && ad[1] == 19 && ad[2] == 15 && ad[3] == 9, "");
 
   constexpr auto ae = ~FourCharsExtVec{1, 2, 10, 20};
-  // CHECK: store <4 x i8> <i8 -2, i8 -3, i8 -11, i8 -21>
+  static_assert(ae[0] == -2 && ae[1] == -3 && ae[2] == -11 && ae[3] == -21, "");
 
   constexpr auto af = !FourCharsExtVec{0, 1, 8, -1};
-  // CHECK: store <4 x i8> <i8 -1, i8 0, i8 0, i8 0>
+  static_assert(af[0] == -1 && af[1] == 0 && af[2] == 0 && af[3] == 0, "");
 }
 
 void FloatUsage() {
   constexpr auto a = FourFloatsVecSize{6, 3, 2, 1} +
                      FourFloatsVecSize{12, 15, 5, 7};
-  // CHECK: <4 x float> <float 1.800000e+01, float 1.800000e+01, float 7.000000e+00, float 8.000000e+00>
+  static_assert(a[0] == 1.800000e+01 && a[1] == 1.800000e+01 && a[2] == 7.000000e+00 && a[3] == 8.000000e+00, "");
+
   constexpr auto b = FourFloatsVecSize{19, 15, 13, 12} -
                      FourFloatsVecSize{13, 14, 5, 3};
-  // CHECK: store <4 x float> <float 6.000000e+00, float 1.000000e+00, float 8.000000e+00, float 9.000000e+00>
+  static_assert(b[0] == 6.000000e+00 && b[1] == 1.000000e+00 && b[2] == 8.000000e+00 && b[3] == 9.000000e+00, "");
+
   constexpr auto c = FourFloatsVecSize{8, 4, 2, 1} *
                      FourFloatsVecSize{3, 4, 5, 6};
-  // CHECK: store <4 x float> <float 2.400000e+01, float 1.600000e+01, float 1.000000e+01, float 6.000000e+00>
+  static_assert(c[0] == 2.400000e+01 && c[1] == 1.600000e+01 && c[2] == 1.000000e+01 && c[3] == 6.000000e+00, "");
+
   constexpr auto d = FourFloatsVecSize{12, 12, 10, 10} /
                      FourFloatsVecSize{6, 4, 5, 2};
-  // CHECK: store <4 x float> <float 2.000000e+00, float 3.000000e+00, float 2.000000e+00, float 5.000000e+00>
+  static_assert(d[0] == 2.000000e+00 && d[1] == 3.000000e+00 && d[2] == 2.000000e+00 && d[3] == 5.000000e+00, "");
 
   constexpr auto f = FourFloatsVecSize{6, 3, 2, 1} + 3;
-  // CHECK: store <4 x float> <float 9.000000e+00, float 6.000000e+00, float 5.000000e+00, float 4.000000e+00>
+  static_assert(f[0] == 9.000000e+00 && f[1] == 6.000000e+00 && f[2] == 5.000000e+00 && f[3] == 4.000000e+00, "");
+
   constexpr auto g = FourFloatsVecSize{19, 15, 12, 10} - 3;
-  // CHECK: store <4 x float> <float 1.600000e+01, float 1.200000e+01, float 9.000000e+00, float 7.000000e+00>
-  constexpr auto h = FourFloatsVecSize{8, 4, 2, 1} * 3;
-  // CHECK: store <4 x float> <float 2.400000e+01, float 1.200000e+01, float 6.000000e+00, float 3.000000e+00>
+  static_assert(g[0] == 1.600000e+01 && g[1] == 1.200000e+01 && g[2] == 9.000000e+00 && g[3] == 7.000000e+00, "");
+
+  constexpr auto h = FourFloatsVecSize{8, 4, 2, 1} * 3;  
+  static_assert(h[0] == 2.400000e+01 && h[1] == 1.200000e+01 && h[2] == 6.000000e+00 && h[3] == 3.000000e+00, "");
+
   constexpr auto j = FourFloatsVecSize{12, 15, 18, 21} / 3;
-  // CHECK: store <4 x float> <float 4.000000e+00, float 5.000000e+00, float 6.000000e+00, float 7.000000e+00>
+  static_assert(j[0] == 4.000000e+00 && j[1] == 5.000000e+00 && j[2] == 6.000000e+00 && j[3] == 7.000000e+00, "");
 
   constexpr auto l = 3 + FourFloatsVecSize{6, 3, 2, 1};
-  // CHECK: store <4 x float> <float 9.000000e+00, float 6.000000e+00, float 5.000000e+00, float 4.000000e+00>
+  static_assert(l[0] == 9.000000e+00 && l[1] == 6.000000e+00 && l[2] == 5.000000e+00 && l[3] == 4.000000e+00, "");
+
   constexpr auto m = 20 - FourFloatsVecSize{19, 15, 12, 10};
-  // CHECK: store <4 x float> <float 1.000000e+00, float 5.000000e+00, float 8.000000e+00, float 1.000000e+01>
+  static_assert(m[0] == 1.000000e+00 && m[1] == 5.000000e+00 && m[2] == 8.000000e+00 && m[3] == 1.000000e+01, "");
+
   constexpr auto n = 3 * FourFloatsVecSize{8, 4, 2, 1};
-  // CHECK: store <4 x float> <float 2.400000e+01, float 1.200000e+01, float 6.000000e+00, float 3.000000e+00>
+  static_assert(n[0] == 2.400000e+01 && n[1] == 1.200000e+01 && n[2] == 6.000000e+00 && n[3] == 3.000000e+00, "");
+
   constexpr auto o = 100 / FourFloatsVecSize{12, 15, 18, 21};
-  // CHECK: store <4 x float> <float 0x4020AAAAA0000000, float 0x401AAAAAA0000000, float 0x401638E380000000, float 0x40130C30C0000000>
+  static_assert(CmpF(o[0], 100.0 / 12) && CmpF(o[1], 100.0 / 15) && CmpF(o[2], 100.0 / 18) && CmpF(o[3], 100.0 / 21), "");
 
   constexpr auto w = FourFloatsVecSize{1, 2, 3, 4} <
                      FourFloatsVecSize{4, 3, 2, 1};
-  // CHECK: store <4 x i32> <i32 -1, i32 -1, i32 0, i32 0>
+  static_assert(w[0] == -1 && w[1] == -1 && w[2] == 0 && w[3] == 0, "");
+
   constexpr auto x = FourFloatsVecSize{1, 2, 3, 4} >
                      FourFloatsVecSize{4, 3, 2, 1};
-  // CHECK: store <4 x i32> <i32 0, i32 0, i32 -1, i32 -1>
+  static_assert(x[0] == 0 && x[1] == 0 && x[2] == -1 && x[3] == -1, "");
+
   constexpr auto y = FourFloatsVecSize{1, 2, 3, 4} <=
                      FourFloatsVecSize{4, 3, 3, 1};
-  // CHECK: store <4 x i32> <i32 -1, i32 -1, i32 -1, i32 0>
+  static_assert(y[0] == -1 && y[1] == -1 && y[2] == -1 && y[3] == 0, "");
+
   constexpr auto z = FourFloatsVecSize{1, 2, 3, 4} >=
                      FourFloatsVecSize{4, 3, 3, 1};
-  // CHECK: store <4 x i32> <i32 0, i32 0, i32 -1, i32 -1>
+  static_assert(z[0] == 0 && z[1] == 0 && z[2] == -1 && z[3] == -1, "");
+
   constexpr auto A = FourFloatsVecSize{1, 2, 3, 4} ==
                      FourFloatsVecSize{4, 3, 3, 1};
-  // CHECK: store <4 x i32> <i32 0, i32 0, i32 -1, i32 0>
+  static_assert(A[0] == 0 && A[1] == 0 && A[2] == -1 && A[3] == 0, "");
+
   constexpr auto B = FourFloatsVecSize{1, 2, 3, 4} !=
                      FourFloatsVecSize{4, 3, 3, 1};
-  // CHECK: store <4 x i32> <i32 -1, i32 -1, i32 0, i32 -1>
+  static_assert(B[0] == -1 && B[1] == -1 && B[2] == 0 && B[3] == -1, "");
 
   constexpr auto C = FourFloatsVecSize{1, 2, 3, 4} < 3;
-  // CHECK: store <4 x i32> <i32 -1, i32 -1, i32 0, i32 0>
+  static_assert(C[0] == -1 && C[1] == -1 && C[2] == 0 && C[3] == 0, "");
+     
   constexpr auto D = FourFloatsVecSize{1, 2, 3, 4} > 3;
-  // CHECK: store <4 x i32> <i32 0, i32 0, i32 0, i32 -1>
+  static_assert(D[0] == 0 && D[1] == 0 && D[2] == 0 && D[3] == -1, "");
+
   constexpr auto E = FourFloatsVecSize{1, 2, 3, 4} <= 3;
-  // CHECK: store <4 x i32> <i32 -1, i32 -1, i32 -1, i32 0>
+  static_assert(E[0] == -1 && E[1] == -1 && E[2] == -1 && E[3] == 0, "");
+
   constexpr auto F = FourFloatsVecSize{1, 2, 3, 4} >= 3;
-  // CHECK: store <4 x i32> <i32 0, i32 0, i32 -1, i32 -1>
+  static_assert(F[0] == 0 && F[1] == 0 && F[2] == -1 && F[3] == -1, "");
+
   constexpr auto G = FourFloatsVecSize{1, 2, 3, 4} == 3;
-  // CHECK: store <4 x i32> <i32 0, i32 0, i32 -1, i32 0>
+  static_assert(G[0] == 0 && G[1] == 0 && G[2] == -1 && G[3] == 0, "");
+
   constexpr auto H = FourFloatsVecSize{1, 2, 3, 4} != 3;
-  // CHECK: store <4 x i32> <i32 -1, i32 -1, i32 0, i32 -1>
+  static_assert(H[0] == -1 && H[1] == -1 && H[2] == 0 && H[3] == -1, "");  
 
   constexpr auto O = FourFloatsVecSize{5, 0, 6, 0} &&
                      FourFloatsVecSize{5, 5, 0, 0};
-  // CHECK: store <4 x i32> <i32 1, i32 0, i32 0, i32 0>
+  static_assert(O[0] == 1 && O[1] == 0 && O[2] == 0 && O[3] == 0, "");
+
   constexpr auto P = FourFloatsVecSize{5, 0, 6, 0} ||
                      FourFloatsVecSize{5, 5, 0, 0};
-  // CHECK: store <4 x i32> <i32 1, i32 1, i32 1, i32 0>
+  static_assert(P[0] == 1 && P[1] == 1 && P[2] == 1 && P[3] == 0, "");
 
   constexpr auto Q = FourFloatsVecSize{5, 0, 6, 0} && 3;
-  // CHECK: store <4 x i32> <i32 1, i32 0, i32 1, i32 0>
+  static_assert(Q[0] == 1 && Q[1] == 0 && Q[2] == 1 && Q[3] == 0, "");
+
   constexpr auto R = FourFloatsVecSize{5, 0, 6, 0} || 3;
-  // CHECK: store <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  static_assert(R[0] == 1 && R[1] == 1 && R[2] == 1 && R[3] == 1, "");
 
   constexpr auto T = CmpMul(a, b);
-  // CHECK: store <4 x float> <float 1.080000e+02, float 1.800000e+01, float 5.600000e+01, float 7.200000e+01>
+  static_assert(T[0] == 1.080000e+02 && T[1] == 1.800000e+01 && T[2] == 5.600000e+01 && T[3] == 7.200000e+01, "");
 
   constexpr auto U = CmpDiv(a, b);
-  // CHECK: store <4 x float> <float 3.000000e+00, float 1.800000e+01, float 8.750000e-01, float 0x3FEC71C720000000>
+  static_assert(CmpF(U[0], a[0] / b[0]) && CmpF(U[1], a[1] / b[1]) && CmpF(U[2], a[2] / b[2]) && CmpF(U[3], a[3] / b[3]), "");
 
   constexpr auto X = CmpAdd(a, b);
-  // CHECK: store <4 x float> <float 2.400000e+01, float 1.900000e+01, float 1.500000e+01, float 1.700000e+01>
+  static_assert(X[0] == 2.400000e+01 && X[1] == 1.900000e+01 && X[2] == 1.500000e+01 && X[3] == 1.700000e+01, "");
 
   constexpr auto Y = CmpSub(a, b);
-  // CHECK: store <4 x float> <float 1.200000e+01, float 1.700000e+01, float -1.000000e+00, float -1.000000e+00>
+  static_assert(Y[0] == 1.200000e+01 && Y[1] == 1.700000e+01 && Y[2] == -1.000000e+00 && Y[3] == -1.000000e+00, "");
 
   constexpr auto Z = -Y;
-  // CHECK: store <4 x float> <float -1.200000e+01, float -1.700000e+01, float 1.000000e+00, float 1.000000e+00>
+  static_assert(Z[0] == -1.200000e+01 && Z[1] == -1.700000e+01 && Z[2] == 1.000000e+00 && Z[3] == 1.000000e+00, "");
 
   // Operator ~ is illegal on floats, so no test for that.
   constexpr auto af = !FourFloatsVecSize{0, 1, 8, -1};
-  // CHECK: store <4 x i32> <i32 -1, i32 0, i32 0, i32 0>
+  static_assert(af[0] == -1 && af[1] == 0 && af[2] == 0 && af[3] == 0, "");
 }
 
 void FloatVecUsage() {
   constexpr auto a = FourFloatsVecSize{6, 3, 2, 1} +
                      FourFloatsVecSize{12, 15, 5, 7};
-  // CHECK: <4 x float> <float 1.800000e+01, float 1.800000e+01, float 7.000000e+00, float 8.000000e+00>
+  static_assert(a[0] == 1.800000e+01 && a[1] == 1.800000e+01 && a[2] == 7.000000e+00 && a[3] == 8.000000e+00, "");
+
   constexpr auto b = FourFloatsVecSize{19, 15, 13, 12} -
                      FourFloatsVecSize{13, 14, 5, 3};
-  // CHECK: store <4 x float> <float 6.000000e+00, float 1.000000e+00, float 8.000000e+00, float 9.000000e+00>
+  static_assert(b[0] == 6.000000e+00 && b[1] == 1.000000e+00 && b[2] == 8.000000e+00 && b[3] == 9.000000e+00, "");
+
   constexpr auto c = FourFloatsVecSize{8, 4, 2, 1} *
                      FourFloatsVecSize{3, 4, 5, 6};
-  // CHECK: store <4 x float> <float 2.400000e+01, float 1.600000e+01, float 1.000000e+01, float 6.000000e+00>
+  static_assert(c[0] == 2.400000e+01 && c[1] == 1.600000e+01 && c[2] == 1.000000e+01 && c[3] == 6.000000e+00, "");
+                      
   constexpr auto d = FourFloatsVecSize{12, 12, 10, 10} /
                      FourFloatsVecSize{6, 4, 5, 2};
-  // CHECK: store <4 x float> <float 2.000000e+00, float 3.000000e+00, float 2.000000e+00, float 5.000000e+00>
+  static_assert(d[0] == 2.000000e+00 && d[1] == 3.000000e+00 && d[2] == 2.000000e+00 && d[3] == 5.000000e+00, "");
 
   constexpr auto f = FourFloatsVecSize{6, 3, 2, 1} + 3;
-  // CHECK: store <4 x float> <float 9.000000e+00, float 6.000000e+00, float 5.000000e+00, float 4.000000e+00>
+  static_assert(f[0] == 9.000000e+00 && f[1] == 6.000000e+00 && f[2] == 5.000000e+00 && f[3] == 4.000000e+00, "");
+
   constexpr auto g = FourFloatsVecSize{19, 15, 12, 10} - 3;
-  // CHECK: store <4 x float> <float 1.600000e+01, float 1.200000e+01, float 9.000000e+00, float 7.000000e+00>
+  static_assert(g[0] == 1.600000e+01 && g[1] == 1.200000e+01 && g[2] == 9.000000e+00 && g[3] == 7.000000e+00, "");
+
   constexpr auto h = FourFloatsVecSize{8, 4, 2, 1} * 3;
-  // CHECK: store <4 x float> <float 2.400000e+01, float 1.200000e+01, float 6.000000e+00, float 3.000000e+00>
+  static_assert(h[0] == 2.400000e+01 && h[1] == 1.200000e+01 && h[2] == 6.000000e+00 && h[3] == 3.000000e+00, "");
+
   constexpr auto j = FourFloatsVecSize{12, 15, 18, 21} / 3;
-  // CHECK: store <4 x float> <float 4.000000e+00, float 5.000000e+00, float 6.000000e+00, float 7.000000e+00>
+  static_assert(j[0] == 4.000000e+00 && j[1] == 5.000000e+00 && j[2] == 6.000000e+00 && j[3] == 7.000000e+00, "");
 
   constexpr auto l = 3 + FourFloatsVecSize{6, 3, 2, 1};
-  // CHECK: store <4 x float> <float 9.000000e+00, float 6.000000e+00, float 5.000000e+00, float 4.000000e+00>
+  static_assert(l[0] == 9.000000e+00 && l[1] == 6.000000e+00 && l[2] == 5.000000e+00 && l[3] == 4.000000e+00, "");
+
   constexpr auto m = 20 - FourFloatsVecSize{19, 15, 12, 10};
-  // CHECK: store <4 x float> <float 1.000000e+00, float 5.000000e+00, float 8.000000e+00, float 1.000000e+01>
+  static_assert(m[0] == 1.000000e+00 && m[1] == 5.000000e+00 && m[2] == 8.000000e+00 && m[3] == 1.000000e+01, "");
+
   constexpr auto n = 3 * FourFloatsVecSize{8, 4, 2, 1};
-  // CHECK: store <4 x float> <float 2.400000e+01, float 1.200000e+01, float 6.000000e+00, float 3.000000e+00>
+  static_assert(n[0] == 2.400000e+01 && n[1] == 1.200000e+01 && n[2] == 6.000000e+00 && n[3] == 3.000000e+00, "");
+
   constexpr auto o = 100 / FourFloatsVecSize{12, 15, 18, 21};
-  // CHECK: store <4 x float> <float 0x4020AAAAA0000000, float 0x401AAAAAA0000000, float 0x401638E380000000, float 0x40130C30C0000000>
+  static_assert(CmpF(o[0], 100.0 / 12) && CmpF(o[1], 100.0 / 15) && CmpF(o[2], 100.0 / 18) && CmpF(o[3], 100.0 / 21), "");
 
   constexpr auto w = FourFloatsVecSize{1, 2, 3, 4} <
                      FourFloatsVecSize{4, 3, 2, 1};
-  // CHECK: store <4 x i32> <i32 -1, i32 -1, i32 0, i32 0>
+  static_assert(w[0] == -1 && w[1] == -1 && w[2] == 0 && w[3] == 0, "");
+
   constexpr auto x = FourFloatsVecSize{1, 2, 3, 4} >
                      FourFloatsVecSize{4, 3, 2, 1};
-  // CHECK: store <4 x i32> <i32 0, i32 0, i32 -1, i32 -1>
+  static_assert(x[0] == 0 && x[1] == 0 && x[2] == -1 && x[2] == -1, "");
+
   constexpr auto y = FourFloatsVecSize{1, 2, 3, 4} <=
                      FourFloatsVecSize{4, 3, 3, 1};
-  // CHECK: store <4 x i32> <i32 -1, i32 -1, i32 -1, i32 0>
+  static_assert(y[0] == -1 && y[1] == -1 && y[2] == -1 && y[3] == 0, "");
+
   constexpr auto z = FourFloatsVecSize{1, 2, 3, 4} >=
                      FourFloatsVecSize{4, 3, 3, 1};
-  // CHECK: store <4 x i32> <i32 0, i32 0, i32 -1, i32 -1>
+  static_assert(z[0] == 0 && z[1] == 0 && z[2] == -1 && z[3] == -1, "");
+
   constexpr auto A = FourFloatsVecSize{1, 2, 3, 4} ==
                      FourFloatsVecSize{4, 3, 3, 1};
-  // CHECK: store <4 x i32> <i32 0, i32 0, i32 -1, i32 0>
+  static_assert(A[0] == 0 && A[1] == 0 && A[2] == -1 && A[3] == 0, "");
+
   constexpr auto B = FourFloatsVecSize{1, 2, 3, 4} !=
                      FourFloatsVecSize{4, 3, 3, 1};
-  // CHECK: store <4 x i32> <i32 -1, i32 -1, i32 0, i32 -1>
+  static_assert(B[0] == -1 && B[1] == -1 && B[2] == 0 && B[3] == -1, "");
 
   constexpr auto C = FourFloatsVecSize{1, 2, 3, 4} < 3;
-  // CHECK: store <4 x i32> <i32 -1, i32 -1, i32 0, i32 0>
+  static_assert(C[0] == -1 && C[1] == -1 && C[2] == 0 && C[3] == 0, "");
+
   constexpr auto D = FourFloatsVecSize{1, 2, 3, 4} > 3;
-  // CHECK: store <4 x i32> <i32 0, i32 0, i32 0, i32 -1>
+  static_assert(D[0] == 0 && D[1] == 0 && D[2] == 0 && D[3] == -1, "");
+
   constexpr auto E = FourFloatsVecSize{1, 2, 3, 4} <= 3;
-  // CHECK: store <4 x i32> <i32 -1, i32 -1, i32 -1, i32 0>
+  static_assert(E[0] == -1 && E[1] == -1 && E[2] == -1 && E[3] == 0, "");
+
   constexpr auto F = FourFloatsVecSize{1, 2, 3, 4} >= 3;
-  // CHECK: store <4 x i32> <i32 0, i32 0, i32 -1, i32 -1>
+  static_assert(F[0] == 0 && F[1] == 0 && F[2] == -1 && F[3] == -1, "");
+
   constexpr auto G = FourFloatsVecSize{1, 2, 3, 4} == 3;
-  // CHECK: store <4 x i32> <i32 0, i32 0, i32 -1, i32 0>
+  static_assert(G[0] == 0 && G[1] == 0 && G[2] == -1 && G[3] == 0, "");
+
   constexpr auto H = FourFloatsVecSize{1, 2, 3, 4} != 3;
-  // CHECK: store <4 x i32> <i32 -1, i32 -1, i32 0, i32 -1>
+  static_assert(H[0] == -1 && H[1] == -1 && H[2] == 0 && H[3] == -1, "");
 
   constexpr auto O = FourFloatsVecSize{5, 0, 6, 0} &&
                      FourFloatsVecSize{5, 5, 0, 0};
-  // CHECK: store <4 x i32> <i32 1, i32 0, i32 0, i32 0>
+  static_assert(O[0] == 1 && O[1] == 0 && O[2] == 0 && O[3] == 0, "");
+
   constexpr auto P = FourFloatsVecSize{5, 0, 6, 0} ||
                      FourFloatsVecSize{5, 5, 0, 0};
-  // CHECK: store <4 x i32> <i32 1, i32 1, i32 1, i32 0>
+  static_assert(P[0] == 1 && P[1] == 1 && P[2] == 1 && P[3] == 0, "");
 
   constexpr auto Q = FourFloatsVecSize{5, 0, 6, 0} && 3;
-  // CHECK: store <4 x i32> <i32 1, i32 0, i32 1, i32 0>
+  static_assert(Q[0] == 1 && Q[1] == 0 && Q[2] == 1 && Q[3] == 0, "");
+
   constexpr auto R = FourFloatsVecSize{5, 0, 6, 0} || 3;
-  // CHECK: store <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  static_assert(R[0] == 1 && R[1] == 1 && R[2] == 1 && R[3] == 1, "");
 
   constexpr auto T = CmpMul(a, b);
-  // CHECK: store <4 x float> <float 1.080000e+02, float 1.800000e+01, float 5.600000e+01, float 7.200000e+01>
+  static_assert(T[0] == 1.080000e+02 && T[1] == 1.800000e+01 && T[2] == 5.600000e+01 && T[3] == 7.200000e+01, ""); 
 
   constexpr auto U = CmpDiv(a, b);
-  // CHECK: store <4 x float> <float 3.000000e+00, float 1.800000e+01, float 8.750000e-01, float 0x3FEC71C720000000>
+  static_assert(CmpF(U[0], a[0] / b[0]) && CmpF(U[1], a[1] / b[1]) && CmpF(U[2], a[2] / b[2]) && CmpF(U[3], a[3] / b[3]), "");
 
   constexpr auto X = CmpAdd(a, b);
-  // CHECK: store <4 x float> <float 2.400000e+01, float 1.900000e+01, float 1.500000e+01, float 1.700000e+01>
-
+  static_assert(X[0] == 2.400000e+01 && X[1] == 1.900000e+01 && X[2] == 1.500000e+01 && X[3] == 1.700000e+01, "");
+ 
   constexpr auto Y = CmpSub(a, b);
-  // CHECK: store <4 x float> <float 1.200000e+01, float 1.700000e+01, float -1.000000e+00, float -1.000000e+00>
+  static_assert(Y[0] == 1.200000e+01 && Y[1] == 1.700000e+01 && Y[2] == -1.000000e+00 && Y[3] == -1.000000e+00, "");
 
   constexpr auto Z = -Y;
-  // CHECK: store <4 x float> <float -1.200000e+01, float -1.700000e+01, float 1.000000e+00, float 1.000000e+00>
+  static_assert(Z[0] == -1.200000e+01 && Z[1] == -1.700000e+01 && Z[2] == 1.000000e+00 && Z[3] == 1.000000e+00, "");
 
   // Operator ~ is illegal on floats, so no test for that.
   constexpr auto af = !FourFloatsVecSize{0, 1, 8, -1};
-  // CHECK: store <4 x i32> <i32 -1, i32 0, i32 0, i32 0>
+  static_assert(af[0] == -1 && af[1] == 0 && af[2] == 0 && af[3] == 0, "");
 }
 
 void I128Usage() {
   constexpr auto a = FourI128VecSize{1, 2, 3, 4};
-  // CHECK: store <4 x i128> <i128 1, i128 2, i128 3, i128 4>
+  static_assert(a[0] == 1 && a[1] == 2 && a[2] == 3 && a[3] == 4, "");
+
   constexpr auto b = a < 3;
-  // CHECK: store <4 x i128> <i128 -1, i128 -1, i128 0, i128 0>
+  static_assert(b[0] == -1 && b[1] == -1 && b[2] == 0 && b[3] == 0, "");
 
   // Operator ~ is illegal on floats, so no test for that.
   constexpr auto c = ~FourI128VecSize{1, 2, 10, 20};
-  // CHECK: store <4 x i128> <i128 -2, i128 -3, i128 -11, i128 -21>
+  static_assert(c[0] == -2 && c[1] == -3 && c[2] == -11 && c[3] == -21, "");
 
   constexpr auto d = !FourI128VecSize{0, 1, 8, -1};
-  // CHECK: store <4 x i128> <i128 -1, i128 0, i128 0, i128 0>
+  static_assert(d[0] == -1 && d[1] == 0 && d[2] == 0 && d[3] == 0, "");
 }
 
 void I128VecUsage() {
   constexpr auto a = FourI128ExtVec{1, 2, 3, 4};
-  // CHECK: store <4 x i128> <i128 1, i128 2, i128 3, i128 4>
+  static_assert(a[0] == 1 && a[1] == 2 && a[2] == 3 && a[3] == 4, "");
+
   constexpr auto b = a < 3;
-  // CHECK: store <4 x i128> <i128 -1, i128 -1, i128 0, i128 0>
+  static_assert(b[0] == -1 && b[1] == -1 && b[2] == 0 && b[3] == 0, "");
 
   // Operator ~ is illegal on floats, so no test for that.
   constexpr auto c = ~FourI128ExtVec{1, 2, 10, 20};
-  // CHECK: store <4 x i128>  <i128 -2, i128 -3, i128 -11, i128 -21>
+  static_assert(c[0] == -2 && c[1] == -3 && c[2] == -11 && c[3] == -21, "");
 
   constexpr auto d = !FourI128ExtVec{0, 1, 8, -1};
-  // CHECK: store <4 x i128>  <i128 -1, i128 0, i128 0, i128 0>
+  static_assert(d[0] == -1 && d[1] == 0 && d[2] == 0 && d[3] == 0, "");
 }
 
 using FourBoolsExtVec __attribute__((ext_vector_type(4))) = bool;
 void BoolVecUsage() {
   constexpr auto a = FourBoolsExtVec{true, false, true, false} <
                      FourBoolsExtVec{false, false, true, true};
-  // CHECK: store i8 bitcast (<8 x i1> <i1 false, i1 false, i1 false, i1 true, i1 undef, i1 undef, i1 undef, i1 undef> to i8), ptr %a, align 1
+  static_assert(a[0] == false && a[1] == false && a[2] == false && a[3] == true, "");
+
   constexpr auto b = FourBoolsExtVec{true, false, true, false} <=
                      FourBoolsExtVec{false, false, true, true};
-  // CHECK: store i8 bitcast (<8 x i1> <i1 false, i1 true, i1 true, i1 true, i1 undef, i1 undef, i1 undef, i1 undef> to i8), ptr %b, align 1
+  static_assert(b[0] == false && b[1] == true && b[2] == true && b[3] == true, "");
+
   constexpr auto c = FourBoolsExtVec{true, false, true, false} ==
                      FourBoolsExtVec{false, false, true, true};
-  // CHECK: store i8 bitcast (<8 x i1> <i1 false, i1 true, i1 true, i1 false, i1 undef, i1 undef, i1 undef, i1 undef> to i8), ptr %c, align 1
+  static_assert(c[0] == false && c[1] == true && c[2] == true && c[3] == false, "");
+
   constexpr auto d = FourBoolsExtVec{true, false, true, false} !=
                      FourBoolsExtVec{false, false, true, true};
-  // CHECK: store i8 bitcast (<8 x i1> <i1 true, i1 false, i1 false, i1 true, i1 undef, i1 undef, i1 undef, i1 undef> to i8), ptr %d, align 1
+  static_assert(d[0] == true && d[1] == false && d[2] == false && d[3] == true, "");
+
   constexpr auto e = FourBoolsExtVec{true, false, true, false} >=
                      FourBoolsExtVec{false, false, true, true};
-  // CHECK: store i8 bitcast (<8 x i1> <i1 true, i1 true, i1 true, i1 false, i1 undef, i1 undef, i1 undef, i1 undef> to i8), ptr %e, align 1
+  static_assert(e[0] == true && e[1] == true && e[2] == true && e[3] == false, "");
+
   constexpr auto f = FourBoolsExtVec{true, false, true, false} >
                      FourBoolsExtVec{false, false, true, true};
-  // CHECK: store i8 bitcast (<8 x i1> <i1 true, i1 false, i1 false, i1 false, i1 undef, i1 undef, i1 undef, i1 undef> to i8), ptr %f, align 1
+  static_assert(f[0] == true && f[1] == false && f[2] == false && f[3] == false, "");
+
   constexpr auto g = FourBoolsExtVec{true, false, true, false} &
                      FourBoolsExtVec{false, false, true, true};
-  // CHECK: store i8 bitcast (<8 x i1> <i1 false, i1 false, i1 true, i1 false, i1 undef, i1 undef, i1 undef, i1 undef> to i8), ptr %g, align 1
+  static_assert(g[0] == false && g[1] == false && g[2] == true && g[3] == false, "");
+
   constexpr auto h = FourBoolsExtVec{true, false, true, false} |
                      FourBoolsExtVec{false, false, true, true};
-  // CHECK: store i8 bitcast (<8 x i1> <i1 true, i1 false, i1 true, i1 true, i1 undef, i1 undef, i1 undef, i1 undef> to i8), ptr %h, align 1
+  static_assert(h[0] == true && h[1] == false && h[2] == true && h[3] == true, "");
+
   constexpr auto i = FourBoolsExtVec{true, false, true, false} ^
                      FourBoolsExtVec { false, false, true, true };
-  // CHECK: store i8 bitcast (<8 x i1> <i1 true, i1 false, i1 false, i1 true, i1 undef, i1 undef, i1 undef, i1 undef> to i8), ptr %i, align 1
+  static_assert(i[0] == true && i[1] == false && i[2] == false && i[3] == true, "");  
+
   constexpr auto j = !FourBoolsExtVec{true, false, true, false};
-  // CHECK: store i8 bitcast (<8 x i1> <i1 false, i1 true, i1 false, i1 true, i1 undef, i1 undef, i1 undef, i1 undef> to i8), ptr %j, align 1
+  static_assert(j[0] == false && j[1] == true && j[2] == false && j[3] == true, "");
+
   constexpr auto k = ~FourBoolsExtVec{true, false, true, false};
-  // CHECK: store i8 bitcast (<8 x i1> <i1 false, i1 true, i1 false, i1 true, i1 undef, i1 undef, i1 undef, i1 undef> to i8), ptr %k, align 1
+  static_assert(k[0] == false && k[1] == true && k[2] == false && k[3] == true, "");
 }

From 50c876a486ebc1a8fbf6f20214b1cf914729c443 Mon Sep 17 00:00:00 2001
From: Mircea Trofin <mtrofin@google.com>
Date: Thu, 15 Aug 2024 13:16:55 -0700
Subject: [PATCH 076/441] [nfc][ctx_prof] Remove the need for `PassBuilder` to
 know about `UseCtxProfile` (#104492)

---
 llvm/include/llvm/Analysis/CtxProfAnalysis.h | 2 +-
 llvm/lib/Analysis/CtxProfAnalysis.cpp        | 3 +++
 llvm/lib/Passes/PassBuilder.cpp              | 2 --
 llvm/lib/Passes/PassRegistry.def             | 2 +-
 4 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/llvm/include/llvm/Analysis/CtxProfAnalysis.h b/llvm/include/llvm/Analysis/CtxProfAnalysis.h
index f0e2aeb0f92f742..d0fb99fe1966a65 100644
--- a/llvm/include/llvm/Analysis/CtxProfAnalysis.h
+++ b/llvm/include/llvm/Analysis/CtxProfAnalysis.h
@@ -79,7 +79,7 @@ class CtxProfAnalysis : public AnalysisInfoMixin<CtxProfAnalysis> {
 
 public:
   static AnalysisKey Key;
-  explicit CtxProfAnalysis(StringRef Profile) : Profile(Profile) {};
+  explicit CtxProfAnalysis(StringRef Profile = "");
 
   using Result = PGOContextualProfile;
 
diff --git a/llvm/lib/Analysis/CtxProfAnalysis.cpp b/llvm/lib/Analysis/CtxProfAnalysis.cpp
index 7b4666b29a19369..d0ccf4ba537f847 100644
--- a/llvm/lib/Analysis/CtxProfAnalysis.cpp
+++ b/llvm/lib/Analysis/CtxProfAnalysis.cpp
@@ -96,6 +96,9 @@ GlobalValue::GUID AssignGUIDPass::getGUID(const Function &F) {
 }
 AnalysisKey CtxProfAnalysis::Key;
 
+CtxProfAnalysis::CtxProfAnalysis(StringRef Profile)
+    : Profile(Profile.empty() ? UseCtxProfile : Profile) {}
+
 PGOContextualProfile CtxProfAnalysis::run(Module &M,
                                           ModuleAnalysisManager &MAM) {
   ErrorOr<std::unique_ptr<MemoryBuffer>> MB = MemoryBuffer::getFile(Profile);
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index c528863d1870a4a..3200767282b226e 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -334,8 +334,6 @@ cl::opt<bool> PrintPipelinePasses(
              "(best-effort only)."));
 } // namespace llvm
 
-extern cl::opt<std::string> UseCtxProfile;
-
 AnalysisKey NoOpModuleAnalysis::Key;
 AnalysisKey NoOpCGSCCAnalysis::Key;
 AnalysisKey NoOpFunctionAnalysis::Key;
diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def
index 18f4aa19224da05..442c972fc616fcf 100644
--- a/llvm/lib/Passes/PassRegistry.def
+++ b/llvm/lib/Passes/PassRegistry.def
@@ -20,7 +20,7 @@
 #endif
 MODULE_ANALYSIS("callgraph", CallGraphAnalysis())
 MODULE_ANALYSIS("collector-metadata", CollectorMetadataAnalysis())
-MODULE_ANALYSIS("ctx-prof-analysis", CtxProfAnalysis(UseCtxProfile))
+MODULE_ANALYSIS("ctx-prof-analysis", CtxProfAnalysis())
 MODULE_ANALYSIS("dxil-metadata", DXILMetadataAnalysis())
 MODULE_ANALYSIS("dxil-resource", DXILResourceAnalysis())
 MODULE_ANALYSIS("inline-advisor", InlineAdvisorAnalysis())

From 2adc012c34b269f80a735c1ec2011f3a98175cbc Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Thu, 15 Aug 2024 13:17:47 -0700
Subject: [PATCH 077/441] [RISCV][GISel] Support nxv16p0 for RV32. (#101573)

Pointers are 32 bits on RV32 so nxv1p0 is lmul=mf2 and nxv16p0 is
lmul=m8.

Split the test so we can have different alignments and register class
sizes for rv32 and rv64 for the pointer tests.
---
 .../Target/RISCV/GISel/RISCVLegalizerInfo.cpp |  7 +-
 .../legalizer/rvv/legalize-load-rv32.mir      | 79 +++++++++++++++++++
 .../legalizer/rvv/legalize-load-rv64.mir      | 60 ++++++++++++++
 .../legalizer/rvv/legalize-load.mir           | 72 -----------------
 .../legalizer/rvv/legalize-store-rv32.mir     | 79 +++++++++++++++++++
 .../legalizer/rvv/legalize-store-rv64.mir     | 60 ++++++++++++++
 .../legalizer/rvv/legalize-store.mir          | 72 -----------------
 7 files changed, 283 insertions(+), 146 deletions(-)
 create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-load-rv32.mir
 create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-load-rv64.mir
 create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-store-rv32.mir
 create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-store-rv64.mir

diff --git a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
index 7491af1aae2a5a3..1978c07f0cafe2e 100644
--- a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
+++ b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
@@ -74,7 +74,9 @@ static LegalityPredicate typeIsLegalPtrVec(unsigned TypeIdx,
   LegalityPredicate P = [=, &ST](const LegalityQuery &Query) {
     return ST.hasVInstructions() &&
            (Query.Types[TypeIdx].getElementCount().getKnownMinValue() != 1 ||
-            ST.getELen() == 64);
+            ST.getELen() == 64) &&
+           (Query.Types[TypeIdx].getElementCount().getKnownMinValue() != 16 ||
+            Query.Types[TypeIdx].getScalarSizeInBits() == 32);
   };
   return all(typeInSet(TypeIdx, PtrVecTys), P);
 }
@@ -127,6 +129,7 @@ RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST)
   const LLT nxv2p0 = LLT::scalable_vector(2, p0);
   const LLT nxv4p0 = LLT::scalable_vector(4, p0);
   const LLT nxv8p0 = LLT::scalable_vector(8, p0);
+  const LLT nxv16p0 = LLT::scalable_vector(16, p0);
 
   using namespace TargetOpcode;
 
@@ -137,7 +140,7 @@ RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST)
                         nxv32s16, nxv1s32, nxv2s32, nxv4s32, nxv8s32, nxv16s32,
                         nxv1s64,  nxv2s64, nxv4s64, nxv8s64};
 
-  auto PtrVecTys = {nxv1p0, nxv2p0, nxv4p0, nxv8p0};
+  auto PtrVecTys = {nxv1p0, nxv2p0, nxv4p0, nxv8p0, nxv16p0};
 
   getActionDefinitionsBuilder({G_ADD, G_SUB, G_AND, G_OR, G_XOR})
       .legalFor({s32, sXLen})
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-load-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-load-rv32.mir
new file mode 100644
index 000000000000000..d8fe9b77de432d9
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-load-rv32.mir
@@ -0,0 +1,79 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=riscv32 -mattr=+v -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name:            vload_nxv1ptr
+body:             |
+  bb.1:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: vload_nxv1ptr
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+    ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(<vscale x 1 x p0>) = G_LOAD [[COPY]](p0) :: (load (<vscale x 1 x p0>))
+    ; CHECK-NEXT: $v8 = COPY [[LOAD]](<vscale x 1 x p0>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %0:_(p0) = COPY $x10
+    %1:_(<vscale x 1 x p0>) = G_LOAD %0(p0) :: (load (<vscale x 1 x p0>), align 4)
+    $v8 = COPY %1(<vscale x 1 x p0>)
+    PseudoRET implicit $v8
+
+...
+---
+name:            vload_nxv2ptr
+body:             |
+  bb.1:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: vload_nxv2ptr
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+    ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(<vscale x 2 x p0>) = G_LOAD [[COPY]](p0) :: (load (<vscale x 2 x p0>))
+    ; CHECK-NEXT: $v8 = COPY [[LOAD]](<vscale x 2 x p0>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %0:_(p0) = COPY $x10
+    %1:_(<vscale x 2 x p0>) = G_LOAD %0(p0) :: (load (<vscale x 2 x p0>), align 8)
+    $v8 = COPY %1(<vscale x 2 x p0>)
+    PseudoRET implicit $v8
+
+...
+---
+name:            vload_nxv8ptr
+body:             |
+  bb.1:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: vload_nxv8ptr
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+    ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(<vscale x 8 x p0>) = G_LOAD [[COPY]](p0) :: (load (<vscale x 8 x p0>))
+    ; CHECK-NEXT: $v8m4 = COPY [[LOAD]](<vscale x 8 x p0>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m4
+    %0:_(p0) = COPY $x10
+    %1:_(<vscale x 8 x p0>) = G_LOAD %0(p0) :: (load (<vscale x 8 x p0>), align 32)
+    $v8m4 = COPY %1(<vscale x 8 x p0>)
+    PseudoRET implicit $v8m4
+
+...
+---
+name:            vload_nxv16ptr
+body:             |
+  bb.1:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: vload_nxv16ptr
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+    ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(<vscale x 16 x p0>) = G_LOAD [[COPY]](p0) :: (load (<vscale x 16 x p0>))
+    ; CHECK-NEXT: $v8m4 = COPY [[LOAD]](<vscale x 16 x p0>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m8
+    %0:_(p0) = COPY $x10
+    %1:_(<vscale x 16 x p0>) = G_LOAD %0(p0) :: (load (<vscale x 16 x p0>), align 64)
+    $v8m4 = COPY %1(<vscale x 16 x p0>)
+    PseudoRET implicit $v8m8
+
+...
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-load-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-load-rv64.mir
new file mode 100644
index 000000000000000..98dee70a42cdba6
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-load-rv64.mir
@@ -0,0 +1,60 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=riscv64 -mattr=+v -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name:            vload_nxv1ptr
+body:             |
+  bb.1:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: vload_nxv1ptr
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+    ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(<vscale x 1 x p0>) = G_LOAD [[COPY]](p0) :: (load (<vscale x 1 x p0>))
+    ; CHECK-NEXT: $v8 = COPY [[LOAD]](<vscale x 1 x p0>)
+    ; CHECK-NEXT: PseudoRET implicit $v8
+    %0:_(p0) = COPY $x10
+    %1:_(<vscale x 1 x p0>) = G_LOAD %0(p0) :: (load (<vscale x 1 x p0>), align 8)
+    $v8 = COPY %1(<vscale x 1 x p0>)
+    PseudoRET implicit $v8
+
+...
+---
+name:            vload_nxv2ptr
+body:             |
+  bb.1:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: vload_nxv2ptr
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+    ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(<vscale x 2 x p0>) = G_LOAD [[COPY]](p0) :: (load (<vscale x 2 x p0>))
+    ; CHECK-NEXT: $v8 = COPY [[LOAD]](<vscale x 2 x p0>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m2
+    %0:_(p0) = COPY $x10
+    %1:_(<vscale x 2 x p0>) = G_LOAD %0(p0) :: (load (<vscale x 2 x p0>), align 16)
+    $v8 = COPY %1(<vscale x 2 x p0>)
+    PseudoRET implicit $v8m2
+
+...
+---
+name:            vload_nxv8ptr
+body:             |
+  bb.1:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: vload_nxv8ptr
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+    ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(<vscale x 8 x p0>) = G_LOAD [[COPY]](p0) :: (load (<vscale x 8 x p0>))
+    ; CHECK-NEXT: $v8m4 = COPY [[LOAD]](<vscale x 8 x p0>)
+    ; CHECK-NEXT: PseudoRET implicit $v8m8
+    %0:_(p0) = COPY $x10
+    %1:_(<vscale x 8 x p0>) = G_LOAD %0(p0) :: (load (<vscale x 8 x p0>), align 64)
+    $v8m4 = COPY %1(<vscale x 8 x p0>)
+    PseudoRET implicit $v8m8
+
+...
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-load.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-load.mir
index 12f218863e4002b..2023bf7c05565b8 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-load.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-load.mir
@@ -203,21 +203,6 @@
     ret <vscale x 2 x i64> %va
   }
 
-  define <vscale x 1 x ptr> @vload_nxv1ptr(ptr %pa) #0 {
-    %va = load <vscale x 1 x ptr>, ptr %pa, align 4
-    ret <vscale x 1 x ptr> %va
-  }
-
-  define <vscale x 2 x ptr> @vload_nxv2ptr(ptr %pa) #0 {
-    %va = load <vscale x 2 x ptr>, ptr %pa, align 8
-    ret <vscale x 2 x ptr> %va
-  }
-
-  define <vscale x 8 x ptr> @vload_nxv8ptr(ptr %pa) #0 {
-    %va = load <vscale x 8 x ptr>, ptr %pa, align 32
-    ret <vscale x 8 x ptr> %va
-  }
-
   attributes #0 = { "target-features"="+v" }
 
 ...
@@ -984,60 +969,3 @@ body:             |
     PseudoRET implicit $v8m2
 
 ...
----
-name:            vload_nxv1ptr
-body:             |
-  bb.1 (%ir-block.0):
-    liveins: $x10
-
-    ; CHECK-LABEL: name: vload_nxv1ptr
-    ; CHECK: liveins: $x10
-    ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
-    ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(<vscale x 1 x p0>) = G_LOAD [[COPY]](p0) :: (load (<vscale x 1 x p0>) from %ir.pa)
-    ; CHECK-NEXT: $v8 = COPY [[LOAD]](<vscale x 1 x p0>)
-    ; CHECK-NEXT: PseudoRET implicit $v8
-    %0:_(p0) = COPY $x10
-    %1:_(<vscale x 1 x p0>) = G_LOAD %0(p0) :: (load (<vscale x 1 x p0>) from %ir.pa)
-    $v8 = COPY %1(<vscale x 1 x p0>)
-    PseudoRET implicit $v8
-
-...
----
-name:            vload_nxv2ptr
-body:             |
-  bb.1 (%ir-block.0):
-    liveins: $x10
-
-    ; CHECK-LABEL: name: vload_nxv2ptr
-    ; CHECK: liveins: $x10
-    ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
-    ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(<vscale x 2 x p0>) = G_LOAD [[COPY]](p0) :: (load (<vscale x 2 x p0>) from %ir.pa)
-    ; CHECK-NEXT: $v8 = COPY [[LOAD]](<vscale x 2 x p0>)
-    ; CHECK-NEXT: PseudoRET implicit $v8
-    %0:_(p0) = COPY $x10
-    %1:_(<vscale x 2 x p0>) = G_LOAD %0(p0) :: (load (<vscale x 2 x p0>) from %ir.pa)
-    $v8 = COPY %1(<vscale x 2 x p0>)
-    PseudoRET implicit $v8
-
-...
----
-name:            vload_nxv8ptr
-body:             |
-  bb.1 (%ir-block.0):
-    liveins: $x10
-
-    ; CHECK-LABEL: name: vload_nxv8ptr
-    ; CHECK: liveins: $x10
-    ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
-    ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(<vscale x 8 x p0>) = G_LOAD [[COPY]](p0) :: (load (<vscale x 8 x p0>) from %ir.pa)
-    ; CHECK-NEXT: $v8m4 = COPY [[LOAD]](<vscale x 8 x p0>)
-    ; CHECK-NEXT: PseudoRET implicit $v8m4
-    %0:_(p0) = COPY $x10
-    %1:_(<vscale x 8 x p0>) = G_LOAD %0(p0) :: (load (<vscale x 8 x p0>) from %ir.pa)
-    $v8m4 = COPY %1(<vscale x 8 x p0>)
-    PseudoRET implicit $v8m4
-
-...
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-store-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-store-rv32.mir
new file mode 100644
index 000000000000000..a93cce6904c2ca8
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-store-rv32.mir
@@ -0,0 +1,79 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=riscv32 -mattr=+v -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name:            vstore_nx1ptr
+body:             |
+  bb.1:
+    liveins: $v8, $x10
+
+    ; CHECK-LABEL: name: vstore_nx1ptr
+    ; CHECK: liveins: $v8, $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<vscale x 1 x p0>) = COPY $v8
+    ; CHECK-NEXT: G_STORE [[COPY1]](<vscale x 1 x p0>), [[COPY]](p0) :: (store (<vscale x 1 x p0>))
+    ; CHECK-NEXT: PseudoRET
+    %0:_(p0) = COPY $x10
+    %1:_(<vscale x 1 x p0>) = COPY $v8
+    G_STORE %1(<vscale x 1 x p0>), %0(p0) :: (store (<vscale x 1 x p0>), align 4)
+    PseudoRET
+
+...
+---
+name:            vstore_nx2ptr
+body:             |
+  bb.1:
+    liveins: $x10, $v8
+
+    ; CHECK-LABEL: name: vstore_nx2ptr
+    ; CHECK: liveins: $x10, $v8
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<vscale x 2 x p0>) = COPY $v8
+    ; CHECK-NEXT: G_STORE [[COPY1]](<vscale x 2 x p0>), [[COPY]](p0) :: (store (<vscale x 2 x p0>))
+    ; CHECK-NEXT: PseudoRET
+    %0:_(p0) = COPY $x10
+    %1:_(<vscale x 2 x p0>) = COPY $v8
+    G_STORE %1(<vscale x 2 x p0>), %0(p0) :: (store (<vscale x 2 x p0>), align 8)
+    PseudoRET
+
+...
+---
+name:            vstore_nx8ptr
+body:             |
+  bb.1:
+    liveins: $x10, $v8m4
+
+    ; CHECK-LABEL: name: vstore_nx8ptr
+    ; CHECK: liveins: $x10, $v8m4
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<vscale x 8 x p0>) = COPY $v8m4
+    ; CHECK-NEXT: G_STORE [[COPY1]](<vscale x 8 x p0>), [[COPY]](p0) :: (store (<vscale x 8 x p0>))
+    ; CHECK-NEXT: PseudoRET
+    %0:_(p0) = COPY $x10
+    %1:_(<vscale x 8 x p0>) = COPY $v8m4
+    G_STORE %1(<vscale x 8 x p0>), %0(p0) :: (store (<vscale x 8 x p0>), align 32)
+    PseudoRET
+
+...
+---
+name:            vstore_nx16ptr
+body:             |
+  bb.1:
+    liveins: $x10, $v8m8
+
+    ; CHECK-LABEL: name: vstore_nx16ptr
+    ; CHECK: liveins: $x10, $v8m8
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<vscale x 16 x p0>) = COPY $v8m8
+    ; CHECK-NEXT: G_STORE [[COPY1]](<vscale x 16 x p0>), [[COPY]](p0) :: (store (<vscale x 8 x p0>), align 64)
+    ; CHECK-NEXT: PseudoRET
+    %0:_(p0) = COPY $x10
+    %1:_(<vscale x 16 x p0>) = COPY $v8m8
+    G_STORE %1(<vscale x 16 x p0>), %0(p0) :: (store (<vscale x 8 x p0>), align 64)
+    PseudoRET
+
+...
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-store-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-store-rv64.mir
new file mode 100644
index 000000000000000..3373ed4d287467f
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-store-rv64.mir
@@ -0,0 +1,60 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=riscv64 -mattr=+v -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name:            vstore_nx1ptr
+body:             |
+  bb.1:
+    liveins: $v8, $x10
+
+    ; CHECK-LABEL: name: vstore_nx1ptr
+    ; CHECK: liveins: $v8, $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<vscale x 1 x p0>) = COPY $v8
+    ; CHECK-NEXT: G_STORE [[COPY1]](<vscale x 1 x p0>), [[COPY]](p0) :: (store (<vscale x 1 x p0>))
+    ; CHECK-NEXT: PseudoRET
+    %0:_(p0) = COPY $x10
+    %1:_(<vscale x 1 x p0>) = COPY $v8
+    G_STORE %1(<vscale x 1 x p0>), %0(p0) :: (store (<vscale x 1 x p0>), align 8)
+    PseudoRET
+
+...
+---
+name:            vstore_nx2ptr
+body:             |
+  bb.1:
+    liveins: $x10, $v8m2
+
+    ; CHECK-LABEL: name: vstore_nx2ptr
+    ; CHECK: liveins: $x10, $v8m2
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<vscale x 2 x p0>) = COPY $v8m2
+    ; CHECK-NEXT: G_STORE [[COPY1]](<vscale x 2 x p0>), [[COPY]](p0) :: (store (<vscale x 2 x p0>))
+    ; CHECK-NEXT: PseudoRET
+    %0:_(p0) = COPY $x10
+    %1:_(<vscale x 2 x p0>) = COPY $v8m2
+    G_STORE %1(<vscale x 2 x p0>), %0(p0) :: (store (<vscale x 2 x p0>), align 16)
+    PseudoRET
+
+...
+---
+name:            vstore_nx8ptr
+body:             |
+  bb.1:
+    liveins: $x10, $v8m8
+
+    ; CHECK-LABEL: name: vstore_nx8ptr
+    ; CHECK: liveins: $x10, $v8m8
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<vscale x 8 x p0>) = COPY $v8m8
+    ; CHECK-NEXT: G_STORE [[COPY1]](<vscale x 8 x p0>), [[COPY]](p0) :: (store (<vscale x 8 x p0>))
+    ; CHECK-NEXT: PseudoRET
+    %0:_(p0) = COPY $x10
+    %1:_(<vscale x 8 x p0>) = COPY $v8m8
+    G_STORE %1(<vscale x 8 x p0>), %0(p0) :: (store (<vscale x 8 x p0>), align 64)
+    PseudoRET
+
+...
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-store.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-store.mir
index b91d25509646f5d..4bb4eb5fa0c72b3 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-store.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/rvv/legalize-store.mir
@@ -203,21 +203,6 @@
     ret void
   }
 
-  define void @vstore_nx1ptr(ptr %pa, <vscale x 1 x ptr> %b) #0 {
-    store <vscale x 1 x ptr> %b, ptr %pa, align 4
-    ret void
-  }
-
-  define void @vstore_nx2ptr(ptr %pa, <vscale x 2 x ptr> %b) #0 {
-    store <vscale x 2 x ptr> %b, ptr %pa, align 8
-    ret void
-  }
-
-  define void @vstore_nx8ptr(ptr %pa, <vscale x 8 x ptr> %b) #0 {
-    store <vscale x 8 x ptr> %b, ptr %pa, align 32
-    ret void
-  }
-
   attributes #0 = { "target-features"="+v" }
 
 ...
@@ -984,60 +969,3 @@ body:             |
     PseudoRET
 
 ...
----
-name:            vstore_nx1ptr
-body:             |
-  bb.1 (%ir-block.0):
-    liveins: $v8, $x10
-
-    ; CHECK-LABEL: name: vstore_nx1ptr
-    ; CHECK: liveins: $v8, $x10
-    ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
-    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<vscale x 1 x p0>) = COPY $v8
-    ; CHECK-NEXT: G_STORE [[COPY1]](<vscale x 1 x p0>), [[COPY]](p0) :: (store (<vscale x 1 x p0>) into %ir.pa)
-    ; CHECK-NEXT: PseudoRET
-    %0:_(p0) = COPY $x10
-    %1:_(<vscale x 1 x p0>) = COPY $v8
-    G_STORE %1(<vscale x 1 x p0>), %0(p0) :: (store (<vscale x 1 x p0>) into %ir.pa)
-    PseudoRET
-
-...
----
-name:            vstore_nx2ptr
-body:             |
-  bb.1 (%ir-block.0):
-    liveins: $x10, $v8m2
-
-    ; CHECK-LABEL: name: vstore_nx2ptr
-    ; CHECK: liveins: $x10, $v8m2
-    ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
-    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<vscale x 2 x p0>) = COPY $v8m2
-    ; CHECK-NEXT: G_STORE [[COPY1]](<vscale x 2 x p0>), [[COPY]](p0) :: (store (<vscale x 2 x p0>) into %ir.pa)
-    ; CHECK-NEXT: PseudoRET
-    %0:_(p0) = COPY $x10
-    %1:_(<vscale x 2 x p0>) = COPY $v8m2
-    G_STORE %1(<vscale x 2 x p0>), %0(p0) :: (store (<vscale x 2 x p0>) into %ir.pa)
-    PseudoRET
-
-...
----
-name:            vstore_nx8ptr
-body:             |
-  bb.1 (%ir-block.0):
-    liveins: $x10, $v8m8
-
-    ; CHECK-LABEL: name: vstore_nx8ptr
-    ; CHECK: liveins: $x10, $v8m8
-    ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10
-    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<vscale x 8 x p0>) = COPY $v8m8
-    ; CHECK-NEXT: G_STORE [[COPY1]](<vscale x 8 x p0>), [[COPY]](p0) :: (store (<vscale x 8 x p0>) into %ir.pa)
-    ; CHECK-NEXT: PseudoRET
-    %0:_(p0) = COPY $x10
-    %1:_(<vscale x 8 x p0>) = COPY $v8m8
-    G_STORE %1(<vscale x 8 x p0>), %0(p0) :: (store (<vscale x 8 x p0>) into %ir.pa)
-    PseudoRET
-
-...

From 6ec169d3501124770c3301dab8156c4640346c40 Mon Sep 17 00:00:00 2001
From: vporpo <vporpodas@google.com>
Date: Thu, 15 Aug 2024 13:28:08 -0700
Subject: [PATCH 078/441] [SandboxIR] Implement BinaryOperator (#104121)

This patch implements sandboxir::BinaryOperator mirroring
llvm::BinaryOperator.
---
 llvm/include/llvm/SandboxIR/SandboxIR.h       |  85 ++++++++
 .../llvm/SandboxIR/SandboxIRValues.def        |  44 ++--
 llvm/lib/SandboxIR/SandboxIR.cpp              | 128 +++++++++++
 llvm/unittests/SandboxIR/SandboxIRTest.cpp    | 202 ++++++++++++++++--
 4 files changed, 435 insertions(+), 24 deletions(-)

diff --git a/llvm/include/llvm/SandboxIR/SandboxIR.h b/llvm/include/llvm/SandboxIR/SandboxIR.h
index a2e2a32e9c01eb9..b1af769f29af540 100644
--- a/llvm/include/llvm/SandboxIR/SandboxIR.h
+++ b/llvm/include/llvm/SandboxIR/SandboxIR.h
@@ -130,6 +130,7 @@ class CastInst;
 class PtrToIntInst;
 class BitCastInst;
 class AllocaInst;
+class BinaryOperator;
 class AtomicCmpXchgInst;
 
 /// Iterator for the `Use` edges of a User's operands.
@@ -249,6 +250,7 @@ class Value {
   friend class InvokeInst;         // For getting `Val`.
   friend class CallBrInst;         // For getting `Val`.
   friend class GetElementPtrInst;  // For getting `Val`.
+  friend class BinaryOperator;     // For getting `Val`.
   friend class AtomicCmpXchgInst;  // For getting `Val`.
   friend class AllocaInst;         // For getting `Val`.
   friend class CastInst;           // For getting `Val`.
@@ -630,6 +632,7 @@ class Instruction : public sandboxir::User {
   friend class InvokeInst;         // For getTopmostLLVMInstruction().
   friend class CallBrInst;         // For getTopmostLLVMInstruction().
   friend class GetElementPtrInst;  // For getTopmostLLVMInstruction().
+  friend class BinaryOperator;     // For getTopmostLLVMInstruction().
   friend class AtomicCmpXchgInst;  // For getTopmostLLVMInstruction().
   friend class AllocaInst;         // For getTopmostLLVMInstruction().
   friend class CastInst;           // For getTopmostLLVMInstruction().
@@ -1432,6 +1435,86 @@ class GetElementPtrInst final
   // TODO: Add missing member functions.
 };
 
+class BinaryOperator : public SingleLLVMInstructionImpl<llvm::BinaryOperator> {
+  static Opcode getBinOpOpcode(llvm::Instruction::BinaryOps BinOp) {
+    switch (BinOp) {
+    case llvm::Instruction::Add:
+      return Opcode::Add;
+    case llvm::Instruction::FAdd:
+      return Opcode::FAdd;
+    case llvm::Instruction::Sub:
+      return Opcode::Sub;
+    case llvm::Instruction::FSub:
+      return Opcode::FSub;
+    case llvm::Instruction::Mul:
+      return Opcode::Mul;
+    case llvm::Instruction::FMul:
+      return Opcode::FMul;
+    case llvm::Instruction::UDiv:
+      return Opcode::UDiv;
+    case llvm::Instruction::SDiv:
+      return Opcode::SDiv;
+    case llvm::Instruction::FDiv:
+      return Opcode::FDiv;
+    case llvm::Instruction::URem:
+      return Opcode::URem;
+    case llvm::Instruction::SRem:
+      return Opcode::SRem;
+    case llvm::Instruction::FRem:
+      return Opcode::FRem;
+    case llvm::Instruction::Shl:
+      return Opcode::Shl;
+    case llvm::Instruction::LShr:
+      return Opcode::LShr;
+    case llvm::Instruction::AShr:
+      return Opcode::AShr;
+    case llvm::Instruction::And:
+      return Opcode::And;
+    case llvm::Instruction::Or:
+      return Opcode::Or;
+    case llvm::Instruction::Xor:
+      return Opcode::Xor;
+    case llvm::Instruction::BinaryOpsEnd:
+      llvm_unreachable("Bad BinOp!");
+    }
+    llvm_unreachable("Unhandled BinOp!");
+  }
+  BinaryOperator(llvm::BinaryOperator *BinOp, Context &Ctx)
+      : SingleLLVMInstructionImpl(ClassID::BinaryOperator,
+                                  getBinOpOpcode(BinOp->getOpcode()), BinOp,
+                                  Ctx) {}
+  friend class Context; // For constructor.
+
+public:
+  static Value *create(Instruction::Opcode Op, Value *LHS, Value *RHS,
+                       BBIterator WhereIt, BasicBlock *WhereBB, Context &Ctx,
+                       const Twine &Name = "");
+  static Value *create(Instruction::Opcode Op, Value *LHS, Value *RHS,
+                       Instruction *InsertBefore, Context &Ctx,
+                       const Twine &Name = "");
+  static Value *create(Instruction::Opcode Op, Value *LHS, Value *RHS,
+                       BasicBlock *InsertAtEnd, Context &Ctx,
+                       const Twine &Name = "");
+
+  static Value *createWithCopiedFlags(Instruction::Opcode Op, Value *LHS,
+                                      Value *RHS, Value *CopyFrom,
+                                      BBIterator WhereIt, BasicBlock *WhereBB,
+                                      Context &Ctx, const Twine &Name = "");
+  static Value *createWithCopiedFlags(Instruction::Opcode Op, Value *LHS,
+                                      Value *RHS, Value *CopyFrom,
+                                      Instruction *InsertBefore, Context &Ctx,
+                                      const Twine &Name = "");
+  static Value *createWithCopiedFlags(Instruction::Opcode Op, Value *LHS,
+                                      Value *RHS, Value *CopyFrom,
+                                      BasicBlock *InsertAtEnd, Context &Ctx,
+                                      const Twine &Name = "");
+  /// For isa/dyn_cast.
+  static bool classof(const Value *From) {
+    return From->getSubclassID() == ClassID::BinaryOperator;
+  }
+  void swapOperands() { swapOperandsInternal(0, 1); }
+};
+
 class AtomicCmpXchgInst
     : public SingleLLVMInstructionImpl<llvm::AtomicCmpXchgInst> {
   AtomicCmpXchgInst(llvm::AtomicCmpXchgInst *Atomic, Context &Ctx)
@@ -1876,6 +1959,8 @@ class Context {
   friend CallBrInst; // For createCallBrInst()
   GetElementPtrInst *createGetElementPtrInst(llvm::GetElementPtrInst *I);
   friend GetElementPtrInst; // For createGetElementPtrInst()
+  BinaryOperator *createBinaryOperator(llvm::BinaryOperator *I);
+  friend BinaryOperator; // For createBinaryOperator()
   AtomicCmpXchgInst *createAtomicCmpXchgInst(llvm::AtomicCmpXchgInst *I);
   friend AtomicCmpXchgInst; // For createAtomicCmpXchgInst()
   AllocaInst *createAllocaInst(llvm::AllocaInst *I);
diff --git a/llvm/include/llvm/SandboxIR/SandboxIRValues.def b/llvm/include/llvm/SandboxIR/SandboxIRValues.def
index 114dc8505ecd6f9..43441ba26ec2cbe 100644
--- a/llvm/include/llvm/SandboxIR/SandboxIRValues.def
+++ b/llvm/include/llvm/SandboxIR/SandboxIRValues.def
@@ -32,19 +32,39 @@ DEF_USER(Constant, Constant)
 #define OPCODES(...)
 #endif
 // clang-format off
-//        ClassID,        Opcode(s),          Class
-DEF_INSTR(Opaque,         OP(Opaque),         OpaqueInst)
+//       ClassID,        Opcode(s),         Class
+DEF_INSTR(Opaque,        OP(Opaque),        OpaqueInst)
 DEF_INSTR(ExtractElement, OP(ExtractElement), ExtractElementInst)
-DEF_INSTR(InsertElement,  OP(InsertElement),  InsertElementInst)
-DEF_INSTR(Select,         OP(Select),         SelectInst)
-DEF_INSTR(Br,             OP(Br),             BranchInst)
-DEF_INSTR(Load,           OP(Load),           LoadInst)
-DEF_INSTR(Store,          OP(Store),          StoreInst)
-DEF_INSTR(Ret,            OP(Ret),            ReturnInst)
-DEF_INSTR(Call,           OP(Call),           CallInst)
-DEF_INSTR(Invoke,         OP(Invoke),         InvokeInst)
-DEF_INSTR(CallBr,         OP(CallBr),         CallBrInst)
-DEF_INSTR(GetElementPtr,  OP(GetElementPtr),  GetElementPtrInst)
+DEF_INSTR(InsertElement, OP(InsertElement), InsertElementInst)
+DEF_INSTR(Select,        OP(Select),        SelectInst)
+DEF_INSTR(Br,            OP(Br),            BranchInst)
+DEF_INSTR(Load,          OP(Load),          LoadInst)
+DEF_INSTR(Store,         OP(Store),         StoreInst)
+DEF_INSTR(Ret,           OP(Ret),           ReturnInst)
+DEF_INSTR(Call,          OP(Call),          CallInst)
+DEF_INSTR(Invoke,        OP(Invoke),        InvokeInst)
+DEF_INSTR(CallBr,        OP(CallBr),        CallBrInst)
+DEF_INSTR(GetElementPtr, OP(GetElementPtr), GetElementPtrInst)
+DEF_INSTR(BinaryOperator, OPCODES( \
+                         OP(Add)  \
+                         OP(FAdd) \
+                         OP(Sub)  \
+                         OP(FSub) \
+                         OP(Mul)  \
+                         OP(FMul) \
+                         OP(UDiv) \
+                         OP(SDiv) \
+                         OP(FDiv) \
+                         OP(URem) \
+                         OP(SRem) \
+                         OP(FRem) \
+                         OP(Shl)  \
+                         OP(LShr) \
+                         OP(AShr) \
+                         OP(And)  \
+                         OP(Or)   \
+                         OP(Xor)  \
+                         ),                 BinaryOperator)
 DEF_INSTR(AtomicCmpXchg, OP(AtomicCmpXchg), AtomicCmpXchgInst)
 DEF_INSTR(Alloca,         OP(Alloca),         AllocaInst)
 DEF_INSTR(Cast,   OPCODES(\
diff --git a/llvm/lib/SandboxIR/SandboxIR.cpp b/llvm/lib/SandboxIR/SandboxIR.cpp
index 3c7dbb70c9f4b7a..52df53892b45061 100644
--- a/llvm/lib/SandboxIR/SandboxIR.cpp
+++ b/llvm/lib/SandboxIR/SandboxIR.cpp
@@ -1219,6 +1219,107 @@ static llvm::Instruction::CastOps getLLVMCastOp(Instruction::Opcode Opc) {
   }
 }
 
+/// \Returns the LLVM opcode that corresponds to \p Opc.
+static llvm::Instruction::BinaryOps getLLVMBinaryOp(Instruction::Opcode Opc) {
+  switch (Opc) {
+  case Instruction::Opcode::Add:
+    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::Add);
+  case Instruction::Opcode::FAdd:
+    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::FAdd);
+  case Instruction::Opcode::Sub:
+    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::Sub);
+  case Instruction::Opcode::FSub:
+    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::FSub);
+  case Instruction::Opcode::Mul:
+    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::Mul);
+  case Instruction::Opcode::FMul:
+    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::FMul);
+  case Instruction::Opcode::UDiv:
+    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::UDiv);
+  case Instruction::Opcode::SDiv:
+    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::SDiv);
+  case Instruction::Opcode::FDiv:
+    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::FDiv);
+  case Instruction::Opcode::URem:
+    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::URem);
+  case Instruction::Opcode::SRem:
+    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::SRem);
+  case Instruction::Opcode::FRem:
+    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::FRem);
+  case Instruction::Opcode::Shl:
+    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::Shl);
+  case Instruction::Opcode::LShr:
+    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::LShr);
+  case Instruction::Opcode::AShr:
+    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::AShr);
+  case Instruction::Opcode::And:
+    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::And);
+  case Instruction::Opcode::Or:
+    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::Or);
+  case Instruction::Opcode::Xor:
+    return static_cast<llvm::Instruction::BinaryOps>(llvm::Instruction::Xor);
+  default:
+    llvm_unreachable("Not a binary op!");
+  }
+}
+Value *BinaryOperator::create(Instruction::Opcode Op, Value *LHS, Value *RHS,
+                              BBIterator WhereIt, BasicBlock *WhereBB,
+                              Context &Ctx, const Twine &Name) {
+  auto &Builder = Ctx.getLLVMIRBuilder();
+  if (WhereIt == WhereBB->end())
+    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
+  else
+    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
+  llvm::Value *NewV =
+      Builder.CreateBinOp(getLLVMBinaryOp(Op), LHS->Val, RHS->Val, Name);
+  if (auto *NewBinOp = dyn_cast<llvm::BinaryOperator>(NewV))
+    return Ctx.createBinaryOperator(NewBinOp);
+  assert(isa<llvm::Constant>(NewV) && "Expected constant");
+  return Ctx.getOrCreateConstant(cast<llvm::Constant>(NewV));
+}
+
+Value *BinaryOperator::create(Instruction::Opcode Op, Value *LHS, Value *RHS,
+                              Instruction *InsertBefore, Context &Ctx,
+                              const Twine &Name) {
+  return create(Op, LHS, RHS, InsertBefore->getIterator(),
+                InsertBefore->getParent(), Ctx, Name);
+}
+
+Value *BinaryOperator::create(Instruction::Opcode Op, Value *LHS, Value *RHS,
+                              BasicBlock *InsertAtEnd, Context &Ctx,
+                              const Twine &Name) {
+  return create(Op, LHS, RHS, InsertAtEnd->end(), InsertAtEnd, Ctx, Name);
+}
+
+Value *BinaryOperator::createWithCopiedFlags(Instruction::Opcode Op, Value *LHS,
+                                             Value *RHS, Value *CopyFrom,
+                                             BBIterator WhereIt,
+                                             BasicBlock *WhereBB, Context &Ctx,
+                                             const Twine &Name) {
+
+  Value *NewV = create(Op, LHS, RHS, WhereIt, WhereBB, Ctx, Name);
+  if (auto *NewBO = dyn_cast<BinaryOperator>(NewV))
+    cast<llvm::BinaryOperator>(NewBO->Val)->copyIRFlags(CopyFrom->Val);
+  return NewV;
+}
+
+Value *BinaryOperator::createWithCopiedFlags(Instruction::Opcode Op, Value *LHS,
+                                             Value *RHS, Value *CopyFrom,
+                                             Instruction *InsertBefore,
+                                             Context &Ctx, const Twine &Name) {
+  return createWithCopiedFlags(Op, LHS, RHS, CopyFrom,
+                               InsertBefore->getIterator(),
+                               InsertBefore->getParent(), Ctx, Name);
+}
+
+Value *BinaryOperator::createWithCopiedFlags(Instruction::Opcode Op, Value *LHS,
+                                             Value *RHS, Value *CopyFrom,
+                                             BasicBlock *InsertAtEnd,
+                                             Context &Ctx, const Twine &Name) {
+  return createWithCopiedFlags(Op, LHS, RHS, CopyFrom, InsertAtEnd->end(),
+                               InsertAtEnd, Ctx, Name);
+}
+
 void AtomicCmpXchgInst::setSyncScopeID(SyncScope::ID SSID) {
   Ctx.getTracker()
       .emplaceIfTracking<GenericSetter<&AtomicCmpXchgInst::getSyncScopeID,
@@ -1628,6 +1729,29 @@ Value *Context::getOrCreateValueInternal(llvm::Value *LLVMV, llvm::User *U) {
         new GetElementPtrInst(LLVMGEP, *this));
     return It->second.get();
   }
+  case llvm::Instruction::Add:
+  case llvm::Instruction::FAdd:
+  case llvm::Instruction::Sub:
+  case llvm::Instruction::FSub:
+  case llvm::Instruction::Mul:
+  case llvm::Instruction::FMul:
+  case llvm::Instruction::UDiv:
+  case llvm::Instruction::SDiv:
+  case llvm::Instruction::FDiv:
+  case llvm::Instruction::URem:
+  case llvm::Instruction::SRem:
+  case llvm::Instruction::FRem:
+  case llvm::Instruction::Shl:
+  case llvm::Instruction::LShr:
+  case llvm::Instruction::AShr:
+  case llvm::Instruction::And:
+  case llvm::Instruction::Or:
+  case llvm::Instruction::Xor: {
+    auto *LLVMBinaryOperator = cast<llvm::BinaryOperator>(LLVMV);
+    It->second = std::unique_ptr<BinaryOperator>(
+        new BinaryOperator(LLVMBinaryOperator, *this));
+    return It->second.get();
+  }
   case llvm::Instruction::AtomicCmpXchg: {
     auto *LLVMAtomicCmpXchg = cast<llvm::AtomicCmpXchgInst>(LLVMV);
     It->second = std::unique_ptr<AtomicCmpXchgInst>(
@@ -1751,6 +1875,10 @@ Context::createGetElementPtrInst(llvm::GetElementPtrInst *I) {
       std::unique_ptr<GetElementPtrInst>(new GetElementPtrInst(I, *this));
   return cast<GetElementPtrInst>(registerValue(std::move(NewPtr)));
 }
+BinaryOperator *Context::createBinaryOperator(llvm::BinaryOperator *I) {
+  auto NewPtr = std::unique_ptr<BinaryOperator>(new BinaryOperator(I, *this));
+  return cast<BinaryOperator>(registerValue(std::move(NewPtr)));
+}
 AtomicCmpXchgInst *
 Context::createAtomicCmpXchgInst(llvm::AtomicCmpXchgInst *I) {
   auto NewPtr =
diff --git a/llvm/unittests/SandboxIR/SandboxIRTest.cpp b/llvm/unittests/SandboxIR/SandboxIRTest.cpp
index 1e4679ed6e802ed..f5e555ba73287b5 100644
--- a/llvm/unittests/SandboxIR/SandboxIRTest.cpp
+++ b/llvm/unittests/SandboxIR/SandboxIRTest.cpp
@@ -54,7 +54,6 @@ define void @foo(i32 %v1) {
   sandboxir::Argument *Arg0 = F->getArg(0);
   sandboxir::BasicBlock *BB = &*F->begin();
   sandboxir::Instruction *AddI = &*BB->begin();
-  sandboxir::OpaqueInst *OpaqueI = cast<sandboxir::OpaqueInst>(AddI);
   sandboxir::Constant *Const0 = cast<sandboxir::Constant>(Ctx.getValue(LLVMC));
 
   EXPECT_TRUE(isa<sandboxir::Function>(F));
@@ -62,42 +61,36 @@ define void @foo(i32 %v1) {
   EXPECT_FALSE(isa<sandboxir::Function>(BB));
   EXPECT_FALSE(isa<sandboxir::Function>(AddI));
   EXPECT_FALSE(isa<sandboxir::Function>(Const0));
-  EXPECT_FALSE(isa<sandboxir::Function>(OpaqueI));
 
   EXPECT_FALSE(isa<sandboxir::Argument>(F));
   EXPECT_TRUE(isa<sandboxir::Argument>(Arg0));
   EXPECT_FALSE(isa<sandboxir::Argument>(BB));
   EXPECT_FALSE(isa<sandboxir::Argument>(AddI));
   EXPECT_FALSE(isa<sandboxir::Argument>(Const0));
-  EXPECT_FALSE(isa<sandboxir::Argument>(OpaqueI));
 
   EXPECT_TRUE(isa<sandboxir::Constant>(F));
   EXPECT_FALSE(isa<sandboxir::Constant>(Arg0));
   EXPECT_FALSE(isa<sandboxir::Constant>(BB));
   EXPECT_FALSE(isa<sandboxir::Constant>(AddI));
   EXPECT_TRUE(isa<sandboxir::Constant>(Const0));
-  EXPECT_FALSE(isa<sandboxir::Constant>(OpaqueI));
 
   EXPECT_FALSE(isa<sandboxir::OpaqueInst>(F));
   EXPECT_FALSE(isa<sandboxir::OpaqueInst>(Arg0));
   EXPECT_FALSE(isa<sandboxir::OpaqueInst>(BB));
-  EXPECT_TRUE(isa<sandboxir::OpaqueInst>(AddI));
+  EXPECT_FALSE(isa<sandboxir::OpaqueInst>(AddI));
   EXPECT_FALSE(isa<sandboxir::OpaqueInst>(Const0));
-  EXPECT_TRUE(isa<sandboxir::OpaqueInst>(OpaqueI));
 
   EXPECT_FALSE(isa<sandboxir::Instruction>(F));
   EXPECT_FALSE(isa<sandboxir::Instruction>(Arg0));
   EXPECT_FALSE(isa<sandboxir::Instruction>(BB));
   EXPECT_TRUE(isa<sandboxir::Instruction>(AddI));
   EXPECT_FALSE(isa<sandboxir::Instruction>(Const0));
-  EXPECT_TRUE(isa<sandboxir::Instruction>(OpaqueI));
 
   EXPECT_TRUE(isa<sandboxir::User>(F));
   EXPECT_FALSE(isa<sandboxir::User>(Arg0));
   EXPECT_FALSE(isa<sandboxir::User>(BB));
   EXPECT_TRUE(isa<sandboxir::User>(AddI));
   EXPECT_TRUE(isa<sandboxir::User>(Const0));
-  EXPECT_TRUE(isa<sandboxir::User>(OpaqueI));
 
 #ifndef NDEBUG
   std::string Buff;
@@ -107,7 +100,6 @@ define void @foo(i32 %v1) {
   BB->dumpOS(BS);
   AddI->dumpOS(BS);
   Const0->dumpOS(BS);
-  OpaqueI->dumpOS(BS);
 #endif
 }
 
@@ -183,7 +175,7 @@ define i32 @foo(i32 %v0, i32 %v1) {
   I0->getOperandUse(0).dumpOS(BS);
   EXPECT_EQ(Buff, R"IR(
 Def:  i32 %v0 ; SB2. (Argument)
-User:   %add0 = add i32 %v0, %v1 ; SB5. (Opaque)
+User:   %add0 = add i32 %v0, %v1 ; SB5. (BinaryOperator)
 OperandNo: 0
 )IR");
 #endif // NDEBUG
@@ -508,8 +500,8 @@ define void @foo(i8 %v1) {
   EXPECT_EQ(Ret->getIterator(), std::next(BB->begin(), 2));
 
   // Check getOpcode().
-  EXPECT_EQ(I0->getOpcode(), sandboxir::Instruction::Opcode::Opaque);
-  EXPECT_EQ(I1->getOpcode(), sandboxir::Instruction::Opcode::Opaque);
+  EXPECT_EQ(I0->getOpcode(), sandboxir::Instruction::Opcode::Add);
+  EXPECT_EQ(I1->getOpcode(), sandboxir::Instruction::Opcode::Sub);
   EXPECT_EQ(Ret->getOpcode(), sandboxir::Instruction::Opcode::Ret);
 
   // Check moveBefore(I).
@@ -1628,6 +1620,192 @@ define void @foo(i32 %arg, float %farg) {
   EXPECT_FALSE(FAdd->getFastMathFlags() != LLVMFAdd->getFastMathFlags());
 }
 
+TEST_F(SandboxIRTest, BinaryOperator) {
+  parseIR(C, R"IR(
+define void @foo(i8 %arg0, i8 %arg1, float %farg0, float %farg1) {
+  %add = add i8 %arg0, %arg1
+  %fadd = fadd float %farg0, %farg1
+  %sub = sub i8 %arg0, %arg1
+  %fsub = fsub float %farg0, %farg1
+  %mul = mul i8 %arg0, %arg1
+  %fmul = fmul float %farg0, %farg1
+  %udiv = udiv i8 %arg0, %arg1
+  %sdiv = sdiv i8 %arg0, %arg1
+  %fdiv = fdiv float %farg0, %farg1
+  %urem = urem i8 %arg0, %arg1
+  %srem = srem i8 %arg0, %arg1
+  %frem = frem float %farg0, %farg1
+  %shl = shl i8 %arg0, %arg1
+  %lshr = lshr i8 %arg0, %arg1
+  %ashr = ashr i8 %arg0, %arg1
+  %and = and i8 %arg0, %arg1
+  %or = or i8 %arg0, %arg1
+  %xor = xor i8 %arg0, %arg1
+
+  %copyfrom = add nsw i8 %arg0, %arg1
+  ret void
+}
+)IR");
+  Function &LLVMF = *M->getFunction("foo");
+  sandboxir::Context Ctx(C);
+
+  auto &F = *Ctx.createFunction(&LLVMF);
+  auto *Arg0 = F.getArg(0);
+  auto *Arg1 = F.getArg(1);
+  auto *FArg0 = F.getArg(2);
+  auto *FArg1 = F.getArg(3);
+  auto *BB = &*F.begin();
+  auto It = BB->begin();
+
+#define CHECK_IBINOP(OPCODE)                                                   \
+  {                                                                            \
+    auto *I = cast<sandboxir::BinaryOperator>(&*It++);                         \
+    EXPECT_EQ(I->getOpcode(), OPCODE);                                         \
+    EXPECT_EQ(I->getOperand(0), Arg0);                                         \
+    EXPECT_EQ(I->getOperand(1), Arg1);                                         \
+  }
+#define CHECK_FBINOP(OPCODE)                                                   \
+  {                                                                            \
+    auto *I = cast<sandboxir::BinaryOperator>(&*It++);                         \
+    EXPECT_EQ(I->getOpcode(), OPCODE);                                         \
+    EXPECT_EQ(I->getOperand(0), FArg0);                                        \
+    EXPECT_EQ(I->getOperand(1), FArg1);                                        \
+  }
+
+  CHECK_IBINOP(sandboxir::Instruction::Opcode::Add);
+  CHECK_FBINOP(sandboxir::Instruction::Opcode::FAdd);
+  CHECK_IBINOP(sandboxir::Instruction::Opcode::Sub);
+  CHECK_FBINOP(sandboxir::Instruction::Opcode::FSub);
+  CHECK_IBINOP(sandboxir::Instruction::Opcode::Mul);
+  CHECK_FBINOP(sandboxir::Instruction::Opcode::FMul);
+  CHECK_IBINOP(sandboxir::Instruction::Opcode::UDiv);
+  CHECK_IBINOP(sandboxir::Instruction::Opcode::SDiv);
+  CHECK_FBINOP(sandboxir::Instruction::Opcode::FDiv);
+  CHECK_IBINOP(sandboxir::Instruction::Opcode::URem);
+  CHECK_IBINOP(sandboxir::Instruction::Opcode::SRem);
+  CHECK_FBINOP(sandboxir::Instruction::Opcode::FRem);
+  CHECK_IBINOP(sandboxir::Instruction::Opcode::Shl);
+  CHECK_IBINOP(sandboxir::Instruction::Opcode::LShr);
+  CHECK_IBINOP(sandboxir::Instruction::Opcode::AShr);
+  CHECK_IBINOP(sandboxir::Instruction::Opcode::And);
+  CHECK_IBINOP(sandboxir::Instruction::Opcode::Or);
+  CHECK_IBINOP(sandboxir::Instruction::Opcode::Xor);
+
+  auto *CopyFrom = cast<sandboxir::BinaryOperator>(&*It++);
+  auto *Ret = cast<sandboxir::ReturnInst>(&*It++);
+
+  {
+    // Check create() WhereIt, WhereBB.
+    auto *NewI =
+        cast<sandboxir::BinaryOperator>(sandboxir::BinaryOperator::create(
+            sandboxir::Instruction::Opcode::Add, Arg0, Arg1,
+            /*WhereIt=*/Ret->getIterator(), /*WhereBB=*/Ret->getParent(), Ctx,
+            "New1"));
+    EXPECT_EQ(NewI->getOpcode(), sandboxir::Instruction::Opcode::Add);
+    EXPECT_EQ(NewI->getOperand(0), Arg0);
+    EXPECT_EQ(NewI->getOperand(1), Arg1);
+#ifndef NDEBUG
+    EXPECT_EQ(NewI->getName(), "New1");
+#endif // NDEBUG
+    EXPECT_EQ(NewI->getNextNode(), Ret);
+  }
+  {
+    // Check create() InsertBefore.
+    auto *NewI =
+        cast<sandboxir::BinaryOperator>(sandboxir::BinaryOperator::create(
+            sandboxir::Instruction::Opcode::Add, Arg0, Arg1,
+            /*InsertBefore=*/Ret, Ctx, "New2"));
+    EXPECT_EQ(NewI->getOpcode(), sandboxir::Instruction::Opcode::Add);
+    EXPECT_EQ(NewI->getOperand(0), Arg0);
+    EXPECT_EQ(NewI->getOperand(1), Arg1);
+#ifndef NDEBUG
+    EXPECT_EQ(NewI->getName(), "New2");
+#endif // NDEBUG
+    EXPECT_EQ(NewI->getNextNode(), Ret);
+  }
+  {
+    // Check create() InsertAtEnd.
+    auto *NewI =
+        cast<sandboxir::BinaryOperator>(sandboxir::BinaryOperator::create(
+            sandboxir::Instruction::Opcode::Add, Arg0, Arg1,
+            /*InsertAtEnd=*/BB, Ctx, "New3"));
+    EXPECT_EQ(NewI->getOpcode(), sandboxir::Instruction::Opcode::Add);
+    EXPECT_EQ(NewI->getOperand(0), Arg0);
+    EXPECT_EQ(NewI->getOperand(1), Arg1);
+#ifndef NDEBUG
+    EXPECT_EQ(NewI->getName(), "New3");
+#endif // NDEBUG
+    EXPECT_EQ(NewI->getNextNode(), nullptr);
+    EXPECT_EQ(NewI->getParent(), BB);
+  }
+  {
+    // Check create() when it gets folded.
+    auto *FortyTwo =
+        sandboxir::Constant::createInt(Type::getInt32Ty(C), 42, Ctx);
+    auto *NewV = sandboxir::BinaryOperator::create(
+        sandboxir::Instruction::Opcode::Add, FortyTwo, FortyTwo,
+        /*InsertBefore=*/Ret, Ctx, "Folded");
+    EXPECT_TRUE(isa<sandboxir::Constant>(NewV));
+  }
+
+  {
+    // Check createWithCopiedFlags() WhereIt, WhereBB.
+    auto *NewI = cast<sandboxir::BinaryOperator>(
+        sandboxir::BinaryOperator::createWithCopiedFlags(
+            sandboxir::Instruction::Opcode::Add, Arg0, Arg1, CopyFrom,
+            /*WhereIt=*/Ret->getIterator(), /*WhereBB=*/Ret->getParent(), Ctx,
+            "NewNSW1"));
+    EXPECT_EQ(NewI->hasNoSignedWrap(), CopyFrom->hasNoSignedWrap());
+    EXPECT_EQ(NewI->getOpcode(), sandboxir::Instruction::Opcode::Add);
+    EXPECT_EQ(NewI->getOperand(0), Arg0);
+    EXPECT_EQ(NewI->getOperand(1), Arg1);
+#ifndef NDEBUG
+    EXPECT_EQ(NewI->getName(), "NewNSW1");
+#endif // NDEBUG
+    EXPECT_EQ(NewI->getNextNode(), Ret);
+  }
+  {
+    // Check createWithCopiedFlags() InsertBefore.
+    auto *NewI = cast<sandboxir::BinaryOperator>(
+        sandboxir::BinaryOperator::createWithCopiedFlags(
+            sandboxir::Instruction::Opcode::Add, Arg0, Arg1, CopyFrom,
+            /*InsertBefore=*/Ret, Ctx, "NewNSW2"));
+    EXPECT_EQ(NewI->hasNoSignedWrap(), CopyFrom->hasNoSignedWrap());
+    EXPECT_EQ(NewI->getOpcode(), sandboxir::Instruction::Opcode::Add);
+    EXPECT_EQ(NewI->getOperand(0), Arg0);
+    EXPECT_EQ(NewI->getOperand(1), Arg1);
+#ifndef NDEBUG
+    EXPECT_EQ(NewI->getName(), "NewNSW2");
+#endif // NDEBUG
+    EXPECT_EQ(NewI->getNextNode(), Ret);
+  }
+  {
+    // Check createWithCopiedFlags() InsertAtEnd.
+    auto *NewI = cast<sandboxir::BinaryOperator>(
+        sandboxir::BinaryOperator::createWithCopiedFlags(
+            sandboxir::Instruction::Opcode::Add, Arg0, Arg1, CopyFrom,
+            /*InsertAtEnd=*/BB, Ctx, "NewNSW3"));
+    EXPECT_EQ(NewI->hasNoSignedWrap(), CopyFrom->hasNoSignedWrap());
+    EXPECT_EQ(NewI->getOpcode(), sandboxir::Instruction::Opcode::Add);
+    EXPECT_EQ(NewI->getOperand(0), Arg0);
+    EXPECT_EQ(NewI->getOperand(1), Arg1);
+#ifndef NDEBUG
+    EXPECT_EQ(NewI->getName(), "NewNSW3");
+#endif // NDEBUG
+    EXPECT_EQ(NewI->getParent(), BB);
+    EXPECT_EQ(NewI->getNextNode(), nullptr);
+  }
+  {
+    // Check createWithCopiedFlags() when it gets folded.
+    auto *FortyTwo =
+        sandboxir::Constant::createInt(Type::getInt32Ty(C), 42, Ctx);
+    auto *NewV = sandboxir::BinaryOperator::createWithCopiedFlags(
+        sandboxir::Instruction::Opcode::Add, FortyTwo, FortyTwo, CopyFrom,
+        /*InsertBefore=*/Ret, Ctx, "Folded");
+    EXPECT_TRUE(isa<sandboxir::Constant>(NewV));
+  }
+}
+
 TEST_F(SandboxIRTest, AtomicCmpXchgInst) {
   parseIR(C, R"IR(
 define void @foo(ptr %ptr, i8 %cmp, i8 %new) {

From c19326cb84e871e1dd77408c2911c7d1651ba34b Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Thu, 15 Aug 2024 13:29:47 -0700
Subject: [PATCH 079/441] [MC] Replace HasAggressiveSymbolFolding with
 SetDirectiveSuppressesReloc. NFC

The variable and its comment could lead to confusion (AMDGPU
unnecessarily set it).
---
 llvm/include/llvm/MC/MCAsmInfo.h                | 17 -----------------
 llvm/lib/MC/MCAsmInfoDarwin.cpp                 |  3 ---
 llvm/lib/MC/MCDwarf.cpp                         |  3 ++-
 .../AMDGPU/MCTargetDesc/AMDGPUMCAsmInfo.cpp     |  1 -
 4 files changed, 2 insertions(+), 22 deletions(-)

diff --git a/llvm/include/llvm/MC/MCAsmInfo.h b/llvm/include/llvm/MC/MCAsmInfo.h
index add60ef7006b85a..7c4f3b63756eaea 100644
--- a/llvm/include/llvm/MC/MCAsmInfo.h
+++ b/llvm/include/llvm/MC/MCAsmInfo.h
@@ -348,21 +348,6 @@ class MCAsmInfo {
   ///   .long a
   bool SetDirectiveSuppressesReloc = false;
 
-  /// False if the assembler requires that we use
-  /// \code
-  ///   Lc = a - b
-  ///   .long Lc
-  /// \endcode
-  //
-  /// instead of
-  //
-  /// \code
-  ///   .long a - b
-  /// \endcode
-  ///
-  ///  Defaults to true.
-  bool HasAggressiveSymbolFolding = true;
-
   /// True is .comm's and .lcomms optional alignment is to be specified in bytes
   /// instead of log2(n).  Defaults to true.
   bool COMMDirectiveAlignmentIsInBytes = true;
@@ -722,8 +707,6 @@ class MCAsmInfo {
     return SetDirectiveSuppressesReloc;
   }
 
-  bool hasAggressiveSymbolFolding() const { return HasAggressiveSymbolFolding; }
-
   bool getCOMMDirectiveAlignmentIsInBytes() const {
     return COMMDirectiveAlignmentIsInBytes;
   }
diff --git a/llvm/lib/MC/MCAsmInfoDarwin.cpp b/llvm/lib/MC/MCAsmInfoDarwin.cpp
index a10be51ee61ac07..e218a42889de909 100644
--- a/llvm/lib/MC/MCAsmInfoDarwin.cpp
+++ b/llvm/lib/MC/MCAsmInfoDarwin.cpp
@@ -76,9 +76,6 @@ MCAsmInfoDarwin::MCAsmInfoDarwin() {
   HasMachoZeroFillDirective = true;  // Uses .zerofill
   HasMachoTBSSDirective = true; // Uses .tbss
 
-  // FIXME: Change this once MC is the system assembler.
-  HasAggressiveSymbolFolding = false;
-
   HiddenVisibilityAttr = MCSA_PrivateExtern;
   HiddenDeclarationVisibilityAttr = MCSA_Invalid;
 
diff --git a/llvm/lib/MC/MCDwarf.cpp b/llvm/lib/MC/MCDwarf.cpp
index 1297dc3828b581b..2db59f8fc944264 100644
--- a/llvm/lib/MC/MCDwarf.cpp
+++ b/llvm/lib/MC/MCDwarf.cpp
@@ -320,9 +320,10 @@ MCDwarfLineTableHeader::Emit(MCStreamer *MCOS, MCDwarfLineTableParams Params,
 static const MCExpr *forceExpAbs(MCStreamer &OS, const MCExpr* Expr) {
   MCContext &Context = OS.getContext();
   assert(!isa<MCSymbolRefExpr>(Expr));
-  if (Context.getAsmInfo()->hasAggressiveSymbolFolding())
+  if (!Context.getAsmInfo()->doesSetDirectiveSuppressReloc())
     return Expr;
 
+  // On Mach-O, try to avoid a relocation by using a set directive.
   MCSymbol *ABS = Context.createTempSymbol();
   OS.emitAssignment(ABS, Expr);
   return MCSymbolRefExpr::create(ABS, Context);
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCAsmInfo.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCAsmInfo.cpp
index 201cc8d01e2d4b4..3c9f6d2938075b8 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCAsmInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCAsmInfo.cpp
@@ -34,7 +34,6 @@ AMDGPUMCAsmInfo::AMDGPUMCAsmInfo(const Triple &TT,
   UsesELFSectionDirectiveForBSS = true;
 
   //===--- Global Variable Emission Directives --------------------------===//
-  HasAggressiveSymbolFolding = true;
   COMMDirectiveAlignmentIsInBytes = false;
   HasNoDeadStrip = true;
   //===--- Dwarf Emission Directives -----------------------------------===//

From 99a10f1fe8a7e4b0fdb4c6dd5e7f24f87e0d3695 Mon Sep 17 00:00:00 2001
From: Kevin McAfee <kmcafee@nvidia.com>
Date: Thu, 15 Aug 2024 13:34:49 -0700
Subject: [PATCH 080/441] Update load intrinsic attributes (#101562)

This patch adds default attributes to many intrinsics and the WillReturn
attribute to some as well. The defaults include WillReturn. The WillReturn
attribute is relevant for dead code elimination as intrinsics without
WillReturn are assumed to have side effects and cannot be removed even
if their return value is unused. It is also relevant for potential
changes to SDAG behavior regarding treatment of intrinsics that function
as loads.
---
 llvm/include/llvm/IR/IntrinsicsBPF.td         |   6 +-
 llvm/include/llvm/IR/IntrinsicsLoongArch.td   |  29 +-
 llvm/include/llvm/IR/IntrinsicsMips.td        |  26 +-
 llvm/include/llvm/IR/IntrinsicsNVVM.td        |   2 +-
 llvm/include/llvm/IR/IntrinsicsSystemZ.td     |   6 +-
 llvm/include/llvm/IR/IntrinsicsVE.td          |   4 +-
 llvm/include/llvm/IR/IntrinsicsVEVL.gen.td    | 128 +--
 llvm/include/llvm/IR/IntrinsicsWebAssembly.td |   2 +-
 llvm/test/CodeGen/BPF/sockex2.ll              |   2 +-
 llvm/test/Transforms/DCE/intrinsics-bpf.ll    |  33 +
 .../Transforms/DCE/intrinsics-loongarch.ll    | 124 +++
 llvm/test/Transforms/DCE/intrinsics-mips.ll   | 131 +++
 llvm/test/Transforms/DCE/intrinsics-nvvm.ll   | 759 ++++++++++++++++++
 .../test/Transforms/DCE/intrinsics-systemz.ll |  33 +
 llvm/test/Transforms/DCE/intrinsics-ve.ll     | 729 +++++++++++++++++
 llvm/test/Transforms/DCE/intrinsics-wasm.ll   |  13 +
 .../Transforms/DCE/nvvm-ldu-ldg-willreturn.ll | 187 -----
 17 files changed, 1927 insertions(+), 287 deletions(-)
 create mode 100644 llvm/test/Transforms/DCE/intrinsics-bpf.ll
 create mode 100644 llvm/test/Transforms/DCE/intrinsics-loongarch.ll
 create mode 100644 llvm/test/Transforms/DCE/intrinsics-mips.ll
 create mode 100644 llvm/test/Transforms/DCE/intrinsics-nvvm.ll
 create mode 100644 llvm/test/Transforms/DCE/intrinsics-systemz.ll
 create mode 100644 llvm/test/Transforms/DCE/intrinsics-ve.ll
 create mode 100644 llvm/test/Transforms/DCE/intrinsics-wasm.ll
 delete mode 100644 llvm/test/Transforms/DCE/nvvm-ldu-ldg-willreturn.ll

diff --git a/llvm/include/llvm/IR/IntrinsicsBPF.td b/llvm/include/llvm/IR/IntrinsicsBPF.td
index c7ec0916f1d1f8f..d02eaa6d0dff691 100644
--- a/llvm/include/llvm/IR/IntrinsicsBPF.td
+++ b/llvm/include/llvm/IR/IntrinsicsBPF.td
@@ -13,11 +13,11 @@
 // Specialized loads from packet
 let TargetPrefix = "bpf" in {  // All intrinsics start with "llvm.bpf."
   def int_bpf_load_byte : ClangBuiltin<"__builtin_bpf_load_byte">,
-              Intrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_i64_ty], [IntrReadMem]>;
+              DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_i64_ty], [IntrReadMem]>;
   def int_bpf_load_half : ClangBuiltin<"__builtin_bpf_load_half">,
-              Intrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_i64_ty], [IntrReadMem]>;
+              DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_i64_ty], [IntrReadMem]>;
   def int_bpf_load_word : ClangBuiltin<"__builtin_bpf_load_word">,
-              Intrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_i64_ty], [IntrReadMem]>;
+              DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_i64_ty], [IntrReadMem]>;
   def int_bpf_pseudo : ClangBuiltin<"__builtin_bpf_pseudo">,
               Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i64_ty]>;
   def int_bpf_preserve_field_info : ClangBuiltin<"__builtin_bpf_preserve_field_info">,
diff --git a/llvm/include/llvm/IR/IntrinsicsLoongArch.td b/llvm/include/llvm/IR/IntrinsicsLoongArch.td
index 9002076e7aecea6..4621f1689b46e04 100644
--- a/llvm/include/llvm/IR/IntrinsicsLoongArch.td
+++ b/llvm/include/llvm/IR/IntrinsicsLoongArch.td
@@ -140,6 +140,11 @@ class VecInt<list<LLVMType> ret_types, list<LLVMType> param_types,
     : Intrinsic<ret_types, param_types, intr_properties>,
       ClangBuiltin<!subst("int_loongarch", "__builtin", NAME)>;
 
+class DefaultAttrsVecInt<list<LLVMType> ret_types, list<LLVMType> param_types,
+             list<IntrinsicProperty> intr_properties = []>
+    : DefaultAttrsIntrinsic<ret_types, param_types, intr_properties>,
+      ClangBuiltin<!subst("int_loongarch", "__builtin", NAME)>;
+
 //===----------------------------------------------------------------------===//
 // LSX
 
@@ -620,22 +625,22 @@ foreach inst = ["vfcmp_caf_d", "vfcmp_cun_d", "vfcmp_ceq_d", "vfcmp_cueq_d",
 
 // LSX load/store
 def int_loongarch_lsx_vld
-  : VecInt<[llvm_v16i8_ty], [llvm_ptr_ty, llvm_i32_ty],
+  : DefaultAttrsVecInt<[llvm_v16i8_ty], [llvm_ptr_ty, llvm_i32_ty],
            [IntrReadMem, IntrArgMemOnly, ImmArg<ArgIndex<1>>]>;
 def int_loongarch_lsx_vldx
-  : VecInt<[llvm_v16i8_ty], [llvm_ptr_ty, llvm_i64_ty],
+  : DefaultAttrsVecInt<[llvm_v16i8_ty], [llvm_ptr_ty, llvm_i64_ty],
            [IntrReadMem, IntrArgMemOnly]>;
 def int_loongarch_lsx_vldrepl_b
-  : VecInt<[llvm_v16i8_ty], [llvm_ptr_ty, llvm_i32_ty],
+  : DefaultAttrsVecInt<[llvm_v16i8_ty], [llvm_ptr_ty, llvm_i32_ty],
            [IntrReadMem, IntrArgMemOnly, ImmArg<ArgIndex<1>>]>;
 def int_loongarch_lsx_vldrepl_h
-  : VecInt<[llvm_v8i16_ty], [llvm_ptr_ty, llvm_i32_ty],
+  : DefaultAttrsVecInt<[llvm_v8i16_ty], [llvm_ptr_ty, llvm_i32_ty],
            [IntrReadMem, IntrArgMemOnly, ImmArg<ArgIndex<1>>]>;
 def int_loongarch_lsx_vldrepl_w
-  : VecInt<[llvm_v4i32_ty], [llvm_ptr_ty, llvm_i32_ty],
+  : DefaultAttrsVecInt<[llvm_v4i32_ty], [llvm_ptr_ty, llvm_i32_ty],
            [IntrReadMem, IntrArgMemOnly, ImmArg<ArgIndex<1>>]>;
 def int_loongarch_lsx_vldrepl_d
-  : VecInt<[llvm_v2i64_ty], [llvm_ptr_ty, llvm_i32_ty],
+  : DefaultAttrsVecInt<[llvm_v2i64_ty], [llvm_ptr_ty, llvm_i32_ty],
            [IntrReadMem, IntrArgMemOnly, ImmArg<ArgIndex<1>>]>;
 
 def int_loongarch_lsx_vst
@@ -1146,22 +1151,22 @@ def int_loongarch_lasx_xvpickve_d_f
 
 // LASX load/store
 def int_loongarch_lasx_xvld
-  : VecInt<[llvm_v32i8_ty], [llvm_ptr_ty, llvm_i32_ty],
+  : DefaultAttrsVecInt<[llvm_v32i8_ty], [llvm_ptr_ty, llvm_i32_ty],
            [IntrReadMem, IntrArgMemOnly, ImmArg<ArgIndex<1>>]>;
 def int_loongarch_lasx_xvldx
-  : VecInt<[llvm_v32i8_ty], [llvm_ptr_ty, llvm_i64_ty],
+  : DefaultAttrsVecInt<[llvm_v32i8_ty], [llvm_ptr_ty, llvm_i64_ty],
            [IntrReadMem, IntrArgMemOnly]>;
 def int_loongarch_lasx_xvldrepl_b
-  : VecInt<[llvm_v32i8_ty], [llvm_ptr_ty, llvm_i32_ty],
+  : DefaultAttrsVecInt<[llvm_v32i8_ty], [llvm_ptr_ty, llvm_i32_ty],
            [IntrReadMem, IntrArgMemOnly, ImmArg<ArgIndex<1>>]>;
 def int_loongarch_lasx_xvldrepl_h
-  : VecInt<[llvm_v16i16_ty], [llvm_ptr_ty, llvm_i32_ty],
+  : DefaultAttrsVecInt<[llvm_v16i16_ty], [llvm_ptr_ty, llvm_i32_ty],
            [IntrReadMem, IntrArgMemOnly, ImmArg<ArgIndex<1>>]>;
 def int_loongarch_lasx_xvldrepl_w
-  : VecInt<[llvm_v8i32_ty], [llvm_ptr_ty, llvm_i32_ty],
+  : DefaultAttrsVecInt<[llvm_v8i32_ty], [llvm_ptr_ty, llvm_i32_ty],
            [IntrReadMem, IntrArgMemOnly, ImmArg<ArgIndex<1>>]>;
 def int_loongarch_lasx_xvldrepl_d
-  : VecInt<[llvm_v4i64_ty], [llvm_ptr_ty, llvm_i32_ty],
+  : DefaultAttrsVecInt<[llvm_v4i64_ty], [llvm_ptr_ty, llvm_i32_ty],
            [IntrReadMem, IntrArgMemOnly, ImmArg<ArgIndex<1>>]>;
 
 def int_loongarch_lasx_xvst
diff --git a/llvm/include/llvm/IR/IntrinsicsMips.td b/llvm/include/llvm/IR/IntrinsicsMips.td
index 3056f37b9d87714..8d12786d52e126a 100644
--- a/llvm/include/llvm/IR/IntrinsicsMips.td
+++ b/llvm/include/llvm/IR/IntrinsicsMips.td
@@ -236,7 +236,7 @@ def int_mips_extpdp: ClangBuiltin<"__builtin_mips_extpdp">,
 def int_mips_wrdsp: ClangBuiltin<"__builtin_mips_wrdsp">,
   Intrinsic<[], [llvm_i32_ty, llvm_i32_ty], [ImmArg<ArgIndex<1>>]>;
 def int_mips_rddsp: ClangBuiltin<"__builtin_mips_rddsp">,
-  Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrReadMem, ImmArg<ArgIndex<0>>]>;
+  DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrReadMem, ImmArg<ArgIndex<0>>]>;
 
 def int_mips_insv: ClangBuiltin<"__builtin_mips_insv">,
   Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrReadMem]>;
@@ -252,22 +252,22 @@ def int_mips_repl_ph: ClangBuiltin<"__builtin_mips_repl_ph">,
   Intrinsic<[mips_v2q15_ty], [llvm_i32_ty], [IntrNoMem]>;
 
 def int_mips_pick_qb: ClangBuiltin<"__builtin_mips_pick_qb">,
-  Intrinsic<[llvm_v4i8_ty], [llvm_v4i8_ty, llvm_v4i8_ty], [IntrReadMem]>;
+  DefaultAttrsIntrinsic<[llvm_v4i8_ty], [llvm_v4i8_ty, llvm_v4i8_ty], [IntrReadMem]>;
 def int_mips_pick_ph: ClangBuiltin<"__builtin_mips_pick_ph">,
-  Intrinsic<[mips_v2q15_ty], [mips_v2q15_ty, mips_v2q15_ty], [IntrReadMem]>;
+  DefaultAttrsIntrinsic<[mips_v2q15_ty], [mips_v2q15_ty, mips_v2q15_ty], [IntrReadMem]>;
 
 def int_mips_mthlip: ClangBuiltin<"__builtin_mips_mthlip">,
   Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i32_ty], []>;
 
 def int_mips_bposge32: ClangBuiltin<"__builtin_mips_bposge32">,
-  Intrinsic<[llvm_i32_ty], [], [IntrReadMem]>;
+  DefaultAttrsIntrinsic<[llvm_i32_ty], [], [IntrReadMem]>;
 
 def int_mips_lbux: ClangBuiltin<"__builtin_mips_lbux">,
-  Intrinsic<[llvm_i32_ty], [llvm_ptr_ty, llvm_i32_ty], [IntrReadMem, IntrArgMemOnly]>;
+  DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_ptr_ty, llvm_i32_ty], [IntrReadMem, IntrArgMemOnly]>;
 def int_mips_lhx: ClangBuiltin<"__builtin_mips_lhx">,
-  Intrinsic<[llvm_i32_ty], [llvm_ptr_ty, llvm_i32_ty], [IntrReadMem, IntrArgMemOnly]>;
+  DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_ptr_ty, llvm_i32_ty], [IntrReadMem, IntrArgMemOnly]>;
 def int_mips_lwx: ClangBuiltin<"__builtin_mips_lwx">,
-  Intrinsic<[llvm_i32_ty], [llvm_ptr_ty, llvm_i32_ty], [IntrReadMem, IntrArgMemOnly]>;
+  DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_ptr_ty, llvm_i32_ty], [IntrReadMem, IntrArgMemOnly]>;
 
 //===----------------------------------------------------------------------===//
 // MIPS DSP Rev 2
@@ -1259,23 +1259,23 @@ def int_mips_insve_d : ClangBuiltin<"__builtin_msa_insve_d">,
             [IntrNoMem, ImmArg<ArgIndex<1>>]>;
 
 def int_mips_ld_b : ClangBuiltin<"__builtin_msa_ld_b">,
-  Intrinsic<[llvm_v16i8_ty], [llvm_ptr_ty, llvm_i32_ty],
+  DefaultAttrsIntrinsic<[llvm_v16i8_ty], [llvm_ptr_ty, llvm_i32_ty],
   [IntrReadMem, IntrArgMemOnly]>;
 def int_mips_ld_h : ClangBuiltin<"__builtin_msa_ld_h">,
-  Intrinsic<[llvm_v8i16_ty], [llvm_ptr_ty, llvm_i32_ty],
+  DefaultAttrsIntrinsic<[llvm_v8i16_ty], [llvm_ptr_ty, llvm_i32_ty],
   [IntrReadMem, IntrArgMemOnly]>;
 def int_mips_ld_w : ClangBuiltin<"__builtin_msa_ld_w">,
-  Intrinsic<[llvm_v4i32_ty], [llvm_ptr_ty, llvm_i32_ty],
+  DefaultAttrsIntrinsic<[llvm_v4i32_ty], [llvm_ptr_ty, llvm_i32_ty],
   [IntrReadMem, IntrArgMemOnly]>;
 def int_mips_ld_d : ClangBuiltin<"__builtin_msa_ld_d">,
-  Intrinsic<[llvm_v2i64_ty], [llvm_ptr_ty, llvm_i32_ty],
+  DefaultAttrsIntrinsic<[llvm_v2i64_ty], [llvm_ptr_ty, llvm_i32_ty],
   [IntrReadMem, IntrArgMemOnly]>;
 
 def int_mips_ldr_d : ClangBuiltin<"__builtin_msa_ldr_d">,
-  Intrinsic<[llvm_v2i64_ty], [llvm_ptr_ty, llvm_i32_ty],
+  DefaultAttrsIntrinsic<[llvm_v2i64_ty], [llvm_ptr_ty, llvm_i32_ty],
   [IntrReadMem, IntrArgMemOnly]>;
 def int_mips_ldr_w : ClangBuiltin<"__builtin_msa_ldr_w">,
-  Intrinsic<[llvm_v4i32_ty], [llvm_ptr_ty, llvm_i32_ty],
+  DefaultAttrsIntrinsic<[llvm_v4i32_ty], [llvm_ptr_ty, llvm_i32_ty],
   [IntrReadMem, IntrArgMemOnly]>;
 
 def int_mips_ldi_b : ClangBuiltin<"__builtin_msa_ldi_b">,
diff --git a/llvm/include/llvm/IR/IntrinsicsNVVM.td b/llvm/include/llvm/IR/IntrinsicsNVVM.td
index 65a3d2d0f943a77..40fd86d3063a169 100644
--- a/llvm/include/llvm/IR/IntrinsicsNVVM.td
+++ b/llvm/include/llvm/IR/IntrinsicsNVVM.td
@@ -4729,7 +4729,7 @@ def int_nvvm_redux_sync_or : ClangBuiltin<"__nvvm_redux_sync_or">,
 class NVVM_WMMA_LD<WMMA_REGS Frag, string Layout, int WithStride>
   : Intrinsic<Frag.regs,
               !if(WithStride, [llvm_anyptr_ty, llvm_i32_ty], [llvm_anyptr_ty]),
-              [IntrReadMem, IntrArgMemOnly, IntrNoCallback, ReadOnly<ArgIndex<0>>, NoCapture<ArgIndex<0>>],
+              [IntrWillReturn, IntrReadMem, IntrArgMemOnly, IntrNoCallback, ReadOnly<ArgIndex<0>>, NoCapture<ArgIndex<0>>],
               WMMA_NAME_LDST<"load", Frag, Layout, WithStride>.intr>;
 
 // WMMA.STORE.D
diff --git a/llvm/include/llvm/IR/IntrinsicsSystemZ.td b/llvm/include/llvm/IR/IntrinsicsSystemZ.td
index 9f4b905fedc7c17..4f925979cf85642 100644
--- a/llvm/include/llvm/IR/IntrinsicsSystemZ.td
+++ b/llvm/include/llvm/IR/IntrinsicsSystemZ.td
@@ -230,11 +230,11 @@ let TargetPrefix = "s390" in {
                                 [IntrNoMem, ImmArg<ArgIndex<1>>]>;
 
   def int_s390_vlbb : ClangBuiltin<"__builtin_s390_vlbb">,
-                      Intrinsic<[llvm_v16i8_ty], [llvm_ptr_ty, llvm_i32_ty],
+                      DefaultAttrsIntrinsic<[llvm_v16i8_ty], [llvm_ptr_ty, llvm_i32_ty],
                                 [IntrReadMem, IntrArgMemOnly, ImmArg<ArgIndex<1>>]>;
 
   def int_s390_vll : ClangBuiltin<"__builtin_s390_vll">,
-                     Intrinsic<[llvm_v16i8_ty], [llvm_i32_ty, llvm_ptr_ty],
+                     DefaultAttrsIntrinsic<[llvm_v16i8_ty], [llvm_i32_ty, llvm_ptr_ty],
                                [IntrReadMem, IntrArgMemOnly]>;
 
   def int_s390_vpdi : ClangBuiltin<"__builtin_s390_vpdi">,
@@ -399,7 +399,7 @@ let TargetPrefix = "s390" in {
 
   // Instructions from the Vector Packed Decimal Facility
   def int_s390_vlrl : ClangBuiltin<"__builtin_s390_vlrlr">,
-                      Intrinsic<[llvm_v16i8_ty], [llvm_i32_ty, llvm_ptr_ty],
+                      DefaultAttrsIntrinsic<[llvm_v16i8_ty], [llvm_i32_ty, llvm_ptr_ty],
                                 [IntrReadMem, IntrArgMemOnly]>;
 
   def int_s390_vstrl : ClangBuiltin<"__builtin_s390_vstrlr">,
diff --git a/llvm/include/llvm/IR/IntrinsicsVE.td b/llvm/include/llvm/IR/IntrinsicsVE.td
index 15b828b320ea2c4..633bc6603bf2311 100644
--- a/llvm/include/llvm/IR/IntrinsicsVE.td
+++ b/llvm/include/llvm/IR/IntrinsicsVE.td
@@ -3,10 +3,10 @@
 // VEL Intrinsic instructions.
 let TargetPrefix = "ve" in {
   def int_ve_vl_pack_f32p : ClangBuiltin<"__builtin_ve_vl_pack_f32p">,
-                            Intrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_ptr_ty],
+                            DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_ptr_ty],
                                       [IntrReadMem]>;
   def int_ve_vl_pack_f32a : ClangBuiltin<"__builtin_ve_vl_pack_f32a">,
-                            Intrinsic<[llvm_i64_ty], [llvm_ptr_ty],
+                            DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_ptr_ty],
                                       [IntrReadMem]>;
 
   def int_ve_vl_extract_vm512u :
diff --git a/llvm/include/llvm/IR/IntrinsicsVEVL.gen.td b/llvm/include/llvm/IR/IntrinsicsVEVL.gen.td
index 554dd85572005e9..0c44dfb9076548e 100644
--- a/llvm/include/llvm/IR/IntrinsicsVEVL.gen.td
+++ b/llvm/include/llvm/IR/IntrinsicsVEVL.gen.td
@@ -1,35 +1,35 @@
-let TargetPrefix = "ve" in def int_ve_vl_vld_vssl : ClangBuiltin<"__builtin_ve_vl_vld_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vld_vssvl : ClangBuiltin<"__builtin_ve_vl_vld_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vldnc_vssl : ClangBuiltin<"__builtin_ve_vl_vldnc_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vldnc_vssvl : ClangBuiltin<"__builtin_ve_vl_vldnc_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vldu_vssl : ClangBuiltin<"__builtin_ve_vl_vldu_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vldu_vssvl : ClangBuiltin<"__builtin_ve_vl_vldu_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vldunc_vssl : ClangBuiltin<"__builtin_ve_vl_vldunc_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vldunc_vssvl : ClangBuiltin<"__builtin_ve_vl_vldunc_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vldlsx_vssl : ClangBuiltin<"__builtin_ve_vl_vldlsx_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vldlsx_vssvl : ClangBuiltin<"__builtin_ve_vl_vldlsx_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vldlsxnc_vssl : ClangBuiltin<"__builtin_ve_vl_vldlsxnc_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vldlsxnc_vssvl : ClangBuiltin<"__builtin_ve_vl_vldlsxnc_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vldlzx_vssl : ClangBuiltin<"__builtin_ve_vl_vldlzx_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vldlzx_vssvl : ClangBuiltin<"__builtin_ve_vl_vldlzx_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vldlzxnc_vssl : ClangBuiltin<"__builtin_ve_vl_vldlzxnc_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vldlzxnc_vssvl : ClangBuiltin<"__builtin_ve_vl_vldlzxnc_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vld2d_vssl : ClangBuiltin<"__builtin_ve_vl_vld2d_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vld2d_vssvl : ClangBuiltin<"__builtin_ve_vl_vld2d_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vld2dnc_vssl : ClangBuiltin<"__builtin_ve_vl_vld2dnc_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vld2dnc_vssvl : ClangBuiltin<"__builtin_ve_vl_vld2dnc_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vldu2d_vssl : ClangBuiltin<"__builtin_ve_vl_vldu2d_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vldu2d_vssvl : ClangBuiltin<"__builtin_ve_vl_vldu2d_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vldu2dnc_vssl : ClangBuiltin<"__builtin_ve_vl_vldu2dnc_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vldu2dnc_vssvl : ClangBuiltin<"__builtin_ve_vl_vldu2dnc_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vldl2dsx_vssl : ClangBuiltin<"__builtin_ve_vl_vldl2dsx_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vldl2dsx_vssvl : ClangBuiltin<"__builtin_ve_vl_vldl2dsx_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vldl2dsxnc_vssl : ClangBuiltin<"__builtin_ve_vl_vldl2dsxnc_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vldl2dsxnc_vssvl : ClangBuiltin<"__builtin_ve_vl_vldl2dsxnc_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vldl2dzx_vssl : ClangBuiltin<"__builtin_ve_vl_vldl2dzx_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vldl2dzx_vssvl : ClangBuiltin<"__builtin_ve_vl_vldl2dzx_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vldl2dzxnc_vssl : ClangBuiltin<"__builtin_ve_vl_vldl2dzxnc_vssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vldl2dzxnc_vssvl : ClangBuiltin<"__builtin_ve_vl_vldl2dzxnc_vssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vld_vssl : ClangBuiltin<"__builtin_ve_vl_vld_vssl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vld_vssvl : ClangBuiltin<"__builtin_ve_vl_vld_vssvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vldnc_vssl : ClangBuiltin<"__builtin_ve_vl_vldnc_vssl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vldnc_vssvl : ClangBuiltin<"__builtin_ve_vl_vldnc_vssvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vldu_vssl : ClangBuiltin<"__builtin_ve_vl_vldu_vssl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vldu_vssvl : ClangBuiltin<"__builtin_ve_vl_vldu_vssvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vldunc_vssl : ClangBuiltin<"__builtin_ve_vl_vldunc_vssl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vldunc_vssvl : ClangBuiltin<"__builtin_ve_vl_vldunc_vssvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vldlsx_vssl : ClangBuiltin<"__builtin_ve_vl_vldlsx_vssl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vldlsx_vssvl : ClangBuiltin<"__builtin_ve_vl_vldlsx_vssvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vldlsxnc_vssl : ClangBuiltin<"__builtin_ve_vl_vldlsxnc_vssl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vldlsxnc_vssvl : ClangBuiltin<"__builtin_ve_vl_vldlsxnc_vssvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vldlzx_vssl : ClangBuiltin<"__builtin_ve_vl_vldlzx_vssl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vldlzx_vssvl : ClangBuiltin<"__builtin_ve_vl_vldlzx_vssvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vldlzxnc_vssl : ClangBuiltin<"__builtin_ve_vl_vldlzxnc_vssl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vldlzxnc_vssvl : ClangBuiltin<"__builtin_ve_vl_vldlzxnc_vssvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vld2d_vssl : ClangBuiltin<"__builtin_ve_vl_vld2d_vssl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vld2d_vssvl : ClangBuiltin<"__builtin_ve_vl_vld2d_vssvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vld2dnc_vssl : ClangBuiltin<"__builtin_ve_vl_vld2dnc_vssl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vld2dnc_vssvl : ClangBuiltin<"__builtin_ve_vl_vld2dnc_vssvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vldu2d_vssl : ClangBuiltin<"__builtin_ve_vl_vldu2d_vssl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vldu2d_vssvl : ClangBuiltin<"__builtin_ve_vl_vldu2d_vssvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vldu2dnc_vssl : ClangBuiltin<"__builtin_ve_vl_vldu2dnc_vssl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vldu2dnc_vssvl : ClangBuiltin<"__builtin_ve_vl_vldu2dnc_vssvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vldl2dsx_vssl : ClangBuiltin<"__builtin_ve_vl_vldl2dsx_vssl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vldl2dsx_vssvl : ClangBuiltin<"__builtin_ve_vl_vldl2dsx_vssvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vldl2dsxnc_vssl : ClangBuiltin<"__builtin_ve_vl_vldl2dsxnc_vssl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vldl2dsxnc_vssvl : ClangBuiltin<"__builtin_ve_vl_vldl2dsxnc_vssvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vldl2dzx_vssl : ClangBuiltin<"__builtin_ve_vl_vldl2dzx_vssl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vldl2dzx_vssvl : ClangBuiltin<"__builtin_ve_vl_vldl2dzx_vssvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vldl2dzxnc_vssl : ClangBuiltin<"__builtin_ve_vl_vldl2dzxnc_vssl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vldl2dzxnc_vssvl : ClangBuiltin<"__builtin_ve_vl_vldl2dzxnc_vssvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<i64>, llvm_ptr_ty, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
 let TargetPrefix = "ve" in def int_ve_vl_vst_vssl : ClangBuiltin<"__builtin_ve_vl_vst_vssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrWriteMem]>;
 let TargetPrefix = "ve" in def int_ve_vl_vst_vssml : ClangBuiltin<"__builtin_ve_vl_vst_vssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>;
 let TargetPrefix = "ve" in def int_ve_vl_vstnc_vssl : ClangBuiltin<"__builtin_ve_vl_vstnc_vssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<i64>, llvm_ptr_ty, LLVMType<i32>], [IntrWriteMem]>;
@@ -1176,38 +1176,38 @@ let TargetPrefix = "ve" in def int_ve_vl_vror_vvl : ClangBuiltin<"__builtin_ve_v
 let TargetPrefix = "ve" in def int_ve_vl_vror_vvml : ClangBuiltin<"__builtin_ve_vl_vror_vvml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
 let TargetPrefix = "ve" in def int_ve_vl_vrxor_vvl : ClangBuiltin<"__builtin_ve_vl_vrxor_vvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i32>], [IntrNoMem]>;
 let TargetPrefix = "ve" in def int_ve_vl_vrxor_vvml : ClangBuiltin<"__builtin_ve_vl_vrxor_vvml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<v256i1>, LLVMType<i32>], [IntrNoMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgt_vvssl : ClangBuiltin<"__builtin_ve_vl_vgt_vvssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgt_vvssvl : ClangBuiltin<"__builtin_ve_vl_vgt_vvssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgt_vvssml : ClangBuiltin<"__builtin_ve_vl_vgt_vvssml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgt_vvssmvl : ClangBuiltin<"__builtin_ve_vl_vgt_vvssmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgtnc_vvssl : ClangBuiltin<"__builtin_ve_vl_vgtnc_vvssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgtnc_vvssvl : ClangBuiltin<"__builtin_ve_vl_vgtnc_vvssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgtnc_vvssml : ClangBuiltin<"__builtin_ve_vl_vgtnc_vvssml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgtnc_vvssmvl : ClangBuiltin<"__builtin_ve_vl_vgtnc_vvssmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgtu_vvssl : ClangBuiltin<"__builtin_ve_vl_vgtu_vvssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgtu_vvssvl : ClangBuiltin<"__builtin_ve_vl_vgtu_vvssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgtu_vvssml : ClangBuiltin<"__builtin_ve_vl_vgtu_vvssml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgtu_vvssmvl : ClangBuiltin<"__builtin_ve_vl_vgtu_vvssmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgtunc_vvssl : ClangBuiltin<"__builtin_ve_vl_vgtunc_vvssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgtunc_vvssvl : ClangBuiltin<"__builtin_ve_vl_vgtunc_vvssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgtunc_vvssml : ClangBuiltin<"__builtin_ve_vl_vgtunc_vvssml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgtunc_vvssmvl : ClangBuiltin<"__builtin_ve_vl_vgtunc_vvssmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgtlsx_vvssl : ClangBuiltin<"__builtin_ve_vl_vgtlsx_vvssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgtlsx_vvssvl : ClangBuiltin<"__builtin_ve_vl_vgtlsx_vvssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgtlsx_vvssml : ClangBuiltin<"__builtin_ve_vl_vgtlsx_vvssml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgtlsx_vvssmvl : ClangBuiltin<"__builtin_ve_vl_vgtlsx_vvssmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgtlsxnc_vvssl : ClangBuiltin<"__builtin_ve_vl_vgtlsxnc_vvssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgtlsxnc_vvssvl : ClangBuiltin<"__builtin_ve_vl_vgtlsxnc_vvssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgtlsxnc_vvssml : ClangBuiltin<"__builtin_ve_vl_vgtlsxnc_vvssml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgtlsxnc_vvssmvl : ClangBuiltin<"__builtin_ve_vl_vgtlsxnc_vvssmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgtlzx_vvssl : ClangBuiltin<"__builtin_ve_vl_vgtlzx_vvssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgtlzx_vvssvl : ClangBuiltin<"__builtin_ve_vl_vgtlzx_vvssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgtlzx_vvssml : ClangBuiltin<"__builtin_ve_vl_vgtlzx_vvssml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgtlzx_vvssmvl : ClangBuiltin<"__builtin_ve_vl_vgtlzx_vvssmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgtlzxnc_vvssl : ClangBuiltin<"__builtin_ve_vl_vgtlzxnc_vvssl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgtlzxnc_vvssvl : ClangBuiltin<"__builtin_ve_vl_vgtlzxnc_vvssvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgtlzxnc_vvssml : ClangBuiltin<"__builtin_ve_vl_vgtlzxnc_vvssml">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrReadMem]>;
-let TargetPrefix = "ve" in def int_ve_vl_vgtlzxnc_vvssmvl : ClangBuiltin<"__builtin_ve_vl_vgtlzxnc_vvssmvl">, Intrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgt_vvssl : ClangBuiltin<"__builtin_ve_vl_vgt_vvssl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgt_vvssvl : ClangBuiltin<"__builtin_ve_vl_vgt_vvssvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgt_vvssml : ClangBuiltin<"__builtin_ve_vl_vgt_vvssml">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgt_vvssmvl : ClangBuiltin<"__builtin_ve_vl_vgt_vvssmvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtnc_vvssl : ClangBuiltin<"__builtin_ve_vl_vgtnc_vvssl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtnc_vvssvl : ClangBuiltin<"__builtin_ve_vl_vgtnc_vvssvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtnc_vvssml : ClangBuiltin<"__builtin_ve_vl_vgtnc_vvssml">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtnc_vvssmvl : ClangBuiltin<"__builtin_ve_vl_vgtnc_vvssmvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtu_vvssl : ClangBuiltin<"__builtin_ve_vl_vgtu_vvssl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtu_vvssvl : ClangBuiltin<"__builtin_ve_vl_vgtu_vvssvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtu_vvssml : ClangBuiltin<"__builtin_ve_vl_vgtu_vvssml">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtu_vvssmvl : ClangBuiltin<"__builtin_ve_vl_vgtu_vvssmvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtunc_vvssl : ClangBuiltin<"__builtin_ve_vl_vgtunc_vvssl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtunc_vvssvl : ClangBuiltin<"__builtin_ve_vl_vgtunc_vvssvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtunc_vvssml : ClangBuiltin<"__builtin_ve_vl_vgtunc_vvssml">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtunc_vvssmvl : ClangBuiltin<"__builtin_ve_vl_vgtunc_vvssmvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtlsx_vvssl : ClangBuiltin<"__builtin_ve_vl_vgtlsx_vvssl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtlsx_vvssvl : ClangBuiltin<"__builtin_ve_vl_vgtlsx_vvssvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtlsx_vvssml : ClangBuiltin<"__builtin_ve_vl_vgtlsx_vvssml">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtlsx_vvssmvl : ClangBuiltin<"__builtin_ve_vl_vgtlsx_vvssmvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtlsxnc_vvssl : ClangBuiltin<"__builtin_ve_vl_vgtlsxnc_vvssl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtlsxnc_vvssvl : ClangBuiltin<"__builtin_ve_vl_vgtlsxnc_vvssvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtlsxnc_vvssml : ClangBuiltin<"__builtin_ve_vl_vgtlsxnc_vvssml">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtlsxnc_vvssmvl : ClangBuiltin<"__builtin_ve_vl_vgtlsxnc_vvssmvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtlzx_vvssl : ClangBuiltin<"__builtin_ve_vl_vgtlzx_vvssl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtlzx_vvssvl : ClangBuiltin<"__builtin_ve_vl_vgtlzx_vvssvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtlzx_vvssml : ClangBuiltin<"__builtin_ve_vl_vgtlzx_vvssml">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtlzx_vvssmvl : ClangBuiltin<"__builtin_ve_vl_vgtlzx_vvssmvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtlzxnc_vvssl : ClangBuiltin<"__builtin_ve_vl_vgtlzxnc_vvssl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtlzxnc_vvssvl : ClangBuiltin<"__builtin_ve_vl_vgtlzxnc_vvssvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtlzxnc_vvssml : ClangBuiltin<"__builtin_ve_vl_vgtlzxnc_vvssml">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrReadMem]>;
+let TargetPrefix = "ve" in def int_ve_vl_vgtlzxnc_vvssmvl : ClangBuiltin<"__builtin_ve_vl_vgtlzxnc_vvssmvl">, DefaultAttrsIntrinsic<[LLVMType<v256f64>], [LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<v256f64>, LLVMType<i32>], [IntrReadMem]>;
 let TargetPrefix = "ve" in def int_ve_vl_vsc_vvssl : ClangBuiltin<"__builtin_ve_vl_vsc_vvssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrWriteMem]>;
 let TargetPrefix = "ve" in def int_ve_vl_vsc_vvssml : ClangBuiltin<"__builtin_ve_vl_vsc_vvssml">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<v256i1>, LLVMType<i32>], [IntrWriteMem]>;
 let TargetPrefix = "ve" in def int_ve_vl_vscnc_vvssl : ClangBuiltin<"__builtin_ve_vl_vscnc_vvssl">, Intrinsic<[], [LLVMType<v256f64>, LLVMType<v256f64>, LLVMType<i64>, LLVMType<i64>, LLVMType<i32>], [IntrWriteMem]>;
diff --git a/llvm/include/llvm/IR/IntrinsicsWebAssembly.td b/llvm/include/llvm/IR/IntrinsicsWebAssembly.td
index 4d2df1c44ebce5c..d03e532964f2878 100644
--- a/llvm/include/llvm/IR/IntrinsicsWebAssembly.td
+++ b/llvm/include/llvm/IR/IntrinsicsWebAssembly.td
@@ -346,7 +346,7 @@ def int_wasm_relaxed_dot_bf16x8_add_f32:
 // TODO: Replace these intrinsic with normal ISel patterns once the XXX
 // instructions are merged to the proposal.
 def int_wasm_loadf16_f32:
-  Intrinsic<[llvm_float_ty],
+  DefaultAttrsIntrinsic<[llvm_float_ty],
             [llvm_ptr_ty],
             [IntrReadMem, IntrArgMemOnly],
              "", [SDNPMemOperand]>;
diff --git a/llvm/test/CodeGen/BPF/sockex2.ll b/llvm/test/CodeGen/BPF/sockex2.ll
index 4131d9dac31d88b..b1264099f64c60b 100644
--- a/llvm/test/CodeGen/BPF/sockex2.ll
+++ b/llvm/test/CodeGen/BPF/sockex2.ll
@@ -311,7 +311,7 @@ flow_dissector.exit.thread:                       ; preds = %86, %12, %196, %199
 ; CHECK-LABEL: bpf_prog2:
 ; CHECK: r0 = *(u16 *)skb[12] # encoding: [0x28,0x00,0x00,0x00,0x0c,0x00,0x00,0x00]
 ; CHECK: r0 = *(u16 *)skb[16] # encoding: [0x28,0x00,0x00,0x00,0x10,0x00,0x00,0x00]
-; CHECK: implicit-def: $r8
+; CHECK: implicit-def: $r7
 ; CHECK: r1 =
 ; CHECK: call 1 # encoding: [0x85,0x00,0x00,0x00,0x01,0x00,0x00,0x00]
 ; CHECK: call 2 # encoding: [0x85,0x00,0x00,0x00,0x02,0x00,0x00,0x00]
diff --git a/llvm/test/Transforms/DCE/intrinsics-bpf.ll b/llvm/test/Transforms/DCE/intrinsics-bpf.ll
new file mode 100644
index 000000000000000..135588ba21cbb15
--- /dev/null
+++ b/llvm/test/Transforms/DCE/intrinsics-bpf.ll
@@ -0,0 +1,33 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S < %s -passes=dce  | FileCheck %s
+
+declare i64 @llvm.bpf.load.half(ptr, i64)
+declare i64 @llvm.bpf.load.word(ptr, i64)
+declare i64 @llvm.bpf.load.byte(ptr, i64)
+
+define void @test_bpf_load_half(ptr %a, i64 %b) {
+; CHECK-LABEL: define void @test_bpf_load_half(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[B:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call i64 @llvm.bpf.load.half(ptr %a, i64 %b)
+  ret void
+}
+
+define void @test_bpf_load_word(ptr %a, i64 %b) {
+; CHECK-LABEL: define void @test_bpf_load_word(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[B:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call i64 @llvm.bpf.load.word(ptr %a, i64 %b)
+  ret void
+}
+
+define void @test_bpf_load_byte(ptr %a, i64 %b) {
+; CHECK-LABEL: define void @test_bpf_load_byte(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[B:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call i64 @llvm.bpf.load.byte(ptr %a, i64 %b)
+  ret void
+}
diff --git a/llvm/test/Transforms/DCE/intrinsics-loongarch.ll b/llvm/test/Transforms/DCE/intrinsics-loongarch.ll
new file mode 100644
index 000000000000000..665069091a48de6
--- /dev/null
+++ b/llvm/test/Transforms/DCE/intrinsics-loongarch.ll
@@ -0,0 +1,124 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S < %s -passes=dce  | FileCheck %s
+
+declare <16 x i8> @llvm.loongarch.lsx.vld(ptr, i32)
+declare <16 x i8> @llvm.loongarch.lsx.vldx(ptr, i64)
+declare <16 x i8> @llvm.loongarch.lsx.vldrepl.b(ptr, i32)
+declare <8 x i16> @llvm.loongarch.lsx.vldrepl.h(ptr, i32)
+declare <4 x i32> @llvm.loongarch.lsx.vldrepl.w(ptr, i32)
+declare <2 x i64> @llvm.loongarch.lsx.vldrepl.d(ptr, i32)
+
+declare <32 x i8> @llvm.loongarch.lasx.xvld(ptr, i32)
+declare <32 x i8> @llvm.loongarch.lasx.xvldx(ptr, i64)
+declare <32 x i8> @llvm.loongarch.lasx.xvldrepl.b(ptr, i32)
+declare <16 x i16> @llvm.loongarch.lasx.xvldrepl.h(ptr, i32)
+declare <8 x i32> @llvm.loongarch.lasx.xvldrepl.w(ptr, i32)
+declare <4 x i64> @llvm.loongarch.lasx.xvldrepl.d(ptr, i32)
+
+define void @test_loongarch_lsx_vld(ptr %a) {
+; CHECK-LABEL: define void @test_loongarch_lsx_vld(
+; CHECK-SAME: ptr [[A:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <16 x i8> @llvm.loongarch.lsx.vld(ptr %a, i32 8)
+  ret void
+}
+
+define void @test_loongarch_lsx_vldx(ptr %a, i64 %b) {
+; CHECK-LABEL: define void @test_loongarch_lsx_vldx(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[B:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <16 x i8> @llvm.loongarch.lsx.vldx(ptr %a, i64 %b)
+  ret void
+}
+
+define void @test_loongarch_lsx_vldrepl_b(ptr %a) {
+; CHECK-LABEL: define void @test_loongarch_lsx_vldrepl_b(
+; CHECK-SAME: ptr [[A:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <16 x i8> @llvm.loongarch.lsx.vldrepl.b(ptr %a, i32 12)
+  ret void
+}
+
+define void @test_loongarch_lsx_vldrepl_h(ptr %a) {
+; CHECK-LABEL: define void @test_loongarch_lsx_vldrepl_h(
+; CHECK-SAME: ptr [[A:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <8 x i16> @llvm.loongarch.lsx.vldrepl.h(ptr %a, i32 12)
+  ret void
+}
+
+define void @test_loongarch_lsx_vldrepl_w(ptr %a) {
+; CHECK-LABEL: define void @test_loongarch_lsx_vldrepl_w(
+; CHECK-SAME: ptr [[A:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <4 x i32> @llvm.loongarch.lsx.vldrepl.w(ptr %a, i32 4)
+  ret void
+}
+
+define void @test_loongarch_lsx_vldrepl_d(ptr %a) {
+; CHECK-LABEL: define void @test_loongarch_lsx_vldrepl_d(
+; CHECK-SAME: ptr [[A:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <2 x i64> @llvm.loongarch.lsx.vldrepl.d(ptr %a, i32 4)
+  ret void
+}
+
+define void @test_loongarch_lasx_xvld(ptr %a) {
+; CHECK-LABEL: define void @test_loongarch_lasx_xvld(
+; CHECK-SAME: ptr [[A:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <32 x i8> @llvm.loongarch.lasx.xvld(ptr %a, i32 4)
+  ret void
+}
+
+define void @test_loongarch_lasx_xvldx(ptr %a, i64 %b) {
+; CHECK-LABEL: define void @test_loongarch_lasx_xvldx(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[B:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <32 x i8> @llvm.loongarch.lasx.xvldx(ptr %a, i64 %b)
+  ret void
+}
+
+define void @test_loongarch_lasx_xvldrepl_b(ptr %a) {
+; CHECK-LABEL: define void @test_loongarch_lasx_xvldrepl_b(
+; CHECK-SAME: ptr [[A:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <32 x i8> @llvm.loongarch.lasx.xvldrepl.b(ptr %a, i32 4)
+  ret void
+}
+
+define void @test_loongarch_lasx_xvldrepl_h(ptr %a) {
+; CHECK-LABEL: define void @test_loongarch_lasx_xvldrepl_h(
+; CHECK-SAME: ptr [[A:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <16 x i16> @llvm.loongarch.lasx.xvldrepl.h(ptr %a, i32 4)
+  ret void
+}
+
+define void @test_loongarch_lasx_xvldrepl_w(ptr %a) {
+; CHECK-LABEL: define void @test_loongarch_lasx_xvldrepl_w(
+; CHECK-SAME: ptr [[A:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <8 x i32> @llvm.loongarch.lasx.xvldrepl.w(ptr %a, i32 4)
+  ret void
+}
+
+define void @test_loongarch_lasx_xvldrepl_d(ptr %a) {
+; CHECK-LABEL: define void @test_loongarch_lasx_xvldrepl_d(
+; CHECK-SAME: ptr [[A:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <4 x i64> @llvm.loongarch.lasx.xvldrepl.d(ptr %a, i32 4)
+  ret void
+}
diff --git a/llvm/test/Transforms/DCE/intrinsics-mips.ll b/llvm/test/Transforms/DCE/intrinsics-mips.ll
new file mode 100644
index 000000000000000..22d709e21e061b5
--- /dev/null
+++ b/llvm/test/Transforms/DCE/intrinsics-mips.ll
@@ -0,0 +1,131 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S < %s -passes=dce  | FileCheck %s
+
+declare i32 @llvm.mips.rddsp(i32)
+declare <4 x i8> @llvm.mips.pick.qb(<4 x i8>, <4 x i8>)
+declare <2 x i16> @llvm.mips.pick.ph(<2 x i16>, <2 x i16>)
+declare i32 @llvm.mips.bposge32()
+declare i32 @llvm.mips.lbux(ptr, i32)
+declare i32 @llvm.mips.lhx(ptr, i32)
+declare i32 @llvm.mips.lwx(ptr, i32)
+declare <16 x i8> @llvm.mips.ld.b(ptr, i32)
+declare <8 x i16> @llvm.mips.ld.h(ptr, i32)
+declare <4 x i32> @llvm.mips.ld.w(ptr, i32)
+declare <2 x i64> @llvm.mips.ld.d(ptr, i32)
+declare <2 x i64> @llvm.mips.ldr.d(ptr, i32)
+declare <4 x i32> @llvm.mips.ldr.w(ptr, i32)
+
+define void @test_mips_rddsp() {
+; CHECK-LABEL: define void @test_mips_rddsp() {
+; CHECK-NEXT:    ret void
+;
+  %v = call i32 @llvm.mips.rddsp(i32 4)
+  ret void
+}
+
+define void @test_llvm_mips_pick_qb(<4 x i8> %a, <4 x i8> %b) {
+; CHECK-LABEL: define void @test_llvm_mips_pick_qb(
+; CHECK-SAME: <4 x i8> [[A:%.*]], <4 x i8> [[B:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <4 x i8> @llvm.mips.pick.qb(<4 x i8> %a, <4 x i8> %b)
+  ret void
+}
+
+define void @test_llvm_mips_pick_ph(<2 x i16> %a, <2 x i16> %b) {
+; CHECK-LABEL: define void @test_llvm_mips_pick_ph(
+; CHECK-SAME: <2 x i16> [[A:%.*]], <2 x i16> [[B:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <2 x i16> @llvm.mips.pick.ph(<2 x i16> %a, <2 x i16> %b)
+  ret void
+}
+
+define void @test_llvm_mips_bposge32() {
+; CHECK-LABEL: define void @test_llvm_mips_bposge32() {
+; CHECK-NEXT:    ret void
+;
+  %v = call i32 @llvm.mips.bposge32()
+  ret void
+}
+
+define void @test_llvm_mips_lbux(ptr %a, i32 %b) {
+; CHECK-LABEL: define void @test_llvm_mips_lbux(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call i32 @llvm.mips.lbux(ptr %a, i32 %b)
+  ret void
+}
+
+define void @test_llvm_mips_lhx(ptr %a, i32 %b) {
+; CHECK-LABEL: define void @test_llvm_mips_lhx(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call i32 @llvm.mips.lhx(ptr %a, i32 %b)
+  ret void
+}
+
+define void @test_llvm_mips_lwx(ptr %a, i32 %b) {
+; CHECK-LABEL: define void @test_llvm_mips_lwx(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call i32 @llvm.mips.lwx(ptr %a, i32 %b)
+  ret void
+}
+
+define void @test_llvm_mips_ld_b(ptr %a, i32 %b) {
+; CHECK-LABEL: define void @test_llvm_mips_ld_b(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <16 x i8> @llvm.mips.ld.b(ptr %a, i32 %b)
+  ret void
+}
+
+define void @test_llvm_mips_ld_h(ptr %a, i32 %b) {
+; CHECK-LABEL: define void @test_llvm_mips_ld_h(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <8 x i16> @llvm.mips.ld.h(ptr %a, i32 %b)
+  ret void
+}
+
+define void @test_llvm_mips_ld_w(ptr %a, i32 %b) {
+; CHECK-LABEL: define void @test_llvm_mips_ld_w(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <4 x i32> @llvm.mips.ld.w(ptr %a, i32 %b)
+  ret void
+}
+
+define void @test_llvm_mips_ld_d(ptr %a, i32 %b) {
+; CHECK-LABEL: define void @test_llvm_mips_ld_d(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <2 x i64> @llvm.mips.ld.d(ptr %a, i32 %b)
+  ret void
+}
+
+define void @test_llvm_mips_ldr_d(ptr %a, i32 %b) {
+; CHECK-LABEL: define void @test_llvm_mips_ldr_d(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <2 x i64> @llvm.mips.ldr.d(ptr %a, i32 %b)
+  ret void
+}
+
+define void @test_llvm_mips_ldr_w(ptr %a, i32 %b) {
+; CHECK-LABEL: define void @test_llvm_mips_ldr_w(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <4 x i32> @llvm.mips.ldr.w(ptr %a, i32 %b)
+  ret void
+}
diff --git a/llvm/test/Transforms/DCE/intrinsics-nvvm.ll b/llvm/test/Transforms/DCE/intrinsics-nvvm.ll
new file mode 100644
index 000000000000000..f0b5c2a5920f723
--- /dev/null
+++ b/llvm/test/Transforms/DCE/intrinsics-nvvm.ll
@@ -0,0 +1,759 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S < %s -passes=dce  | FileCheck %s
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.a.row.f16.p0i8(i8 addrspace(0)* %src );
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_a_row_f16_p0i8(i8 addrspace(0)* %src ) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_a_row_f16_p0i8(
+; CHECK-SAME: ptr [[SRC:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.a.row.f16.p0i8(i8 addrspace(0)* %src );
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.a.row.stride.f16.p0i8(i8 addrspace(0)* %src , i32 %stride);
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_a_row_stride_f16_p0i8(i8 addrspace(0)* %src , i32 %stride) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_a_row_stride_f16_p0i8(
+; CHECK-SAME: ptr [[SRC:%.*]], i32 [[STRIDE:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.a.row.stride.f16.p0i8(i8 addrspace(0)* %src , i32 %stride);
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.a.row.f16.p3i8(i8 addrspace(3)* %src );
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_a_row_f16_p3i8(i8 addrspace(3)* %src ) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_a_row_f16_p3i8(
+; CHECK-SAME: ptr addrspace(3) [[SRC:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.a.row.f16.p3i8(i8 addrspace(3)* %src );
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.a.row.stride.f16.p3i8(i8 addrspace(3)* %src , i32 %stride);
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_a_row_stride_f16_p3i8(i8 addrspace(3)* %src , i32 %stride) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_a_row_stride_f16_p3i8(
+; CHECK-SAME: ptr addrspace(3) [[SRC:%.*]], i32 [[STRIDE:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.a.row.stride.f16.p3i8(i8 addrspace(3)* %src , i32 %stride);
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.a.row.f16.p1i8(i8 addrspace(1)* %src );
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_a_row_f16_p1i8(i8 addrspace(1)* %src ) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_a_row_f16_p1i8(
+; CHECK-SAME: ptr addrspace(1) [[SRC:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.a.row.f16.p1i8(i8 addrspace(1)* %src );
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.a.row.stride.f16.p1i8(i8 addrspace(1)* %src , i32 %stride);
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_a_row_stride_f16_p1i8(i8 addrspace(1)* %src , i32 %stride) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_a_row_stride_f16_p1i8(
+; CHECK-SAME: ptr addrspace(1) [[SRC:%.*]], i32 [[STRIDE:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.a.row.stride.f16.p1i8(i8 addrspace(1)* %src , i32 %stride);
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.a.col.f16.p0i8(i8 addrspace(0)* %src );
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_a_col_f16_p0i8(i8 addrspace(0)* %src ) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_a_col_f16_p0i8(
+; CHECK-SAME: ptr [[SRC:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.a.col.f16.p0i8(i8 addrspace(0)* %src );
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.a.col.stride.f16.p0i8(i8 addrspace(0)* %src , i32 %stride);
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_a_col_stride_f16_p0i8(i8 addrspace(0)* %src , i32 %stride) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_a_col_stride_f16_p0i8(
+; CHECK-SAME: ptr [[SRC:%.*]], i32 [[STRIDE:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.a.col.stride.f16.p0i8(i8 addrspace(0)* %src , i32 %stride);
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.a.col.f16.p3i8(i8 addrspace(3)* %src );
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_a_col_f16_p3i8(i8 addrspace(3)* %src ) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_a_col_f16_p3i8(
+; CHECK-SAME: ptr addrspace(3) [[SRC:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.a.col.f16.p3i8(i8 addrspace(3)* %src );
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.a.col.stride.f16.p3i8(i8 addrspace(3)* %src , i32 %stride);
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_a_col_stride_f16_p3i8(i8 addrspace(3)* %src , i32 %stride) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_a_col_stride_f16_p3i8(
+; CHECK-SAME: ptr addrspace(3) [[SRC:%.*]], i32 [[STRIDE:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.a.col.stride.f16.p3i8(i8 addrspace(3)* %src , i32 %stride);
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.a.col.f16.p1i8(i8 addrspace(1)* %src );
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_a_col_f16_p1i8(i8 addrspace(1)* %src ) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_a_col_f16_p1i8(
+; CHECK-SAME: ptr addrspace(1) [[SRC:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.a.col.f16.p1i8(i8 addrspace(1)* %src );
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.a.col.stride.f16.p1i8(i8 addrspace(1)* %src , i32 %stride);
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_a_col_stride_f16_p1i8(i8 addrspace(1)* %src , i32 %stride) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_a_col_stride_f16_p1i8(
+; CHECK-SAME: ptr addrspace(1) [[SRC:%.*]], i32 [[STRIDE:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.a.col.stride.f16.p1i8(i8 addrspace(1)* %src , i32 %stride);
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.b.row.f16.p0i8(i8 addrspace(0)* %src );
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_b_row_f16_p0i8(i8 addrspace(0)* %src ) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_b_row_f16_p0i8(
+; CHECK-SAME: ptr [[SRC:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.b.row.f16.p0i8(i8 addrspace(0)* %src );
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.b.row.stride.f16.p0i8(i8 addrspace(0)* %src , i32 %stride);
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_b_row_stride_f16_p0i8(i8 addrspace(0)* %src , i32 %stride) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_b_row_stride_f16_p0i8(
+; CHECK-SAME: ptr [[SRC:%.*]], i32 [[STRIDE:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.b.row.stride.f16.p0i8(i8 addrspace(0)* %src , i32 %stride);
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.b.row.f16.p3i8(i8 addrspace(3)* %src );
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_b_row_f16_p3i8(i8 addrspace(3)* %src ) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_b_row_f16_p3i8(
+; CHECK-SAME: ptr addrspace(3) [[SRC:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.b.row.f16.p3i8(i8 addrspace(3)* %src );
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.b.row.stride.f16.p3i8(i8 addrspace(3)* %src , i32 %stride);
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_b_row_stride_f16_p3i8(i8 addrspace(3)* %src , i32 %stride) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_b_row_stride_f16_p3i8(
+; CHECK-SAME: ptr addrspace(3) [[SRC:%.*]], i32 [[STRIDE:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.b.row.stride.f16.p3i8(i8 addrspace(3)* %src , i32 %stride);
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.b.row.f16.p1i8(i8 addrspace(1)* %src );
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_b_row_f16_p1i8(i8 addrspace(1)* %src ) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_b_row_f16_p1i8(
+; CHECK-SAME: ptr addrspace(1) [[SRC:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.b.row.f16.p1i8(i8 addrspace(1)* %src );
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.b.row.stride.f16.p1i8(i8 addrspace(1)* %src , i32 %stride);
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_b_row_stride_f16_p1i8(i8 addrspace(1)* %src , i32 %stride) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_b_row_stride_f16_p1i8(
+; CHECK-SAME: ptr addrspace(1) [[SRC:%.*]], i32 [[STRIDE:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.b.row.stride.f16.p1i8(i8 addrspace(1)* %src , i32 %stride);
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.b.col.f16.p0i8(i8 addrspace(0)* %src );
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_b_col_f16_p0i8(i8 addrspace(0)* %src ) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_b_col_f16_p0i8(
+; CHECK-SAME: ptr [[SRC:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.b.col.f16.p0i8(i8 addrspace(0)* %src );
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.b.col.stride.f16.p0i8(i8 addrspace(0)* %src , i32 %stride);
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_b_col_stride_f16_p0i8(i8 addrspace(0)* %src , i32 %stride) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_b_col_stride_f16_p0i8(
+; CHECK-SAME: ptr [[SRC:%.*]], i32 [[STRIDE:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.b.col.stride.f16.p0i8(i8 addrspace(0)* %src , i32 %stride);
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.b.col.f16.p3i8(i8 addrspace(3)* %src );
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_b_col_f16_p3i8(i8 addrspace(3)* %src ) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_b_col_f16_p3i8(
+; CHECK-SAME: ptr addrspace(3) [[SRC:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.b.col.f16.p3i8(i8 addrspace(3)* %src );
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.b.col.stride.f16.p3i8(i8 addrspace(3)* %src , i32 %stride);
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_b_col_stride_f16_p3i8(i8 addrspace(3)* %src , i32 %stride) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_b_col_stride_f16_p3i8(
+; CHECK-SAME: ptr addrspace(3) [[SRC:%.*]], i32 [[STRIDE:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.b.col.stride.f16.p3i8(i8 addrspace(3)* %src , i32 %stride);
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.b.col.f16.p1i8(i8 addrspace(1)* %src );
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_b_col_f16_p1i8(i8 addrspace(1)* %src ) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_b_col_f16_p1i8(
+; CHECK-SAME: ptr addrspace(1) [[SRC:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.b.col.f16.p1i8(i8 addrspace(1)* %src );
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.b.col.stride.f16.p1i8(i8 addrspace(1)* %src , i32 %stride);
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_b_col_stride_f16_p1i8(i8 addrspace(1)* %src , i32 %stride) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_b_col_stride_f16_p1i8(
+; CHECK-SAME: ptr addrspace(1) [[SRC:%.*]], i32 [[STRIDE:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.b.col.stride.f16.p1i8(i8 addrspace(1)* %src , i32 %stride);
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.c.row.f16.p0i8(i8 addrspace(0)* %src );
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_c_row_f16_p0i8(i8 addrspace(0)* %src ) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_c_row_f16_p0i8(
+; CHECK-SAME: ptr [[SRC:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.c.row.f16.p0i8(i8 addrspace(0)* %src );
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.c.row.stride.f16.p0i8(i8 addrspace(0)* %src , i32 %stride);
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_c_row_stride_f16_p0i8(i8 addrspace(0)* %src , i32 %stride) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_c_row_stride_f16_p0i8(
+; CHECK-SAME: ptr [[SRC:%.*]], i32 [[STRIDE:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.c.row.stride.f16.p0i8(i8 addrspace(0)* %src , i32 %stride);
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.c.row.f16.p3i8(i8 addrspace(3)* %src );
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_c_row_f16_p3i8(i8 addrspace(3)* %src ) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_c_row_f16_p3i8(
+; CHECK-SAME: ptr addrspace(3) [[SRC:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.c.row.f16.p3i8(i8 addrspace(3)* %src );
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.c.row.stride.f16.p3i8(i8 addrspace(3)* %src , i32 %stride);
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_c_row_stride_f16_p3i8(i8 addrspace(3)* %src , i32 %stride) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_c_row_stride_f16_p3i8(
+; CHECK-SAME: ptr addrspace(3) [[SRC:%.*]], i32 [[STRIDE:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.c.row.stride.f16.p3i8(i8 addrspace(3)* %src , i32 %stride);
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.c.row.f16.p1i8(i8 addrspace(1)* %src );
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_c_row_f16_p1i8(i8 addrspace(1)* %src ) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_c_row_f16_p1i8(
+; CHECK-SAME: ptr addrspace(1) [[SRC:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.c.row.f16.p1i8(i8 addrspace(1)* %src );
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.c.row.stride.f16.p1i8(i8 addrspace(1)* %src , i32 %stride);
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_c_row_stride_f16_p1i8(i8 addrspace(1)* %src , i32 %stride) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_c_row_stride_f16_p1i8(
+; CHECK-SAME: ptr addrspace(1) [[SRC:%.*]], i32 [[STRIDE:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.c.row.stride.f16.p1i8(i8 addrspace(1)* %src , i32 %stride);
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.c.col.f16.p0i8(i8 addrspace(0)* %src );
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_c_col_f16_p0i8(i8 addrspace(0)* %src ) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_c_col_f16_p0i8(
+; CHECK-SAME: ptr [[SRC:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.c.col.f16.p0i8(i8 addrspace(0)* %src );
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.c.col.stride.f16.p0i8(i8 addrspace(0)* %src , i32 %stride);
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_c_col_stride_f16_p0i8(i8 addrspace(0)* %src , i32 %stride) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_c_col_stride_f16_p0i8(
+; CHECK-SAME: ptr [[SRC:%.*]], i32 [[STRIDE:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.c.col.stride.f16.p0i8(i8 addrspace(0)* %src , i32 %stride);
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.c.col.f16.p3i8(i8 addrspace(3)* %src );
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_c_col_f16_p3i8(i8 addrspace(3)* %src ) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_c_col_f16_p3i8(
+; CHECK-SAME: ptr addrspace(3) [[SRC:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.c.col.f16.p3i8(i8 addrspace(3)* %src );
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.c.col.stride.f16.p3i8(i8 addrspace(3)* %src , i32 %stride);
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_c_col_stride_f16_p3i8(i8 addrspace(3)* %src , i32 %stride) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_c_col_stride_f16_p3i8(
+; CHECK-SAME: ptr addrspace(3) [[SRC:%.*]], i32 [[STRIDE:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.c.col.stride.f16.p3i8(i8 addrspace(3)* %src , i32 %stride);
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.c.col.f16.p1i8(i8 addrspace(1)* %src );
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_c_col_f16_p1i8(i8 addrspace(1)* %src ) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_c_col_f16_p1i8(
+; CHECK-SAME: ptr addrspace(1) [[SRC:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.c.col.f16.p1i8(i8 addrspace(1)* %src );
+  ret void;
+}
+
+
+declare {<2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.c.col.stride.f16.p1i8(i8 addrspace(1)* %src , i32 %stride);
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_c_col_stride_f16_p1i8(i8 addrspace(1)* %src , i32 %stride) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_c_col_stride_f16_p1i8(
+; CHECK-SAME: ptr addrspace(1) [[SRC:%.*]], i32 [[STRIDE:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {<2 x half>, <2 x half>, <2 x half>, <2 x half>} @llvm.nvvm.wmma.m16n16k16.load.c.col.stride.f16.p1i8(i8 addrspace(1)* %src , i32 %stride);
+  ret void;
+}
+
+
+declare {float, float, float, float, float, float, float, float} @llvm.nvvm.wmma.m16n16k16.load.c.row.f32.p0i8(i8 addrspace(0)* %src );
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_c_row_f32_p0i8(i8 addrspace(0)* %src ) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_c_row_f32_p0i8(
+; CHECK-SAME: ptr [[SRC:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {float, float, float, float, float, float, float, float} @llvm.nvvm.wmma.m16n16k16.load.c.row.f32.p0i8(i8 addrspace(0)* %src );
+  ret void;
+}
+
+
+declare {float, float, float, float, float, float, float, float} @llvm.nvvm.wmma.m16n16k16.load.c.row.stride.f32.p0i8(i8 addrspace(0)* %src , i32 %stride);
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_c_row_stride_f32_p0i8(i8 addrspace(0)* %src , i32 %stride) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_c_row_stride_f32_p0i8(
+; CHECK-SAME: ptr [[SRC:%.*]], i32 [[STRIDE:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {float, float, float, float, float, float, float, float} @llvm.nvvm.wmma.m16n16k16.load.c.row.stride.f32.p0i8(i8 addrspace(0)* %src , i32 %stride);
+  ret void;
+}
+
+
+declare {float, float, float, float, float, float, float, float} @llvm.nvvm.wmma.m16n16k16.load.c.row.f32.p3i8(i8 addrspace(3)* %src );
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_c_row_f32_p3i8(i8 addrspace(3)* %src ) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_c_row_f32_p3i8(
+; CHECK-SAME: ptr addrspace(3) [[SRC:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {float, float, float, float, float, float, float, float} @llvm.nvvm.wmma.m16n16k16.load.c.row.f32.p3i8(i8 addrspace(3)* %src );
+  ret void;
+}
+
+
+declare {float, float, float, float, float, float, float, float} @llvm.nvvm.wmma.m16n16k16.load.c.row.stride.f32.p3i8(i8 addrspace(3)* %src , i32 %stride);
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_c_row_stride_f32_p3i8(i8 addrspace(3)* %src , i32 %stride) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_c_row_stride_f32_p3i8(
+; CHECK-SAME: ptr addrspace(3) [[SRC:%.*]], i32 [[STRIDE:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {float, float, float, float, float, float, float, float} @llvm.nvvm.wmma.m16n16k16.load.c.row.stride.f32.p3i8(i8 addrspace(3)* %src , i32 %stride);
+  ret void;
+}
+
+
+declare {float, float, float, float, float, float, float, float} @llvm.nvvm.wmma.m16n16k16.load.c.row.f32.p1i8(i8 addrspace(1)* %src );
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_c_row_f32_p1i8(i8 addrspace(1)* %src ) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_c_row_f32_p1i8(
+; CHECK-SAME: ptr addrspace(1) [[SRC:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {float, float, float, float, float, float, float, float} @llvm.nvvm.wmma.m16n16k16.load.c.row.f32.p1i8(i8 addrspace(1)* %src );
+  ret void;
+}
+
+
+declare {float, float, float, float, float, float, float, float} @llvm.nvvm.wmma.m16n16k16.load.c.row.stride.f32.p1i8(i8 addrspace(1)* %src , i32 %stride);
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_c_row_stride_f32_p1i8(i8 addrspace(1)* %src , i32 %stride) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_c_row_stride_f32_p1i8(
+; CHECK-SAME: ptr addrspace(1) [[SRC:%.*]], i32 [[STRIDE:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {float, float, float, float, float, float, float, float} @llvm.nvvm.wmma.m16n16k16.load.c.row.stride.f32.p1i8(i8 addrspace(1)* %src , i32 %stride);
+  ret void;
+}
+
+
+declare {float, float, float, float, float, float, float, float} @llvm.nvvm.wmma.m16n16k16.load.c.col.f32.p0i8(i8 addrspace(0)* %src );
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_c_col_f32_p0i8(i8 addrspace(0)* %src ) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_c_col_f32_p0i8(
+; CHECK-SAME: ptr [[SRC:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {float, float, float, float, float, float, float, float} @llvm.nvvm.wmma.m16n16k16.load.c.col.f32.p0i8(i8 addrspace(0)* %src );
+  ret void;
+}
+
+
+declare {float, float, float, float, float, float, float, float} @llvm.nvvm.wmma.m16n16k16.load.c.col.stride.f32.p0i8(i8 addrspace(0)* %src , i32 %stride);
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_c_col_stride_f32_p0i8(i8 addrspace(0)* %src , i32 %stride) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_c_col_stride_f32_p0i8(
+; CHECK-SAME: ptr [[SRC:%.*]], i32 [[STRIDE:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {float, float, float, float, float, float, float, float} @llvm.nvvm.wmma.m16n16k16.load.c.col.stride.f32.p0i8(i8 addrspace(0)* %src , i32 %stride);
+  ret void;
+}
+
+
+declare {float, float, float, float, float, float, float, float} @llvm.nvvm.wmma.m16n16k16.load.c.col.f32.p3i8(i8 addrspace(3)* %src );
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_c_col_f32_p3i8(i8 addrspace(3)* %src ) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_c_col_f32_p3i8(
+; CHECK-SAME: ptr addrspace(3) [[SRC:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {float, float, float, float, float, float, float, float} @llvm.nvvm.wmma.m16n16k16.load.c.col.f32.p3i8(i8 addrspace(3)* %src );
+  ret void;
+}
+
+
+declare {float, float, float, float, float, float, float, float} @llvm.nvvm.wmma.m16n16k16.load.c.col.stride.f32.p3i8(i8 addrspace(3)* %src , i32 %stride);
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_c_col_stride_f32_p3i8(i8 addrspace(3)* %src , i32 %stride) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_c_col_stride_f32_p3i8(
+; CHECK-SAME: ptr addrspace(3) [[SRC:%.*]], i32 [[STRIDE:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {float, float, float, float, float, float, float, float} @llvm.nvvm.wmma.m16n16k16.load.c.col.stride.f32.p3i8(i8 addrspace(3)* %src , i32 %stride);
+  ret void;
+}
+
+
+declare {float, float, float, float, float, float, float, float} @llvm.nvvm.wmma.m16n16k16.load.c.col.f32.p1i8(i8 addrspace(1)* %src );
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_c_col_f32_p1i8(i8 addrspace(1)* %src ) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_c_col_f32_p1i8(
+; CHECK-SAME: ptr addrspace(1) [[SRC:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {float, float, float, float, float, float, float, float} @llvm.nvvm.wmma.m16n16k16.load.c.col.f32.p1i8(i8 addrspace(1)* %src );
+  ret void;
+}
+
+
+declare {float, float, float, float, float, float, float, float} @llvm.nvvm.wmma.m16n16k16.load.c.col.stride.f32.p1i8(i8 addrspace(1)* %src , i32 %stride);
+
+define void @test_llvm_nvvm_wmma_m16n16k16_load_c_col_stride_f32_p1i8(i8 addrspace(1)* %src , i32 %stride) {
+; CHECK-LABEL: define void @test_llvm_nvvm_wmma_m16n16k16_load_c_col_stride_f32_p1i8(
+; CHECK-SAME: ptr addrspace(1) [[SRC:%.*]], i32 [[STRIDE:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call {float, float, float, float, float, float, float, float} @llvm.nvvm.wmma.m16n16k16.load.c.col.stride.f32.p1i8(i8 addrspace(1)* %src , i32 %stride);
+  ret void;
+}
+
+declare i8 @llvm.nvvm.ldu.global.i.i8.p1(ptr addrspace(1) %ptr, i32 %align)
+declare i16 @llvm.nvvm.ldu.global.i.i16.p1(ptr addrspace(1) %ptr, i32 %align)
+declare i32 @llvm.nvvm.ldu.global.i.i32.p1(ptr addrspace(1) %ptr, i32 %align)
+declare i64 @llvm.nvvm.ldu.global.i.i64.p1(ptr addrspace(1) %ptr, i32 %align)
+declare ptr @llvm.nvvm.ldu.global.p.p1(ptr addrspace(1) %ptr, i32 %align)
+declare float @llvm.nvvm.ldu.global.f.f32.p1(ptr addrspace(1) %ptr, i32 %align)
+declare double @llvm.nvvm.ldu.global.f.f64.p1(ptr addrspace(1) %ptr, i32 %align)
+declare half @llvm.nvvm.ldu.global.f.f16.p1(ptr addrspace(1) %ptr, i32 %align)
+declare <2 x half> @llvm.nvvm.ldu.global.f.v2f16.p1(ptr addrspace(1) %ptr, i32 %align)
+
+declare i8 @llvm.nvvm.ldg.global.i.i8.p1(ptr addrspace(1) %ptr, i32 %align)
+declare i16 @llvm.nvvm.ldg.global.i.i16.p1(ptr addrspace(1) %ptr, i32 %align)
+declare i32 @llvm.nvvm.ldg.global.i.i32.p1(ptr addrspace(1) %ptr, i32 %align)
+declare i64 @llvm.nvvm.ldg.global.i.i64.p1(ptr addrspace(1) %ptr, i32 %align)
+declare ptr @llvm.nvvm.ldg.global.p.p1(ptr addrspace(1) %ptr, i32 %align)
+declare float @llvm.nvvm.ldg.global.f.f32.p1(ptr addrspace(1) %ptr, i32 %align)
+declare double @llvm.nvvm.ldg.global.f.f64.p1(ptr addrspace(1) %ptr, i32 %align)
+declare half @llvm.nvvm.ldg.global.f.f16.p1(ptr addrspace(1) %ptr, i32 %align)
+declare <2 x half> @llvm.nvvm.ldg.global.f.v2f16.p1(ptr addrspace(1) %ptr, i32 %align)
+
+define void @test_ldu_i8_dead(ptr addrspace(1) %ptr) {
+; CHECK-LABEL: define void @test_ldu_i8_dead(
+; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %val = tail call i8 @llvm.nvvm.ldu.global.i.i8.p1(ptr addrspace(1) %ptr, i32 4)
+  ret void
+}
+
+define void @test_ldu_i16_dead(ptr addrspace(1) %ptr) {
+; CHECK-LABEL: define void @test_ldu_i16_dead(
+; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %val = tail call i16 @llvm.nvvm.ldu.global.i.i16.p1(ptr addrspace(1) %ptr, i32 2)
+  ret void
+}
+
+define void @test_ldu_i32_dead(ptr addrspace(1) %ptr) {
+; CHECK-LABEL: define void @test_ldu_i32_dead(
+; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %val = tail call i32 @llvm.nvvm.ldu.global.i.i32.p1(ptr addrspace(1) %ptr, i32 4)
+  ret void
+}
+
+define void @test_ldu_i64_dead(ptr addrspace(1) %ptr) {
+; CHECK-LABEL: define void @test_ldu_i64_dead(
+; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %val = tail call i64 @llvm.nvvm.ldu.global.i.i64.p1(ptr addrspace(1) %ptr, i32 8)
+  ret void
+}
+
+define void @test_ldu_p_dead(ptr addrspace(1) %ptr) {
+; CHECK-LABEL: define void @test_ldu_p_dead(
+; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %val = tail call ptr @llvm.nvvm.ldu.global.p.p1(ptr addrspace(1) %ptr, i32 8)
+  ret void
+}
+
+define void @test_ldu_f32_dead(ptr addrspace(1) %ptr) {
+; CHECK-LABEL: define void @test_ldu_f32_dead(
+; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %val = tail call float @llvm.nvvm.ldu.global.f.f32.p1(ptr addrspace(1) %ptr, i32 4)
+  ret void
+}
+
+define void @test_ldu_f64_dead(ptr addrspace(1) %ptr) {
+; CHECK-LABEL: define void @test_ldu_f64_dead(
+; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %val = tail call double @llvm.nvvm.ldu.global.f.f64.p1(ptr addrspace(1) %ptr, i32 8)
+  ret void
+}
+
+define void @test_ldu_f16_dead(ptr addrspace(1) %ptr) {
+; CHECK-LABEL: define void @test_ldu_f16_dead(
+; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %val = tail call half @llvm.nvvm.ldu.global.f.f16.p1(ptr addrspace(1) %ptr, i32 2)
+  ret void
+}
+
+define void @test_ldu_v2f16_dead(ptr addrspace(1) %ptr) {
+; CHECK-LABEL: define void @test_ldu_v2f16_dead(
+; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %val = tail call <2 x half> @llvm.nvvm.ldu.global.f.v2f16.p1(ptr addrspace(1) %ptr, i32 4)
+  ret void
+}
+
+define void @test_ldg_i8_dead(ptr addrspace(1) %ptr) {
+; CHECK-LABEL: define void @test_ldg_i8_dead(
+; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %val = tail call i8 @llvm.nvvm.ldg.global.i.i8.p1(ptr addrspace(1) %ptr, i32 4)
+  ret void
+}
+
+define void @test_ldg_i16_dead(ptr addrspace(1) %ptr) {
+; CHECK-LABEL: define void @test_ldg_i16_dead(
+; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %val = tail call i16 @llvm.nvvm.ldg.global.i.i16.p1(ptr addrspace(1) %ptr, i32 2)
+  ret void
+}
+
+define void @test_ldg_i32_dead(ptr addrspace(1) %ptr) {
+; CHECK-LABEL: define void @test_ldg_i32_dead(
+; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %val = tail call i32 @llvm.nvvm.ldg.global.i.i32.p1(ptr addrspace(1) %ptr, i32 4)
+  ret void
+}
+
+define void @test_ldg_i64_dead(ptr addrspace(1) %ptr) {
+; CHECK-LABEL: define void @test_ldg_i64_dead(
+; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %val = tail call i64 @llvm.nvvm.ldg.global.i.i64.p1(ptr addrspace(1) %ptr, i32 8)
+  ret void
+}
+
+define void @test_ldg_p_dead(ptr addrspace(1) %ptr) {
+; CHECK-LABEL: define void @test_ldg_p_dead(
+; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %val = tail call ptr @llvm.nvvm.ldg.global.p.p1(ptr addrspace(1) %ptr, i32 8)
+  ret void
+}
+
+define void @test_ldg_f32_dead(ptr addrspace(1) %ptr) {
+; CHECK-LABEL: define void @test_ldg_f32_dead(
+; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %val = tail call float @llvm.nvvm.ldg.global.f.f32.p1(ptr addrspace(1) %ptr, i32 4)
+  ret void
+}
+
+define void @test_ldg_f64_dead(ptr addrspace(1) %ptr) {
+; CHECK-LABEL: define void @test_ldg_f64_dead(
+; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %val = tail call double @llvm.nvvm.ldg.global.f.f64.p1(ptr addrspace(1) %ptr, i32 8)
+  ret void
+}
+
+define void @test_ldg_f16_dead(ptr addrspace(1) %ptr) {
+; CHECK-LABEL: define void @test_ldg_f16_dead(
+; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %val = tail call half @llvm.nvvm.ldg.global.f.f16.p1(ptr addrspace(1) %ptr, i32 2)
+  ret void
+}
+
+define void @test_ldg_v2f16_dead(ptr addrspace(1) %ptr) {
+; CHECK-LABEL: define void @test_ldg_v2f16_dead(
+; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %val = tail call <2 x half> @llvm.nvvm.ldg.global.f.v2f16.p1(ptr addrspace(1) %ptr, i32 4)
+  ret void
+}
diff --git a/llvm/test/Transforms/DCE/intrinsics-systemz.ll b/llvm/test/Transforms/DCE/intrinsics-systemz.ll
new file mode 100644
index 000000000000000..d54d8d02cbb78d0
--- /dev/null
+++ b/llvm/test/Transforms/DCE/intrinsics-systemz.ll
@@ -0,0 +1,33 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S < %s -passes=dce  | FileCheck %s
+
+declare <16 x i8> @llvm.s390.vlbb(ptr, i32)
+declare <16 x i8> @llvm.s390.vll(i32, ptr)
+declare <16 x i8> @llvm.s390.vlrl(i32, ptr)
+
+define void @test_llvm_s390.vlbb(ptr %a) {
+; CHECK-LABEL: define void @test_llvm_s390.vlbb(
+; CHECK-SAME: ptr [[A:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <16 x i8> @llvm.s390.vlbb(ptr %a, i32 8)
+  ret void
+}
+
+define void @test_llvm_s390.vll(i32 %a, ptr %b) {
+; CHECK-LABEL: define void @test_llvm_s390.vll(
+; CHECK-SAME: i32 [[A:%.*]], ptr [[B:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <16 x i8> @llvm.s390.vll(i32 %a, ptr %b)
+  ret void
+}
+
+define void @test_llvm_s390.vlrl(i32 %a, ptr %b) {
+; CHECK-LABEL: define void @test_llvm_s390.vlrl(
+; CHECK-SAME: i32 [[A:%.*]], ptr [[B:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <16 x i8> @llvm.s390.vlrl(i32 %a, ptr %b)
+  ret void
+}
diff --git a/llvm/test/Transforms/DCE/intrinsics-ve.ll b/llvm/test/Transforms/DCE/intrinsics-ve.ll
new file mode 100644
index 000000000000000..5c6d207587f8e4d
--- /dev/null
+++ b/llvm/test/Transforms/DCE/intrinsics-ve.ll
@@ -0,0 +1,729 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S < %s -passes=dce  | FileCheck %s
+
+declare i64 @llvm.ve.vl.pack.f32p(ptr, ptr)
+
+define void @test_llvm_ve_vl_pack_f32p(ptr %a, ptr %b) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_pack_f32p(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call i64 @llvm.ve.vl.pack.f32p(ptr %a, ptr %b)
+  ret void
+}
+
+
+declare i64 @llvm.ve.vl.pack.f32a(ptr)
+
+define void @test_llvm_ve_vl_pack_f32a(ptr %a) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_pack_f32a(
+; CHECK-SAME: ptr [[A:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call i64 @llvm.ve.vl.pack.f32a(ptr %a)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vld.vssl(i64, ptr, i32)
+
+define void @test_llvm_ve_vl_vld_vssl(i64 %a, ptr %b, i32 %c) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vld_vssl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], i32 [[C:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vld.vssl(i64 %a, ptr %b, i32 %c)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vld.vssvl(i64, ptr, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vld_vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vld_vssvl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], <256 x double> [[C:%.*]], i32 [[D:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vld.vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vldnc.vssl(i64, ptr, i32)
+
+define void @test_llvm_ve_vl_vldnc_vssl(i64 %a, ptr %b, i32 %c) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vldnc_vssl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], i32 [[C:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vldnc.vssl(i64 %a, ptr %b, i32 %c)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vldnc.vssvl(i64, ptr, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vldnc_vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vldnc_vssvl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], <256 x double> [[C:%.*]], i32 [[D:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vldnc.vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vldu.vssl(i64, ptr, i32)
+
+define void @test_llvm_ve_vl_vldu_vssl(i64 %a, ptr %b, i32 %c) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vldu_vssl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], i32 [[C:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vldu.vssl(i64 %a, ptr %b, i32 %c)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vldu.vssvl(i64, ptr, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vldu_vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vldu_vssvl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], <256 x double> [[C:%.*]], i32 [[D:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vldu.vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vldunc.vssl(i64, ptr, i32)
+
+define void @test_llvm_ve_vl_vldunc_vssl(i64 %a, ptr %b, i32 %c) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vldunc_vssl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], i32 [[C:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vldunc.vssl(i64 %a, ptr %b, i32 %c)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vldunc.vssvl(i64, ptr, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vldunc_vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vldunc_vssvl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], <256 x double> [[C:%.*]], i32 [[D:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vldunc.vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vldlsx.vssl(i64, ptr, i32)
+
+define void @test_llvm_ve_vl_vldlsx_vssl(i64 %a, ptr %b, i32 %c) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vldlsx_vssl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], i32 [[C:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vldlsx.vssl(i64 %a, ptr %b, i32 %c)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vldlsx.vssvl(i64, ptr, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vldlsx_vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vldlsx_vssvl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], <256 x double> [[C:%.*]], i32 [[D:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vldlsx.vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vldlsxnc.vssl(i64, ptr, i32)
+
+define void @test_llvm_ve_vl_vldlsxnc_vssl(i64 %a, ptr %b, i32 %c) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vldlsxnc_vssl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], i32 [[C:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vldlsxnc.vssl(i64 %a, ptr %b, i32 %c)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vldlsxnc.vssvl(i64, ptr, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vldlsxnc_vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vldlsxnc_vssvl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], <256 x double> [[C:%.*]], i32 [[D:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vldlsxnc.vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vldlzx.vssl(i64, ptr, i32)
+
+define void @test_llvm_ve_vl_vldlzx_vssl(i64 %a, ptr %b, i32 %c) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vldlzx_vssl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], i32 [[C:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vldlzx.vssl(i64 %a, ptr %b, i32 %c)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vldlzx.vssvl(i64, ptr, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vldlzx_vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vldlzx_vssvl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], <256 x double> [[C:%.*]], i32 [[D:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vldlzx.vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vldlzxnc.vssl(i64, ptr, i32)
+
+define void @test_llvm_ve_vl_vldlzxnc_vssl(i64 %a, ptr %b, i32 %c) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vldlzxnc_vssl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], i32 [[C:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vldlzxnc.vssl(i64 %a, ptr %b, i32 %c)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vldlzxnc.vssvl(i64, ptr, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vldlzxnc_vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vldlzxnc_vssvl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], <256 x double> [[C:%.*]], i32 [[D:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vldlzxnc.vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vld2d.vssl(i64, ptr, i32)
+
+define void @test_llvm_ve_vl_vld2d_vssl(i64 %a, ptr %b, i32 %c) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vld2d_vssl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], i32 [[C:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vld2d.vssl(i64 %a, ptr %b, i32 %c)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vld2d.vssvl(i64, ptr, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vld2d_vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vld2d_vssvl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], <256 x double> [[C:%.*]], i32 [[D:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vld2d.vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vld2dnc.vssl(i64, ptr, i32)
+
+define void @test_llvm_ve_vl_vld2dnc_vssl(i64 %a, ptr %b, i32 %c) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vld2dnc_vssl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], i32 [[C:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vld2dnc.vssl(i64 %a, ptr %b, i32 %c)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vld2dnc.vssvl(i64, ptr, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vld2dnc_vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vld2dnc_vssvl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], <256 x double> [[C:%.*]], i32 [[D:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vld2dnc.vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vldu2d.vssl(i64, ptr, i32)
+
+define void @test_llvm_ve_vl_vldu2d_vssl(i64 %a, ptr %b, i32 %c) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vldu2d_vssl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], i32 [[C:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vldu2d.vssl(i64 %a, ptr %b, i32 %c)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vldu2d.vssvl(i64, ptr, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vldu2d_vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vldu2d_vssvl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], <256 x double> [[C:%.*]], i32 [[D:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vldu2d.vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vldu2dnc.vssl(i64, ptr, i32)
+
+define void @test_llvm_ve_vl_vldu2dnc_vssl(i64 %a, ptr %b, i32 %c) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vldu2dnc_vssl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], i32 [[C:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vldu2dnc.vssl(i64 %a, ptr %b, i32 %c)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vldu2dnc.vssvl(i64, ptr, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vldu2dnc_vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vldu2dnc_vssvl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], <256 x double> [[C:%.*]], i32 [[D:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vldu2dnc.vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vldl2dsx.vssl(i64, ptr, i32)
+
+define void @test_llvm_ve_vl_vldl2dsx_vssl(i64 %a, ptr %b, i32 %c) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vldl2dsx_vssl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], i32 [[C:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vldl2dsx.vssl(i64 %a, ptr %b, i32 %c)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vldl2dsx.vssvl(i64, ptr, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vldl2dsx_vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vldl2dsx_vssvl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], <256 x double> [[C:%.*]], i32 [[D:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vldl2dsx.vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vldl2dsxnc.vssl(i64, ptr, i32)
+
+define void @test_llvm_ve_vl_vldl2dsxnc_vssl(i64 %a, ptr %b, i32 %c) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vldl2dsxnc_vssl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], i32 [[C:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vldl2dsxnc.vssl(i64 %a, ptr %b, i32 %c)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vldl2dsxnc.vssvl(i64, ptr, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vldl2dsxnc_vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vldl2dsxnc_vssvl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], <256 x double> [[C:%.*]], i32 [[D:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vldl2dsxnc.vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vldl2dzx.vssl(i64, ptr, i32)
+
+define void @test_llvm_ve_vl_vldl2dzx_vssl(i64 %a, ptr %b, i32 %c) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vldl2dzx_vssl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], i32 [[C:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vldl2dsxnc.vssl(i64 %a, ptr %b, i32 %c)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vldl2dzx.vssvl(i64, ptr, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vldl2dzx_vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vldl2dzx_vssvl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], <256 x double> [[C:%.*]], i32 [[D:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vldl2dzx.vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vldl2dzxnc.vssl(i64, ptr, i32)
+
+define void @test_llvm_ve_vl_vldl2dzxnc_vssl(i64 %a, ptr %b, i32 %c) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vldl2dzxnc_vssl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], i32 [[C:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vldl2dzxnc.vssl(i64 %a, ptr %b, i32 %c)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vldl2dzxnc.vssvl(i64, ptr, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vldl2dzxnc_vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vldl2dzxnc_vssvl(
+; CHECK-SAME: i64 [[A:%.*]], ptr [[B:%.*]], <256 x double> [[C:%.*]], i32 [[D:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vldl2dzxnc.vssvl(i64 %a, ptr %b, <256 x double> %c, i32 %d)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgt.vvssl(<256 x double>, i64, i64, i32)
+
+define void @test_llvm_ve_vl_vgt_vvssl(<256 x double> %a, i64 %b, i64 %c, i32 %d) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgt_vvssl(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], i32 [[D:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgt.vvssl(<256 x double> %a, i64 %b, i64 %c, i32 %d)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgt.vvssvl(<256 x double>, i64, i64, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vgt_vvssvl(<256 x double> %a, i64 %b, i64 %c, <256 x double> %d, i32 %e) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgt_vvssvl(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], <256 x double> [[D:%.*]], i32 [[E:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgt.vvssvl(<256 x double> %a, i64 %b, i64 %c, <256 x double> %d, i32 %e)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgt.vvssml(<256 x double>, i64, i64, <256 x i1>, i32)
+
+define void @test_llvm_ve_vl_vgt_vvssml(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, i32 %e) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgt_vvssml(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], <256 x i1> [[D:%.*]], i32 [[E:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgt.vvssml(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, i32 %e)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgt.vvssmvl(<256 x double>, i64, i64, <256 x i1>, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vgt_vvssmvl(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, <256 x double> %e, i32 %f) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgt_vvssmvl(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], <256 x i1> [[D:%.*]], <256 x double> [[E:%.*]], i32 [[F:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgt.vvssmvl(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, <256 x double> %e, i32 %f)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgtnc.vvssl(<256 x double>, i64, i64, i32)
+
+define void @test_llvm_ve_vl_vgtnc_vvssl(<256 x double> %a, i64 %b, i64 %c, i32 %d) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgtnc_vvssl(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], i32 [[D:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgtnc.vvssl(<256 x double> %a, i64 %b, i64 %c, i32 %d)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgtnc.vvssvl(<256 x double>, i64, i64, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vgtnc_vvssvl(<256 x double> %a, i64 %b, i64 %c, <256 x double> %d, i32 %e) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgtnc_vvssvl(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], <256 x double> [[D:%.*]], i32 [[E:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgtnc.vvssvl(<256 x double> %a, i64 %b, i64 %c, <256 x double> %d, i32 %e)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgtnc.vvssml(<256 x double>, i64, i64, <256 x i1>, i32)
+
+define void @test_llvm_ve_vl_vgtnc_vvssml(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, i32 %e) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgtnc_vvssml(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], <256 x i1> [[D:%.*]], i32 [[E:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgtnc.vvssml(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, i32 %e)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgtnc.vvssmvl(<256 x double>, i64, i64, <256 x i1>, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vgtnc_vvssmvl(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, <256 x double> %e, i32 %f) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgtnc_vvssmvl(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], <256 x i1> [[D:%.*]], <256 x double> [[E:%.*]], i32 [[F:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgtnc.vvssmvl(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, <256 x double> %e, i32 %f)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgtu.vvssl(<256 x double>, i64, i64, i32)
+
+define void @test_llvm_ve_vl_vgtu_vvssl(<256 x double> %a, i64 %b, i64 %c, i32 %d) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgtu_vvssl(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], i32 [[D:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgtu.vvssl(<256 x double> %a, i64 %b, i64 %c, i32 %d)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgtu.vvssvl(<256 x double>, i64, i64, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vgtu_vvssvl(<256 x double> %a, i64 %b, i64 %c, <256 x double> %d, i32 %e) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgtu_vvssvl(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], <256 x double> [[D:%.*]], i32 [[E:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgtu.vvssvl(<256 x double> %a, i64 %b, i64 %c, <256 x double> %d, i32 %e)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgtu.vvssml(<256 x double>, i64, i64, <256 x i1>, i32)
+
+define void @test_llvm_ve_vl_vgtu_vvssml(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, i32 %e) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgtu_vvssml(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], <256 x i1> [[D:%.*]], i32 [[E:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgtu.vvssml(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, i32 %e)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgtu.vvssmvl(<256 x double>, i64, i64, <256 x i1>, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vgtu_vvssmvl(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, <256 x double> %e, i32 %f) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgtu_vvssmvl(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], <256 x i1> [[D:%.*]], <256 x double> [[E:%.*]], i32 [[F:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgtu.vvssmvl(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, <256 x double> %e, i32 %f)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgtunc.vvssl(<256 x double>, i64, i64, i32)
+
+define void @test_llvm_ve_vl_vgtunc_vvssl(<256 x double> %a, i64 %b, i64 %c, i32 %d) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgtunc_vvssl(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], i32 [[D:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgtunc.vvssl(<256 x double> %a, i64 %b, i64 %c, i32 %d)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgtunc.vvssvl(<256 x double>, i64, i64, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vgtunc_vvssvl(<256 x double> %a, i64 %b, i64 %c, <256 x double> %d, i32 %e) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgtunc_vvssvl(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], <256 x double> [[D:%.*]], i32 [[E:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgtunc.vvssvl(<256 x double> %a, i64 %b, i64 %c, <256 x double> %d, i32 %e)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgtunc.vvssml(<256 x double>, i64, i64, <256 x i1>, i32)
+
+define void @test_llvm_ve_vl_vgtunc_vvssml(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, i32 %e) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgtunc_vvssml(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], <256 x i1> [[D:%.*]], i32 [[E:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgtunc.vvssml(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, i32 %e)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgtunc.vvssmvl(<256 x double>, i64, i64, <256 x i1>, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vgtunc_vvssmvl(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, <256 x double> %e, i32 %f) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgtunc_vvssmvl(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], <256 x i1> [[D:%.*]], <256 x double> [[E:%.*]], i32 [[F:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgtunc.vvssmvl(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, <256 x double> %e, i32 %f)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgtlsx.vvssl(<256 x double>, i64, i64, i32)
+
+define void @test_llvm_ve_vl_vgtlsx_vvssl(<256 x double> %a, i64 %b, i64 %c, i32 %d) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgtlsx_vvssl(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], i32 [[D:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgtlsx.vvssl(<256 x double> %a, i64 %b, i64 %c, i32 %d)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgtlsx.vvssvl(<256 x double>, i64, i64, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vgtlsx_vvssvl(<256 x double> %a, i64 %b, i64 %c, <256 x double> %d, i32 %e) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgtlsx_vvssvl(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], <256 x double> [[D:%.*]], i32 [[E:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgtlsx.vvssvl(<256 x double> %a, i64 %b, i64 %c, <256 x double> %d, i32 %e)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgtlsx.vvssml(<256 x double>, i64, i64, <256 x i1>, i32)
+
+define void @test_llvm_ve_vl_vgtlsx_vvssml(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, i32 %e) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgtlsx_vvssml(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], <256 x i1> [[D:%.*]], i32 [[E:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgtlsx.vvssml(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, i32 %e)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgtlsx.vvssmvl(<256 x double>, i64, i64, <256 x i1>, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vgtlsx_vvssmvl(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, <256 x double> %e, i32 %f) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgtlsx_vvssmvl(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], <256 x i1> [[D:%.*]], <256 x double> [[E:%.*]], i32 [[F:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgtlsx.vvssmvl(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, <256 x double> %e, i32 %f)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgtlsxnc.vvssl(<256 x double>, i64, i64, i32)
+
+define void @test_llvm_ve_vl_vgtlsxnc_vvssl(<256 x double> %a, i64 %b, i64 %c, i32 %d) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgtlsxnc_vvssl(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], i32 [[D:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgtlsxnc.vvssl(<256 x double> %a, i64 %b, i64 %c, i32 %d)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgtlsxnc.vvssvl(<256 x double>, i64, i64, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vgtlsxnc_vvssvl(<256 x double> %a, i64 %b, i64 %c, <256 x double> %d, i32 %e) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgtlsxnc_vvssvl(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], <256 x double> [[D:%.*]], i32 [[E:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgtlsxnc.vvssvl(<256 x double> %a, i64 %b, i64 %c, <256 x double> %d, i32 %e)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgtlsxnc.vvssml(<256 x double>, i64, i64, <256 x i1>, i32)
+
+define void @test_llvm_ve_vl_vgtlsxnc_vvssml(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, i32 %e) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgtlsxnc_vvssml(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], <256 x i1> [[D:%.*]], i32 [[E:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgtlsxnc.vvssml(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, i32 %e)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgtlsxnc.vvssmvl(<256 x double>, i64, i64, <256 x i1>, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vgtlsxnc_vvssmvl(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, <256 x double> %e, i32 %f) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgtlsxnc_vvssmvl(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], <256 x i1> [[D:%.*]], <256 x double> [[E:%.*]], i32 [[F:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgtlsxnc.vvssmvl(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, <256 x double> %e, i32 %f)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgtlzx.vvssl(<256 x double>, i64, i64, i32)
+
+define void @test_llvm_ve_vl_vgtlzx_vvssl(<256 x double> %a, i64 %b, i64 %c, i32 %d) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgtlzx_vvssl(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], i32 [[D:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgtlzx.vvssl(<256 x double> %a, i64 %b, i64 %c, i32 %d)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgtlzx.vvssvl(<256 x double>, i64, i64, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vgtlzx_vvssvl(<256 x double> %a, i64 %b, i64 %c, <256 x double> %d, i32 %e) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgtlzx_vvssvl(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], <256 x double> [[D:%.*]], i32 [[E:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgtlzx.vvssvl(<256 x double> %a, i64 %b, i64 %c, <256 x double> %d, i32 %e)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgtlzx.vvssml(<256 x double>, i64, i64, <256 x i1>, i32)
+
+define void @test_llvm_ve_vl_vgtlzx_vvssml(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, i32 %e) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgtlzx_vvssml(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], <256 x i1> [[D:%.*]], i32 [[E:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgtlzx.vvssml(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, i32 %e)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgtlzx.vvssmvl(<256 x double>, i64, i64, <256 x i1>, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vgtlzx_vvssmvl(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, <256 x double> %e, i32 %f) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgtlzx_vvssmvl(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], <256 x i1> [[D:%.*]], <256 x double> [[E:%.*]], i32 [[F:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgtlzx.vvssmvl(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, <256 x double> %e, i32 %f)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgtlzxnc.vvssl(<256 x double>, i64, i64, i32)
+
+define void @test_llvm_ve_vl_vgtlzxnc_vvssl(<256 x double> %a, i64 %b, i64 %c, i32 %d) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgtlzxnc_vvssl(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], i32 [[D:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgtlzxnc.vvssl(<256 x double> %a, i64 %b, i64 %c, i32 %d)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgtlzxnc.vvssvl(<256 x double>, i64, i64, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vgtlzxnc_vvssvl(<256 x double> %a, i64 %b, i64 %c, <256 x double> %d, i32 %e) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgtlzxnc_vvssvl(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], <256 x double> [[D:%.*]], i32 [[E:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgtlzxnc.vvssvl(<256 x double> %a, i64 %b, i64 %c, <256 x double> %d, i32 %e)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgtlzxnc.vvssml(<256 x double>, i64, i64, <256 x i1>, i32)
+
+define void @test_llvm_ve_vl_vgtlzxnc_vvssml(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, i32 %e) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgtlzxnc_vvssml(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], <256 x i1> [[D:%.*]], i32 [[E:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgtlzxnc.vvssml(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, i32 %e)
+  ret void
+}
+
+declare <256 x double> @llvm.ve.vl.vgtlzxnc.vvssmvl(<256 x double>, i64, i64, <256 x i1>, <256 x double>, i32)
+
+define void @test_llvm_ve_vl_vgtlzxnc_vvssmvl(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, <256 x double> %e, i32 %f) {
+; CHECK-LABEL: define void @test_llvm_ve_vl_vgtlzxnc_vvssmvl(
+; CHECK-SAME: <256 x double> [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]], <256 x i1> [[D:%.*]], <256 x double> [[E:%.*]], i32 [[F:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call <256 x double> @llvm.ve.vl.vgtlzxnc.vvssmvl(<256 x double> %a, i64 %b, i64 %c, <256 x i1> %d, <256 x double> %e, i32 %f)
+  ret void
+}
diff --git a/llvm/test/Transforms/DCE/intrinsics-wasm.ll b/llvm/test/Transforms/DCE/intrinsics-wasm.ll
new file mode 100644
index 000000000000000..8d6a60965a46d12
--- /dev/null
+++ b/llvm/test/Transforms/DCE/intrinsics-wasm.ll
@@ -0,0 +1,13 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S < %s -passes=dce  | FileCheck %s
+
+declare float @llvm.wasm.loadf16.f32(ptr)
+
+define void @test_llvm_wasm_loadf16_f32(ptr %a) {
+; CHECK-LABEL: define void @test_llvm_wasm_loadf16_f32(
+; CHECK-SAME: ptr [[A:%.*]]) {
+; CHECK-NEXT:    ret void
+;
+  %v = call float @llvm.wasm.loadf16.f32(ptr %a)
+  ret void
+}
diff --git a/llvm/test/Transforms/DCE/nvvm-ldu-ldg-willreturn.ll b/llvm/test/Transforms/DCE/nvvm-ldu-ldg-willreturn.ll
deleted file mode 100644
index 64a023ef4513768..000000000000000
--- a/llvm/test/Transforms/DCE/nvvm-ldu-ldg-willreturn.ll
+++ /dev/null
@@ -1,187 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -S < %s -passes=dce  | FileCheck %s
-
-; ldu/ldg intrinsics were erroneously not marked IntrWillReturn, preventing
-; them from being eliminated at IR level when dead.
-
-declare i8 @llvm.nvvm.ldu.global.i.i8.p1(ptr addrspace(1) %ptr, i32 %align)
-declare i16 @llvm.nvvm.ldu.global.i.i16.p1(ptr addrspace(1) %ptr, i32 %align)
-declare i32 @llvm.nvvm.ldu.global.i.i32.p1(ptr addrspace(1) %ptr, i32 %align)
-declare i64 @llvm.nvvm.ldu.global.i.i64.p1(ptr addrspace(1) %ptr, i32 %align)
-declare ptr @llvm.nvvm.ldu.global.p.p1(ptr addrspace(1) %ptr, i32 %align)
-declare float @llvm.nvvm.ldu.global.f.f32.p1(ptr addrspace(1) %ptr, i32 %align)
-declare double @llvm.nvvm.ldu.global.f.f64.p1(ptr addrspace(1) %ptr, i32 %align)
-declare half @llvm.nvvm.ldu.global.f.f16.p1(ptr addrspace(1) %ptr, i32 %align)
-declare <2 x half> @llvm.nvvm.ldu.global.f.v2f16.p1(ptr addrspace(1) %ptr, i32 %align)
-
-declare i8 @llvm.nvvm.ldg.global.i.i8.p1(ptr addrspace(1) %ptr, i32 %align)
-declare i16 @llvm.nvvm.ldg.global.i.i16.p1(ptr addrspace(1) %ptr, i32 %align)
-declare i32 @llvm.nvvm.ldg.global.i.i32.p1(ptr addrspace(1) %ptr, i32 %align)
-declare i64 @llvm.nvvm.ldg.global.i.i64.p1(ptr addrspace(1) %ptr, i32 %align)
-declare ptr @llvm.nvvm.ldg.global.p.p1(ptr addrspace(1) %ptr, i32 %align)
-declare float @llvm.nvvm.ldg.global.f.f32.p1(ptr addrspace(1) %ptr, i32 %align)
-declare double @llvm.nvvm.ldg.global.f.f64.p1(ptr addrspace(1) %ptr, i32 %align)
-declare half @llvm.nvvm.ldg.global.f.f16.p1(ptr addrspace(1) %ptr, i32 %align)
-declare <2 x half> @llvm.nvvm.ldg.global.f.v2f16.p1(ptr addrspace(1) %ptr, i32 %align)
-
-define void @test_ldu_i8_dead(ptr addrspace(1) %ptr) {
-; CHECK-LABEL: define void @test_ldu_i8_dead(
-; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
-; CHECK-NEXT:    ret void
-;
-  %val = tail call i8 @llvm.nvvm.ldu.global.i.i8.p1(ptr addrspace(1) %ptr, i32 4)
-  ret void
-}
-
-define void @test_ldu_i16_dead(ptr addrspace(1) %ptr) {
-; CHECK-LABEL: define void @test_ldu_i16_dead(
-; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
-; CHECK-NEXT:    ret void
-;
-  %val = tail call i16 @llvm.nvvm.ldu.global.i.i16.p1(ptr addrspace(1) %ptr, i32 2)
-  ret void
-}
-
-define void @test_ldu_i32_dead(ptr addrspace(1) %ptr) {
-; CHECK-LABEL: define void @test_ldu_i32_dead(
-; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
-; CHECK-NEXT:    ret void
-;
-  %val = tail call i32 @llvm.nvvm.ldu.global.i.i32.p1(ptr addrspace(1) %ptr, i32 4)
-  ret void
-}
-
-define void @test_ldu_i64_dead(ptr addrspace(1) %ptr) {
-; CHECK-LABEL: define void @test_ldu_i64_dead(
-; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
-; CHECK-NEXT:    ret void
-;
-  %val = tail call i64 @llvm.nvvm.ldu.global.i.i64.p1(ptr addrspace(1) %ptr, i32 8)
-  ret void
-}
-
-define void @test_ldu_p_dead(ptr addrspace(1) %ptr) {
-; CHECK-LABEL: define void @test_ldu_p_dead(
-; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
-; CHECK-NEXT:    ret void
-;
-  %val = tail call ptr @llvm.nvvm.ldu.global.p.p1(ptr addrspace(1) %ptr, i32 8)
-  ret void
-}
-
-define void @test_ldu_f32_dead(ptr addrspace(1) %ptr) {
-; CHECK-LABEL: define void @test_ldu_f32_dead(
-; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
-; CHECK-NEXT:    ret void
-;
-  %val = tail call float @llvm.nvvm.ldu.global.f.f32.p1(ptr addrspace(1) %ptr, i32 4)
-  ret void
-}
-
-define void @test_ldu_f64_dead(ptr addrspace(1) %ptr) {
-; CHECK-LABEL: define void @test_ldu_f64_dead(
-; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
-; CHECK-NEXT:    ret void
-;
-  %val = tail call double @llvm.nvvm.ldu.global.f.f64.p1(ptr addrspace(1) %ptr, i32 8)
-  ret void
-}
-
-define void @test_ldu_f16_dead(ptr addrspace(1) %ptr) {
-; CHECK-LABEL: define void @test_ldu_f16_dead(
-; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
-; CHECK-NEXT:    ret void
-;
-  %val = tail call half @llvm.nvvm.ldu.global.f.f16.p1(ptr addrspace(1) %ptr, i32 2)
-  ret void
-}
-
-define void @test_ldu_v2f16_dead(ptr addrspace(1) %ptr) {
-; CHECK-LABEL: define void @test_ldu_v2f16_dead(
-; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
-; CHECK-NEXT:    ret void
-;
-  %val = tail call <2 x half> @llvm.nvvm.ldu.global.f.v2f16.p1(ptr addrspace(1) %ptr, i32 4)
-  ret void
-}
-
-define void @test_ldg_i8_dead(ptr addrspace(1) %ptr) {
-; CHECK-LABEL: define void @test_ldg_i8_dead(
-; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
-; CHECK-NEXT:    ret void
-;
-  %val = tail call i8 @llvm.nvvm.ldg.global.i.i8.p1(ptr addrspace(1) %ptr, i32 4)
-  ret void
-}
-
-define void @test_ldg_i16_dead(ptr addrspace(1) %ptr) {
-; CHECK-LABEL: define void @test_ldg_i16_dead(
-; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
-; CHECK-NEXT:    ret void
-;
-  %val = tail call i16 @llvm.nvvm.ldg.global.i.i16.p1(ptr addrspace(1) %ptr, i32 2)
-  ret void
-}
-
-define void @test_ldg_i32_dead(ptr addrspace(1) %ptr) {
-; CHECK-LABEL: define void @test_ldg_i32_dead(
-; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
-; CHECK-NEXT:    ret void
-;
-  %val = tail call i32 @llvm.nvvm.ldg.global.i.i32.p1(ptr addrspace(1) %ptr, i32 4)
-  ret void
-}
-
-define void @test_ldg_i64_dead(ptr addrspace(1) %ptr) {
-; CHECK-LABEL: define void @test_ldg_i64_dead(
-; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
-; CHECK-NEXT:    ret void
-;
-  %val = tail call i64 @llvm.nvvm.ldg.global.i.i64.p1(ptr addrspace(1) %ptr, i32 8)
-  ret void
-}
-
-define void @test_ldg_p_dead(ptr addrspace(1) %ptr) {
-; CHECK-LABEL: define void @test_ldg_p_dead(
-; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
-; CHECK-NEXT:    ret void
-;
-  %val = tail call ptr @llvm.nvvm.ldg.global.p.p1(ptr addrspace(1) %ptr, i32 8)
-  ret void
-}
-
-define void @test_ldg_f32_dead(ptr addrspace(1) %ptr) {
-; CHECK-LABEL: define void @test_ldg_f32_dead(
-; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
-; CHECK-NEXT:    ret void
-;
-  %val = tail call float @llvm.nvvm.ldg.global.f.f32.p1(ptr addrspace(1) %ptr, i32 4)
-  ret void
-}
-
-define void @test_ldg_f64_dead(ptr addrspace(1) %ptr) {
-; CHECK-LABEL: define void @test_ldg_f64_dead(
-; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
-; CHECK-NEXT:    ret void
-;
-  %val = tail call double @llvm.nvvm.ldg.global.f.f64.p1(ptr addrspace(1) %ptr, i32 8)
-  ret void
-}
-
-define void @test_ldg_f16_dead(ptr addrspace(1) %ptr) {
-; CHECK-LABEL: define void @test_ldg_f16_dead(
-; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
-; CHECK-NEXT:    ret void
-;
-  %val = tail call half @llvm.nvvm.ldg.global.f.f16.p1(ptr addrspace(1) %ptr, i32 2)
-  ret void
-}
-
-define void @test_ldg_v2f16_dead(ptr addrspace(1) %ptr) {
-; CHECK-LABEL: define void @test_ldg_v2f16_dead(
-; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
-; CHECK-NEXT:    ret void
-;
-  %val = tail call <2 x half> @llvm.nvvm.ldg.global.f.v2f16.p1(ptr addrspace(1) %ptr, i32 4)
-  ret void
-}

From 290e4858fd3d1278ca8306da351f7ab89240306e Mon Sep 17 00:00:00 2001
From: earnol <earnol@users.noreply.github.com>
Date: Thu, 15 Aug 2024 16:48:02 -0400
Subject: [PATCH 081/441] [ubsan] Limit _BitInt ubsan tests to x86-64 platform
 only (#104494)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The patch https://github.com/llvm/llvm-project/pull/104462 broke Sun
Solaris build.
Limit the tests to the tested platform only.
Tests changed are
compiler-rt/test/ubsan/TestCases/Integer/bit-int-pass.c and
compiler-rt/test/ubsan/TestCases/Integer/bit-int.c

---------

Co-authored-by: Eänolituri Lómitaurë <vladislav.aranov@ericsson.com>
Co-authored-by: Aaron Ballman <aaron@aaronballman.com>
Co-authored-by: Paul Kirth <paulkirth@google.com>
Co-authored-by: Vitaly Buka <vitalybuka@gmail.com>
---
 compiler-rt/test/ubsan/TestCases/Integer/bit-int-pass.c | 1 +
 compiler-rt/test/ubsan/TestCases/Integer/bit-int.c      | 4 ++++
 2 files changed, 5 insertions(+)

diff --git a/compiler-rt/test/ubsan/TestCases/Integer/bit-int-pass.c b/compiler-rt/test/ubsan/TestCases/Integer/bit-int-pass.c
index a25428f0eb872f9..2255d2d08c76616 100644
--- a/compiler-rt/test/ubsan/TestCases/Integer/bit-int-pass.c
+++ b/compiler-rt/test/ubsan/TestCases/Integer/bit-int-pass.c
@@ -1,3 +1,4 @@
+// REQUIRES: x86_64-target-arch
 // RUN: %clang -Wno-constant-conversion -Wno-array-bounds -Wno-division-by-zero -Wno-shift-negative-value -Wno-shift-count-negative -Wno-int-to-pointer-cast -O0 -fsanitize=alignment,array-bounds,bool,float-cast-overflow,implicit-integer-sign-change,implicit-signed-integer-truncation,implicit-unsigned-integer-truncation,integer-divide-by-zero,nonnull-attribute,null,nullability-arg,nullability-assign,nullability-return,pointer-overflow,returns-nonnull-attribute,shift-base,shift-exponent,signed-integer-overflow,unreachable,unsigned-integer-overflow,unsigned-shift-base,vla-bound %s -o %t1 && %run %t1 2>&1 | FileCheck %s
 
 #include <stdint.h>
diff --git a/compiler-rt/test/ubsan/TestCases/Integer/bit-int.c b/compiler-rt/test/ubsan/TestCases/Integer/bit-int.c
index 18dc10bf3a38899..48f718285b06a1e 100644
--- a/compiler-rt/test/ubsan/TestCases/Integer/bit-int.c
+++ b/compiler-rt/test/ubsan/TestCases/Integer/bit-int.c
@@ -1,5 +1,9 @@
+// REQUIRES: x86_64-target-arch
 // RUN: %clang -Wno-constant-conversion -Wno-array-bounds -Wno-division-by-zero -Wno-shift-negative-value -Wno-shift-count-negative -Wno-int-to-pointer-cast -O0 -fsanitize=array-bounds,float-cast-overflow,implicit-integer-sign-change,implicit-signed-integer-truncation,implicit-unsigned-integer-truncation,integer-divide-by-zero,pointer-overflow,shift-base,shift-exponent,signed-integer-overflow,unsigned-integer-overflow,unsigned-shift-base,vla-bound %s -o %t1 && %run %t1 2>&1 | FileCheck %s
 
+// FIXME: make the test pass on windows.
+// XFAIL: target={{.*windows-msvc.*}}
+
 #include <stdint.h>
 #include <stdio.h>
 

From fcc318ff7960d7de8cbac56eb4f32b44b5261677 Mon Sep 17 00:00:00 2001
From: Mehdi Amini <joker.eph@gmail.com>
Date: Thu, 15 Aug 2024 22:48:43 +0200
Subject: [PATCH 082/441] Revert "[DXIL][Analysis] Implement enough of
 DXILResourceAnalysis for buffers" (#104504)

Reverts llvm/llvm-project#100699

This broke a few bots unfortunately.
---
 llvm/include/llvm/Analysis/DXILResource.h     |  13 +-
 llvm/include/llvm/IR/IntrinsicsDirectX.td     |  10 -
 llvm/lib/Analysis/DXILResource.cpp            | 380 +-----------------
 .../DXILResource/buffer-frombinding.ll        | 126 ------
 4 files changed, 8 insertions(+), 521 deletions(-)
 delete mode 100644 llvm/test/Analysis/DXILResource/buffer-frombinding.ll

diff --git a/llvm/include/llvm/Analysis/DXILResource.h b/llvm/include/llvm/Analysis/DXILResource.h
index 3ba0ae5de61d515..7db088df260f304 100644
--- a/llvm/include/llvm/Analysis/DXILResource.h
+++ b/llvm/include/llvm/Analysis/DXILResource.h
@@ -18,20 +18,19 @@
 namespace llvm {
 class CallInst;
 class MDTuple;
-class TargetExtType;
 
 namespace dxil {
 
 class ResourceInfo {
   struct ResourceBinding {
-    uint32_t RecordID;
+    uint32_t UniqueID;
     uint32_t Space;
     uint32_t LowerBound;
     uint32_t Size;
 
     bool operator==(const ResourceBinding &RHS) const {
-      return std::tie(RecordID, Space, LowerBound, Size) ==
-             std::tie(RHS.RecordID, RHS.Space, RHS.LowerBound, RHS.Size);
+      return std::tie(UniqueID, Space, LowerBound, Size) ==
+             std::tie(RHS.UniqueID, RHS.Space, RHS.LowerBound, RHS.Size);
     }
     bool operator!=(const ResourceBinding &RHS) const {
       return !(*this == RHS);
@@ -129,9 +128,9 @@ class ResourceInfo {
   bool isFeedback() const;
   bool isMultiSample() const;
 
-  void bind(uint32_t RecordID, uint32_t Space, uint32_t LowerBound,
+  void bind(uint32_t UniqueID, uint32_t Space, uint32_t LowerBound,
             uint32_t Size) {
-    Binding.RecordID = RecordID;
+    Binding.UniqueID = UniqueID;
     Binding.Space = Space;
     Binding.LowerBound = LowerBound;
     Binding.Size = Size;
@@ -216,8 +215,6 @@ class ResourceInfo {
 
   ResourceBinding getBinding() const { return Binding; }
   std::pair<uint32_t, uint32_t> getAnnotateProps() const;
-
-  void print(raw_ostream &OS) const;
 };
 
 } // namespace dxil
diff --git a/llvm/include/llvm/IR/IntrinsicsDirectX.td b/llvm/include/llvm/IR/IntrinsicsDirectX.td
index c9102aa3dd972bc..904801e6e9e95f0 100644
--- a/llvm/include/llvm/IR/IntrinsicsDirectX.td
+++ b/llvm/include/llvm/IR/IntrinsicsDirectX.td
@@ -20,16 +20,6 @@ def int_dx_flattened_thread_id_in_group : Intrinsic<[llvm_i32_ty], [], [IntrNoMe
 def int_dx_create_handle : ClangBuiltin<"__builtin_hlsl_create_handle">,
     Intrinsic<[ llvm_ptr_ty ], [llvm_i8_ty], [IntrWillReturn]>;
 
-// Create resource handle given binding information. Returns a `target("dx.")`
-// type appropriate for the kind of resource given a register space ID, lower
-// bound and range size of the binding, as well as an index and an indicator
-// whether that index may be non-uniform.
-def int_dx_handle_fromBinding
-    : DefaultAttrsIntrinsic<
-          [llvm_any_ty],
-          [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i1_ty],
-          [IntrNoMem]>;
-
 def int_dx_all : DefaultAttrsIntrinsic<[llvm_i1_ty], [llvm_any_ty]>;
 def int_dx_any : DefaultAttrsIntrinsic<[llvm_i1_ty], [llvm_any_ty]>;
 def int_dx_clamp : DefaultAttrsIntrinsic<[llvm_any_ty], [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>]>;
diff --git a/llvm/lib/Analysis/DXILResource.cpp b/llvm/lib/Analysis/DXILResource.cpp
index d7d10ec4d8d0d6c..9941a55d46d6459 100644
--- a/llvm/lib/Analysis/DXILResource.cpp
+++ b/llvm/lib/Analysis/DXILResource.cpp
@@ -8,14 +8,9 @@
 
 #include "llvm/Analysis/DXILResource.h"
 #include "llvm/ADT/APInt.h"
-#include "llvm/IR/Constants.h"
 #include "llvm/IR/DerivedTypes.h"
-#include "llvm/IR/DiagnosticInfo.h"
 #include "llvm/IR/Instructions.h"
-#include "llvm/IR/Intrinsics.h"
-#include "llvm/IR/IntrinsicsDirectX.h"
 #include "llvm/IR/Metadata.h"
-#include "llvm/IR/Module.h"
 #include "llvm/InitializePasses.h"
 
 #define DEBUG_TYPE "dxil-resource"
@@ -23,131 +18,6 @@
 using namespace llvm;
 using namespace dxil;
 
-static constexpr StringRef getResourceClassName(ResourceClass RC) {
-  switch (RC) {
-  case ResourceClass::SRV:
-    return "SRV";
-  case ResourceClass::UAV:
-    return "UAV";
-  case ResourceClass::CBuffer:
-    return "CBuffer";
-  case ResourceClass::Sampler:
-    return "Sampler";
-  }
-  llvm_unreachable("Unhandled ResourceClass");
-}
-
-static constexpr StringRef getResourceKindName(ResourceKind RK) {
-  switch (RK) {
-  case ResourceKind::Texture1D:
-    return "Texture1D";
-  case ResourceKind::Texture2D:
-    return "Texture2D";
-  case ResourceKind::Texture2DMS:
-    return "Texture2DMS";
-  case ResourceKind::Texture3D:
-    return "Texture3D";
-  case ResourceKind::TextureCube:
-    return "TextureCube";
-  case ResourceKind::Texture1DArray:
-    return "Texture1DArray";
-  case ResourceKind::Texture2DArray:
-    return "Texture2DArray";
-  case ResourceKind::Texture2DMSArray:
-    return "Texture2DMSArray";
-  case ResourceKind::TextureCubeArray:
-    return "TextureCubeArray";
-  case ResourceKind::TypedBuffer:
-    return "TypedBuffer";
-  case ResourceKind::RawBuffer:
-    return "RawBuffer";
-  case ResourceKind::StructuredBuffer:
-    return "StructuredBuffer";
-  case ResourceKind::CBuffer:
-    return "CBuffer";
-  case ResourceKind::Sampler:
-    return "Sampler";
-  case ResourceKind::TBuffer:
-    return "TBuffer";
-  case ResourceKind::RTAccelerationStructure:
-    return "RTAccelerationStructure";
-  case ResourceKind::FeedbackTexture2D:
-    return "FeedbackTexture2D";
-  case ResourceKind::FeedbackTexture2DArray:
-    return "FeedbackTexture2DArray";
-  case ResourceKind::NumEntries:
-  case ResourceKind::Invalid:
-    return "<invalid>";
-  }
-  llvm_unreachable("Unhandled ResourceKind");
-}
-
-static constexpr StringRef getElementTypeName(ElementType ET) {
-  switch (ET) {
-  case ElementType::I1:
-    return "i1";
-  case ElementType::I16:
-    return "i16";
-  case ElementType::U16:
-    return "u16";
-  case ElementType::I32:
-    return "i32";
-  case ElementType::U32:
-    return "u32";
-  case ElementType::I64:
-    return "i64";
-  case ElementType::U64:
-    return "u64";
-  case ElementType::F16:
-    return "f16";
-  case ElementType::F32:
-    return "f32";
-  case ElementType::F64:
-    return "f64";
-  case ElementType::SNormF16:
-    return "snorm_f16";
-  case ElementType::UNormF16:
-    return "unorm_f16";
-  case ElementType::SNormF32:
-    return "snorm_f32";
-  case ElementType::UNormF32:
-    return "unorm_f32";
-  case ElementType::SNormF64:
-    return "snorm_f64";
-  case ElementType::UNormF64:
-    return "unorm_f64";
-  case ElementType::PackedS8x32:
-    return "p32i8";
-  case ElementType::PackedU8x32:
-    return "p32u8";
-  case ElementType::Invalid:
-    return "<invalid>";
-  }
-  llvm_unreachable("Unhandled ElementType");
-}
-
-static constexpr StringRef getSamplerTypeName(SamplerType ST) {
-  switch (ST) {
-  case SamplerType::Default:
-    return "Default";
-  case SamplerType::Comparison:
-    return "Comparison";
-  case SamplerType::Mono:
-    return "Mono";
-  }
-  llvm_unreachable("Unhandled SamplerType");
-}
-
-static constexpr StringRef getSamplerFeedbackTypeName(SamplerFeedbackType SFT) {
-  switch (SFT) {
-  case SamplerFeedbackType::MinMip:
-    return "MinMip";
-  case SamplerFeedbackType::MipRegionUsed:
-    return "MipRegionUsed";
-  }
-  llvm_unreachable("Unhandled SamplerFeedbackType");
-}
-
 bool ResourceInfo::isUAV() const { return RC == ResourceClass::UAV; }
 
 bool ResourceInfo::isCBuffer() const { return RC == ResourceClass::CBuffer; }
@@ -370,7 +240,7 @@ MDTuple *ResourceInfo::getAsMetadata(LLVMContext &Ctx) const {
         Constant::getIntegerValue(I1Ty, APInt(1, V)));
   };
 
-  MDVals.push_back(getIntMD(Binding.RecordID));
+  MDVals.push_back(getIntMD(Binding.UniqueID));
   MDVals.push_back(ValueAsMetadata::get(Symbol));
   MDVals.push_back(MDString::get(Ctx, Name));
   MDVals.push_back(getIntMD(Binding.Space));
@@ -460,248 +330,6 @@ std::pair<uint32_t, uint32_t> ResourceInfo::getAnnotateProps() const {
   return {Word0, Word1};
 }
 
-void ResourceInfo::print(raw_ostream &OS) const {
-  OS << "  Symbol: ";
-  Symbol->printAsOperand(OS);
-  OS << "\n";
-
-  OS << "  Name: \"" << Name << "\"\n"
-     << "  Binding:\n"
-     << "    Record ID: " << Binding.RecordID << "\n"
-     << "    Space: " << Binding.Space << "\n"
-     << "    Lower Bound: " << Binding.LowerBound << "\n"
-     << "    Size: " << Binding.Size << "\n"
-     << "  Class: " << getResourceClassName(RC) << "\n"
-     << "  Kind: " << getResourceKindName(Kind) << "\n";
-
-  if (isCBuffer()) {
-    OS << "  CBuffer size: " << CBufferSize << "\n";
-  } else if (isSampler()) {
-    OS << "  Sampler Type: " << getSamplerTypeName(SamplerTy) << "\n";
-  } else {
-    if (isUAV()) {
-      OS << "  Globally Coherent: " << UAVFlags.GloballyCoherent << "\n"
-         << "  HasCounter: " << UAVFlags.HasCounter << "\n"
-         << "  IsROV: " << UAVFlags.IsROV << "\n";
-    }
-    if (isMultiSample())
-      OS << "  Sample Count: " << MultiSample.Count << "\n";
-
-    if (isStruct()) {
-      OS << "  Buffer Stride: " << Struct.Stride << "\n";
-      OS << "  Alignment: " << Struct.AlignLog2 << "\n";
-    } else if (isTyped()) {
-      OS << "  Element Type: " << getElementTypeName(Typed.ElementTy) << "\n"
-         << "  Element Count: " << Typed.ElementCount << "\n";
-    } else if (isFeedback())
-      OS << "  Feedback Type: " << getSamplerFeedbackTypeName(Feedback.Type)
-         << "\n";
-  }
-}
-
-//===----------------------------------------------------------------------===//
-// ResourceMapper
-
-static dxil::ElementType toDXILElementType(Type *Ty, bool IsSigned) {
-  // TODO: Handle unorm, snorm, and packed.
-  Ty = Ty->getScalarType();
-
-  if (Ty->isIntegerTy()) {
-    switch (Ty->getIntegerBitWidth()) {
-    case 16:
-      return IsSigned ? ElementType::I16 : ElementType::U16;
-    case 32:
-      return IsSigned ? ElementType::I32 : ElementType::U32;
-    case 64:
-      return IsSigned ? ElementType::I64 : ElementType::U64;
-    case 1:
-    default:
-      return ElementType::Invalid;
-    }
-  } else if (Ty->isFloatTy()) {
-    return ElementType::F32;
-  } else if (Ty->isDoubleTy()) {
-    return ElementType::F64;
-  } else if (Ty->isHalfTy()) {
-    return ElementType::F16;
-  }
-
-  return ElementType::Invalid;
-}
-
-namespace {
-
-class ResourceMapper {
-  Module &M;
-  LLVMContext &Context;
-  DXILResourceMap &Resources;
-
-  // In DXC, Record ID is unique per resource type. Match that.
-  uint32_t NextUAV = 0;
-  uint32_t NextSRV = 0;
-  uint32_t NextCBuf = 0;
-  uint32_t NextSmp = 0;
-
-public:
-  ResourceMapper(Module &M,
-                 MapVector<CallInst *, dxil::ResourceInfo> &Resources)
-      : M(M), Context(M.getContext()), Resources(Resources) {}
-
-  void diagnoseHandle(CallInst *CI, const Twine &Msg,
-                      DiagnosticSeverity Severity = DS_Error) {
-    std::string S;
-    raw_string_ostream SS(S);
-    CI->printAsOperand(SS);
-    DiagnosticInfoUnsupported Diag(*CI->getFunction(), Msg + ": " + SS.str(),
-                                   CI->getDebugLoc(), Severity);
-    Context.diagnose(Diag);
-  }
-
-  ResourceInfo *mapBufferType(CallInst *CI, TargetExtType *HandleTy,
-                              bool IsTyped) {
-    if (HandleTy->getNumTypeParameters() != 1 ||
-        HandleTy->getNumIntParameters() != (IsTyped ? 3 : 2)) {
-      diagnoseHandle(CI, Twine("Invalid buffer target type"));
-      return nullptr;
-    }
-
-    Type *ElTy = HandleTy->getTypeParameter(0);
-    unsigned IsWriteable = HandleTy->getIntParameter(0);
-    unsigned IsROV = HandleTy->getIntParameter(1);
-    bool IsSigned = IsTyped && HandleTy->getIntParameter(2);
-
-    ResourceClass RC = IsWriteable ? ResourceClass::UAV : ResourceClass::SRV;
-    ResourceKind Kind;
-    if (IsTyped)
-      Kind = ResourceKind::TypedBuffer;
-    else if (ElTy->isIntegerTy(8))
-      Kind = ResourceKind::RawBuffer;
-    else
-      Kind = ResourceKind::StructuredBuffer;
-
-    // TODO: We need to lower to a typed pointer, can we smuggle the type
-    // through?
-    Value *Symbol = UndefValue::get(PointerType::getUnqual(Context));
-    // TODO: We don't actually keep track of the name right now...
-    StringRef Name = "";
-
-    auto [It, Success] = Resources.try_emplace(CI, RC, Kind, Symbol, Name);
-    assert(Success && "Mapping the same CallInst again?");
-    (void)Success;
-    // We grab a pointer into the map's storage, which isn't generally safe.
-    // Since we're just using this to fill in the info the map won't mutate and
-    // the pointer stays valid for as long as we need it to.
-    ResourceInfo *RI = &(It->second);
-
-    if (RI->isUAV())
-      // TODO: We need analysis for GloballyCoherent and HasCounter
-      RI->setUAV(false, false, IsROV);
-
-    if (RI->isTyped()) {
-      dxil::ElementType ET = toDXILElementType(ElTy, IsSigned);
-      uint32_t Count = 1;
-      if (auto *VTy = dyn_cast<FixedVectorType>(ElTy))
-        Count = VTy->getNumElements();
-      RI->setTyped(ET, Count);
-    } else if (RI->isStruct()) {
-      const DataLayout &DL = M.getDataLayout();
-
-      // This mimics what DXC does. Notably, we only ever set the alignment if
-      // the type is actually a struct type.
-      uint32_t Stride = DL.getTypeAllocSize(ElTy);
-      MaybeAlign Alignment;
-      if (auto *STy = dyn_cast<StructType>(ElTy))
-        Alignment = DL.getStructLayout(STy)->getAlignment();
-      RI->setStruct(Stride, Alignment);
-    }
-
-    return RI;
-  }
-
-  ResourceInfo *mapHandleIntrin(CallInst *CI) {
-    FunctionType *FTy = CI->getFunctionType();
-    Type *RetTy = FTy->getReturnType();
-    auto *HandleTy = dyn_cast<TargetExtType>(RetTy);
-    if (!HandleTy) {
-      diagnoseHandle(CI, "dx.handle.fromBinding requires target type");
-      return nullptr;
-    }
-
-    StringRef TypeName = HandleTy->getName();
-    if (TypeName == "dx.TypedBuffer") {
-      return mapBufferType(CI, HandleTy, /*IsTyped=*/true);
-    } else if (TypeName == "dx.RawBuffer") {
-      return mapBufferType(CI, HandleTy, /*IsTyped=*/false);
-    } else if (TypeName == "dx.CBuffer") {
-      // TODO: implement
-      diagnoseHandle(CI, "dx.CBuffer handles are not implemented yet");
-      return nullptr;
-    } else if (TypeName == "dx.Sampler") {
-      // TODO: implement
-      diagnoseHandle(CI, "dx.Sampler handles are not implemented yet");
-      return nullptr;
-    } else if (TypeName == "dx.Texture") {
-      // TODO: implement
-      diagnoseHandle(CI, "dx.Texture handles are not implemented yet");
-      return nullptr;
-    }
-
-    diagnoseHandle(CI, "Invalid target(dx) type");
-    return nullptr;
-  }
-
-  ResourceInfo *mapHandleFromBinding(CallInst *CI) {
-    assert(CI->getIntrinsicID() == Intrinsic::dx_handle_fromBinding &&
-           "Must be dx.handle.fromBinding intrinsic");
-
-    ResourceInfo *RI = mapHandleIntrin(CI);
-    if (!RI)
-      return nullptr;
-
-    uint32_t NextID;
-    if (RI->isCBuffer())
-      NextID = NextCBuf++;
-    else if (RI->isSampler())
-      NextID = NextSmp++;
-    else if (RI->isUAV())
-      NextID = NextUAV++;
-    else
-      NextID = NextSRV++;
-
-    uint32_t Space = cast<ConstantInt>(CI->getArgOperand(0))->getZExtValue();
-    uint32_t LowerBound =
-        cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
-    uint32_t Size = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
-
-    RI->bind(NextID, Space, LowerBound, Size);
-
-    return RI;
-  }
-
-  void mapResources() {
-    for (Function &F : M.functions()) {
-      if (!F.isDeclaration())
-        continue;
-      LLVM_DEBUG(dbgs() << "Function: " << F.getName() << "\n");
-      Intrinsic::ID ID = F.getIntrinsicID();
-      switch (ID) {
-      default:
-        // TODO: handle `dx.op` functions.
-        continue;
-      case Intrinsic::dx_handle_fromBinding:
-        for (User *U : F.users()) {
-          LLVM_DEBUG(dbgs() << "  Visiting: " << *U << "\n");
-          if (CallInst *CI = dyn_cast<CallInst>(U))
-            mapHandleFromBinding(CI);
-        }
-        break;
-      }
-    }
-  }
-};
-
-} // namespace
-
 //===----------------------------------------------------------------------===//
 // DXILResourceAnalysis and DXILResourcePrinterPass
 
@@ -711,7 +339,6 @@ AnalysisKey DXILResourceAnalysis::Key;
 DXILResourceMap DXILResourceAnalysis::run(Module &M,
                                           ModuleAnalysisManager &AM) {
   DXILResourceMap Data;
-  ResourceMapper(M, Data).mapResources();
   return Data;
 }
 
@@ -724,7 +351,7 @@ PreservedAnalyses DXILResourcePrinterPass::run(Module &M,
     OS << "Binding for ";
     Handle->print(OS);
     OS << "\n";
-    Info.print(OS);
+    // TODO: Info.print(OS);
     OS << "\n";
   }
 
@@ -746,7 +373,6 @@ void DXILResourceWrapperPass::getAnalysisUsage(AnalysisUsage &AU) const {
 
 bool DXILResourceWrapperPass::runOnModule(Module &M) {
   ResourceMap.reset(new DXILResourceMap());
-  ResourceMapper(M, *ResourceMap).mapResources();
   return false;
 }
 
@@ -761,7 +387,7 @@ void DXILResourceWrapperPass::print(raw_ostream &OS, const Module *) const {
     OS << "Binding for ";
     Handle->print(OS);
     OS << "\n";
-    Info.print(OS);
+    // TODO: Info.print(OS);
     OS << "\n";
   }
 }
diff --git a/llvm/test/Analysis/DXILResource/buffer-frombinding.ll b/llvm/test/Analysis/DXILResource/buffer-frombinding.ll
deleted file mode 100644
index 4349adb8ef8ebb3..000000000000000
--- a/llvm/test/Analysis/DXILResource/buffer-frombinding.ll
+++ /dev/null
@@ -1,126 +0,0 @@
-; RUN: opt -S -disable-output -passes="print<dxil-resource>" < %s 2>&1 | FileCheck %s
-
-@G = external constant <4 x float>, align 4
-
-define void @test_typedbuffer() {
-  ; RWBuffer<float4> Buf : register(u5, space3)
-  %typed0 = call target("dx.TypedBuffer", <4 x float>, 1, 0, 0)
-              @llvm.dx.handle.fromBinding.tdx.TypedBuffer_f32_1_0(
-                  i32 3, i32 5, i32 1, i32 0, i1 false)
-  ; CHECK: Binding for %typed0
-  ; CHECK:   Symbol: ptr undef
-  ; CHECK:   Name: ""
-  ; CHECK:   Binding:
-  ; CHECK:     Record ID: 0
-  ; CHECK:     Space: 3
-  ; CHECK:     Lower Bound: 5
-  ; CHECK:     Size: 1
-  ; CHECK:   Class: UAV
-  ; CHECK:   Kind: TypedBuffer
-  ; CHECK:   Globally Coherent: 0
-  ; CHECK:   HasCounter: 0
-  ; CHECK:   IsROV: 0
-  ; CHECK:   Element Type: f32
-  ; CHECK:   Element Count: 4
-
-  ; RWBuffer<int> Buf : register(u7, space2)
-  %typed1 = call target("dx.TypedBuffer", i32, 1, 0, 1)
-      @llvm.dx.handle.fromBinding.tdx.TypedBuffer_i32_1_0t(
-          i32 2, i32 7, i32 1, i32 0, i1 false)
-  ; CHECK: Binding for %typed1
-  ; CHECK:   Symbol: ptr undef
-  ; CHECK:   Name: ""
-  ; CHECK:   Binding:
-  ; CHECK:     Record ID: 1
-  ; CHECK:     Space: 2
-  ; CHECK:     Lower Bound: 7
-  ; CHECK:     Size: 1
-  ; CHECK:   Class: UAV
-  ; CHECK:   Kind: TypedBuffer
-  ; CHECK:   Globally Coherent: 0
-  ; CHECK:   HasCounter: 0
-  ; CHECK:   IsROV: 0
-  ; CHECK:   Element Type: i32
-  ; CHECK:   Element Count: 1
-
-  ; Buffer<uint4> Buf[24] : register(t3, space5)
-  %typed2 = call target("dx.TypedBuffer", <4 x i32>, 0, 0, 0)
-      @llvm.dx.handle.fromBinding.tdx.TypedBuffer_i32_0_0t(
-          i32 2, i32 7, i32 24, i32 0, i1 false)
-  ; CHECK: Binding for %typed2
-  ; CHECK:   Symbol: ptr undef
-  ; CHECK:   Name: ""
-  ; CHECK:   Binding:
-  ; CHECK:     Record ID: 0
-  ; CHECK:     Space: 2
-  ; CHECK:     Lower Bound: 7
-  ; CHECK:     Size: 24
-  ; CHECK:   Class: SRV
-  ; CHECK:   Kind: TypedBuffer
-  ; CHECK:   Element Type: u32
-  ; CHECK:   Element Count: 4
-
-  ret void
-}
-
-define void @test_structbuffer() {
-  ; struct S { float4 a; uint4 b; };
-  ; StructuredBuffer<S> Buf : register(t2, space4)
-  %struct0 = call target("dx.RawBuffer", {<4 x float>, <4 x i32>}, 0, 0)
-      @llvm.dx.handle.fromBinding.tdx.RawBuffer_sl_v4f32v4i32s_0_0t(
-          i32 4, i32 2, i32 1, i32 0, i1 false)
-  ; CHECK: Binding for %struct0
-  ; CHECK:   Symbol: ptr undef
-  ; CHECK:   Name: ""
-  ; CHECK:   Binding:
-  ; CHECK:     Record ID: 1
-  ; CHECK:     Space: 4
-  ; CHECK:     Lower Bound: 2
-  ; CHECK:     Size: 1
-  ; CHECK:   Class: SRV
-  ; CHECK:   Kind: StructuredBuffer
-  ; CHECK:   Buffer Stride: 32
-  ; CHECK:   Alignment: 4
-
-  ret void
-}
-
-define void @test_bytebuffer() {
-  ; ByteAddressBuffer Buf : register(t8, space1)
-  %byteaddr0 = call target("dx.RawBuffer", i8, 0, 0)
-      @llvm.dx.handle.fromBinding.tdx.RawBuffer_i8_0_0t(
-          i32 1, i32 8, i32 1, i32 0, i1 false)
-  ; CHECK: Binding for %byteaddr0
-  ; CHECK:   Symbol: ptr undef
-  ; CHECK:   Name: ""
-  ; CHECK:   Binding:
-  ; CHECK:     Record ID: 2
-  ; CHECK:     Space: 1
-  ; CHECK:     Lower Bound: 8
-  ; CHECK:     Size: 1
-  ; CHECK:   Class: SRV
-  ; CHECK:   Kind: RawBuffer
-
-  ret void
-}
-
-; Note: We need declarations for each handle.fromBinding in the same
-; order as they appear in source to ensure that we can put our CHECK
-; lines along side the thing they're checking.
-declare target("dx.TypedBuffer", <4 x float>, 1, 0, 0)
-        @llvm.dx.handle.fromBinding.tdx.TypedBuffer_v4f32_1_0_0t(
-        i32, i32, i32, i32, i1) #0
-declare target("dx.TypedBuffer", i32, 1, 0, 1)
-        @llvm.dx.handle.fromBinding.tdx.TypedBuffer_i32_1_0_1t(
-            i32, i32, i32, i32, i1) #0
-declare target("dx.TypedBuffer", <4 x i32>, 0, 0, 0)
-        @llvm.dx.handle.fromBinding.tdx.TypedBuffer_v4i32_0_0_0t(
-            i32, i32, i32, i32, i1) #0
-declare target("dx.RawBuffer", { <4 x float>, <4 x i32> }, 0, 0)
-        @llvm.dx.handle.fromBinding.tdx.RawBuffer_sl_v4f32v4i32s_0_0t(
-            i32, i32, i32, i32, i1) #0
-declare target("dx.RawBuffer", i8, 0, 0)
-        @llvm.dx.handle.fromBinding.tdx.RawBuffer_i8_0_0t(
-            i32, i32, i32, i32, i1) #0
-
-attributes #0 = { nocallback nofree nosync nounwind willreturn memory(none) }

From 89d0a5c9e8852444399645d474543730ca544ede Mon Sep 17 00:00:00 2001
From: Vitaly Buka <vitalybuka@google.com>
Date: Thu, 15 Aug 2024 13:50:02 -0700
Subject: [PATCH 083/441] [asan] De-prioritize VReport `DTLS_Find` (#104401)

---
 compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.cpp     | 2 +-
 .../sanitizer_common/TestCases/Linux/resize_tls_dynamic.cpp     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.cpp
index ee293bbd68875c0..e2a66142e584e63 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.cpp
@@ -66,7 +66,7 @@ static DTLS::DTVBlock *DTLS_NextBlock(atomic_uintptr_t *cur) {
 }
 
 static DTLS::DTV *DTLS_Find(uptr id) {
-  VReport(2, "__tls_get_addr: DTLS_Find %p %zd\n", (void *)&dtls, id);
+  VReport(3, "__tls_get_addr: DTLS_Find %p %zd\n", (void *)&dtls, id);
   static constexpr uptr kPerBlock = ARRAY_SIZE(DTLS::DTVBlock::dtvs);
   DTLS::DTVBlock *cur = DTLS_NextBlock(&dtls.dtv_block);
   if (!cur)
diff --git a/compiler-rt/test/sanitizer_common/TestCases/Linux/resize_tls_dynamic.cpp b/compiler-rt/test/sanitizer_common/TestCases/Linux/resize_tls_dynamic.cpp
index 2d6e6c60f2437e5..c288e1d69baf92f 100644
--- a/compiler-rt/test/sanitizer_common/TestCases/Linux/resize_tls_dynamic.cpp
+++ b/compiler-rt/test/sanitizer_common/TestCases/Linux/resize_tls_dynamic.cpp
@@ -1,6 +1,6 @@
 // RUN: %clangxx %s -DBUILD_DSO -fPIC -shared -o %t.so
 // RUN: %clangxx --std=c++11 %s -o %t
-// RUN: %env_tool_opts=verbosity=2 %run %t 2>&1 | FileCheck %s
+// RUN: %env_tool_opts=verbosity=3 %run %t 2>&1 | FileCheck %s
 
 // Does not call __tls_get_addr
 // UNSUPPORTED: i386-linux

From 274370d60332e76c374fd7bf87ade16319c2782c Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Thu, 15 Aug 2024 13:51:13 -0700
Subject: [PATCH 084/441] MCAsmInfo: Replace some Mach-O specific check with
 isMachO(). NFC

`HasMachO*` might lure contributors to add other object file format
propery when it is not really necessary.
---
 llvm/include/llvm/MC/MCAsmInfo.h           | 16 +---------------
 llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp |  7 +++----
 llvm/lib/MC/MCAsmInfoDarwin.cpp            |  3 ---
 3 files changed, 4 insertions(+), 22 deletions(-)

diff --git a/llvm/include/llvm/MC/MCAsmInfo.h b/llvm/include/llvm/MC/MCAsmInfo.h
index 7c4f3b63756eaea..dba5d97465af28a 100644
--- a/llvm/include/llvm/MC/MCAsmInfo.h
+++ b/llvm/include/llvm/MC/MCAsmInfo.h
@@ -85,14 +85,6 @@ class MCAsmInfo {
   /// Default is false.
   bool HasSubsectionsViaSymbols = false;
 
-  /// True if this is a MachO target that supports the macho-specific .zerofill
-  /// directive for emitting BSS Symbols.  Default is false.
-  bool HasMachoZeroFillDirective = false;
-
-  /// True if this is a MachO target that supports the macho-specific .tbss
-  /// directive for emitting thread local BSS Symbols.  Default is false.
-  bool HasMachoTBSSDirective = false;
-
   /// True if this is a non-GNU COFF target. The COFF port of the GNU linker
   /// doesn't handle associative comdats in the way that we would like to use
   /// them.
@@ -401,10 +393,6 @@ class MCAsmInfo {
   /// undefined symbol.  Defaults to nullptr.
   const char *WeakRefDirective = nullptr;
 
-  /// True if we have a directive to declare a global as being a weak defined
-  /// symbol.  Defaults to false.
-  bool HasWeakDefDirective = false;
-
   /// True if we have a directive to declare a global as being a weak defined
   /// symbol that can be hidden (unexported).  Defaults to false.
   bool HasWeakDefCanBeHiddenDirective = false;
@@ -603,8 +591,7 @@ class MCAsmInfo {
 
   // Accessors.
 
-  bool hasMachoZeroFillDirective() const { return HasMachoZeroFillDirective; }
-  bool hasMachoTBSSDirective() const { return HasMachoTBSSDirective; }
+  bool isMachO() const { return HasSubsectionsViaSymbols; }
   bool hasCOFFAssociativeComdats() const { return HasCOFFAssociativeComdats; }
   bool hasCOFFComdatConstants() const { return HasCOFFComdatConstants; }
   bool hasVisibilityOnlyWithLinkage() const {
@@ -730,7 +717,6 @@ class MCAsmInfo {
   bool hasAltEntry() const { return HasAltEntry; }
   const char *getWeakDirective() const { return WeakDirective; }
   const char *getWeakRefDirective() const { return WeakRefDirective; }
-  bool hasWeakDefDirective() const { return HasWeakDefDirective; }
 
   bool hasWeakDefCanBeHiddenDirective() const {
     return HasWeakDefCanBeHiddenDirective;
diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
index 7ab2f2c13def879..3c792fc89c9b6ab 100644
--- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
@@ -636,7 +636,7 @@ void AsmPrinter::emitLinkage(const GlobalValue *GV, MCSymbol *GVSym) const {
   case GlobalValue::LinkOnceODRLinkage:
   case GlobalValue::WeakAnyLinkage:
   case GlobalValue::WeakODRLinkage:
-    if (MAI->hasWeakDefDirective()) {
+    if (MAI->isMachO()) {
       // .globl _foo
       OutStreamer->emitSymbolAttribute(GVSym, MCSA_Global);
 
@@ -779,8 +779,7 @@ void AsmPrinter::emitGlobalVariable(const GlobalVariable *GV) {
 
   // If we have a bss global going to a section that supports the
   // zerofill directive, do so here.
-  if (GVKind.isBSS() && MAI->hasMachoZeroFillDirective() &&
-      TheSection->isVirtualSection()) {
+  if (GVKind.isBSS() && MAI->isMachO() && TheSection->isVirtualSection()) {
     if (Size == 0)
       Size = 1; // zerofill of 0 bytes is undefined.
     emitLinkage(GV, GVSym);
@@ -825,7 +824,7 @@ void AsmPrinter::emitGlobalVariable(const GlobalVariable *GV) {
   // TLOF class.  This will also make it more obvious that stuff like
   // MCStreamer::EmitTBSSSymbol is macho specific and only called from macho
   // specific code.
-  if (GVKind.isThreadLocal() && MAI->hasMachoTBSSDirective()) {
+  if (GVKind.isThreadLocal() && MAI->isMachO()) {
     // Emit the .tbss symbol
     MCSymbol *MangSym =
         OutContext.getOrCreateSymbol(GVSym->getName() + Twine("$tlv$init"));
diff --git a/llvm/lib/MC/MCAsmInfoDarwin.cpp b/llvm/lib/MC/MCAsmInfoDarwin.cpp
index e218a42889de909..8d883e45a91cc60 100644
--- a/llvm/lib/MC/MCAsmInfoDarwin.cpp
+++ b/llvm/lib/MC/MCAsmInfoDarwin.cpp
@@ -69,12 +69,9 @@ MCAsmInfoDarwin::MCAsmInfoDarwin() {
   InlineAsmEnd = " InlineAsm End";
 
   // Directives:
-  HasWeakDefDirective = true;
   HasWeakDefCanBeHiddenDirective = true;
   WeakRefDirective = "\t.weak_reference ";
   ZeroDirective = "\t.space\t";  // ".space N" emits N zeros.
-  HasMachoZeroFillDirective = true;  // Uses .zerofill
-  HasMachoTBSSDirective = true; // Uses .tbss
 
   HiddenVisibilityAttr = MCSA_PrivateExtern;
   HiddenDeclarationVisibilityAttr = MCSA_Invalid;

From 4e078e3797098daa40d254447c499bcf61415308 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Thu, 15 Aug 2024 14:01:09 -0700
Subject: [PATCH 085/441] [MC] Replace hasAltEntry() with isMachO()

All Mach-O targets have this property, so just remove this variable,
which could lure contributors to add unneeded object file format
specific properties.
---
 llvm/include/llvm/MC/MCAsmInfo.h           | 5 -----
 llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp | 2 +-
 llvm/lib/MC/MCAsmInfoDarwin.cpp            | 1 -
 3 files changed, 1 insertion(+), 7 deletions(-)

diff --git a/llvm/include/llvm/MC/MCAsmInfo.h b/llvm/include/llvm/MC/MCAsmInfo.h
index dba5d97465af28a..278f16d685eed99 100644
--- a/llvm/include/llvm/MC/MCAsmInfo.h
+++ b/llvm/include/llvm/MC/MCAsmInfo.h
@@ -382,10 +382,6 @@ class MCAsmInfo {
   /// to false.
   bool HasNoDeadStrip = false;
 
-  /// True if this target supports the MachO .alt_entry directive.  Defaults to
-  /// false.
-  bool HasAltEntry = false;
-
   /// Used to declare a global as being a weak symbol. Defaults to ".weak".
   const char *WeakDirective;
 
@@ -714,7 +710,6 @@ class MCAsmInfo {
   bool hasFourStringsDotFile() const { return HasFourStringsDotFile; }
   bool hasIdentDirective() const { return HasIdentDirective; }
   bool hasNoDeadStrip() const { return HasNoDeadStrip; }
-  bool hasAltEntry() const { return HasAltEntry; }
   const char *getWeakDirective() const { return WeakDirective; }
   const char *getWeakRefDirective() const { return WeakRefDirective; }
 
diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
index 3c792fc89c9b6ab..60cb26973ead414 100644
--- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
@@ -2147,7 +2147,7 @@ void AsmPrinter::emitGlobalAlias(const Module &M, const GlobalAlias &GA) {
 
   const MCExpr *Expr = lowerConstant(GA.getAliasee());
 
-  if (MAI->hasAltEntry() && isa<MCBinaryExpr>(Expr))
+  if (MAI->isMachO() && isa<MCBinaryExpr>(Expr))
     OutStreamer->emitSymbolAttribute(Name, MCSA_AltEntry);
 
   // Emit the directives as assignments aka .set:
diff --git a/llvm/lib/MC/MCAsmInfoDarwin.cpp b/llvm/lib/MC/MCAsmInfoDarwin.cpp
index 8d883e45a91cc60..9cba775e9168b6c 100644
--- a/llvm/lib/MC/MCAsmInfoDarwin.cpp
+++ b/llvm/lib/MC/MCAsmInfoDarwin.cpp
@@ -81,7 +81,6 @@ MCAsmInfoDarwin::MCAsmInfoDarwin() {
 
   HasDotTypeDotSizeDirective = false;
   HasNoDeadStrip = true;
-  HasAltEntry = true;
 
   DwarfUsesRelocationsAcrossSections = false;
   SetDirectiveSuppressesReloc = true;

From c1a750b8bf7c7bfcd4e72537fcf76e5f23d0f06a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Valentin=20Clement=20=28=E3=83=90=E3=83=AC=E3=83=B3?=
 =?UTF-8?q?=E3=82=BF=E3=82=A4=E3=83=B3=20=E3=82=AF=E3=83=AC=E3=83=A1?=
 =?UTF-8?q?=E3=83=B3=29?= <clementval@gmail.com>
Date: Thu, 15 Aug 2024 14:11:03 -0700
Subject: [PATCH 086/441] [flang] Make sure range is valid (#104281)

Attempt to solve #102495
---
 flang/lib/Parser/provenance.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flang/lib/Parser/provenance.cpp b/flang/lib/Parser/provenance.cpp
index e31038b09e40726..fe92aa7f64fb1db 100644
--- a/flang/lib/Parser/provenance.cpp
+++ b/flang/lib/Parser/provenance.cpp
@@ -248,7 +248,7 @@ static void EmitPrefix(llvm::raw_ostream &o, llvm::raw_ostream::Colors color,
 
 std::optional<ProvenanceRange> AllSources::GetInclusionInfo(
     const std::optional<ProvenanceRange> &range) const {
-  if (!range)
+  if (!range || !IsValid(range->start()))
     return std::nullopt;
   const Origin &origin{MapToOrigin(range->start())};
 

From 99313575b6bb7d5acbe437c2022b6e5dc7f297f1 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Thu, 15 Aug 2024 14:17:26 -0700
Subject: [PATCH 087/441] [-Wunsafe-buffer-usage] Fix warning after #102953

---
 clang/lib/Analysis/UnsafeBufferUsage.cpp | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/clang/lib/Analysis/UnsafeBufferUsage.cpp b/clang/lib/Analysis/UnsafeBufferUsage.cpp
index 95c288bfe3fd2f7..da7446913f7c87e 100644
--- a/clang/lib/Analysis/UnsafeBufferUsage.cpp
+++ b/clang/lib/Analysis/UnsafeBufferUsage.cpp
@@ -2797,9 +2797,8 @@ fixVariable(const VarDecl *VD, FixitStrategy::Kind K,
     return {};
   }
   case FixitStrategy::Kind::Array: {
-    if (VD->isLocalVarDecl())
-      if (auto CAT = Ctx.getAsConstantArrayType(VD->getType()))
-        return fixVariableWithArray(VD, Tracker, Ctx, Handler);
+    if (VD->isLocalVarDecl() && Ctx.getAsConstantArrayType(VD->getType()))
+      return fixVariableWithArray(VD, Tracker, Ctx, Handler);
 
     DEBUG_NOTE_DECL_FAIL(VD, " : not a local const-size array");
     return {};

From 9f1dc01e0ab4f91c4052a712ce590d2e86b81dc3 Mon Sep 17 00:00:00 2001
From: Vlad Serebrennikov <serebrennikov.vladislav@gmail.com>
Date: Fri, 16 Aug 2024 00:24:07 +0300
Subject: [PATCH 088/441] [clang][NFC] Fix table of contents in `Sema.h`

Add APINotes and bounds safety to the table of contents, and move their declarations appropriately.
---
 clang/include/clang/Sema/Sema.h | 168 ++++++++++++++++----------------
 1 file changed, 85 insertions(+), 83 deletions(-)

diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 88e82dca007b8bd..ea2977838466410 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -536,38 +536,40 @@ class Sema final : public SemaBase {
   // Table of Contents
   // -----------------
   // 1. Semantic Analysis (Sema.cpp)
-  // 2. C++ Access Control (SemaAccess.cpp)
-  // 3. Attributes (SemaAttr.cpp)
-  // 4. Availability Attribute Handling (SemaAvailability.cpp)
-  // 5. Casts (SemaCast.cpp)
-  // 6. Extra Semantic Checking (SemaChecking.cpp)
-  // 7. C++ Coroutines (SemaCoroutine.cpp)
-  // 8. C++ Scope Specifiers (SemaCXXScopeSpec.cpp)
-  // 9. Declarations (SemaDecl.cpp)
-  // 10. Declaration Attribute Handling (SemaDeclAttr.cpp)
-  // 11. C++ Declarations (SemaDeclCXX.cpp)
-  // 12. C++ Exception Specifications (SemaExceptionSpec.cpp)
-  // 13. Expressions (SemaExpr.cpp)
-  // 14. C++ Expressions (SemaExprCXX.cpp)
-  // 15. Member Access Expressions (SemaExprMember.cpp)
-  // 16. Initializers (SemaInit.cpp)
-  // 17. C++ Lambda Expressions (SemaLambda.cpp)
-  // 18. Name Lookup (SemaLookup.cpp)
-  // 19. Modules (SemaModule.cpp)
-  // 20. C++ Overloading (SemaOverload.cpp)
-  // 21. Statements (SemaStmt.cpp)
-  // 22. `inline asm` Statement (SemaStmtAsm.cpp)
-  // 23. Statement Attribute Handling (SemaStmtAttr.cpp)
-  // 24. C++ Templates (SemaTemplate.cpp)
-  // 25. C++ Template Argument Deduction (SemaTemplateDeduction.cpp)
-  // 26. C++ Template Deduction Guide (SemaTemplateDeductionGuide.cpp)
-  // 27. C++ Template Instantiation (SemaTemplateInstantiate.cpp)
-  // 28. C++ Template Declaration Instantiation
+  // 2. API Notes (SemaAPINotes.cpp)
+  // 3. C++ Access Control (SemaAccess.cpp)
+  // 4. Attributes (SemaAttr.cpp)
+  // 5. Availability Attribute Handling (SemaAvailability.cpp)
+  // 6. Bounds Safety (SemaBoundsSafety.cpp)
+  // 7. Casts (SemaCast.cpp)
+  // 8. Extra Semantic Checking (SemaChecking.cpp)
+  // 9. C++ Coroutines (SemaCoroutine.cpp)
+  // 10. C++ Scope Specifiers (SemaCXXScopeSpec.cpp)
+  // 11. Declarations (SemaDecl.cpp)
+  // 12. Declaration Attribute Handling (SemaDeclAttr.cpp)
+  // 13. C++ Declarations (SemaDeclCXX.cpp)
+  // 14. C++ Exception Specifications (SemaExceptionSpec.cpp)
+  // 15. Expressions (SemaExpr.cpp)
+  // 16. C++ Expressions (SemaExprCXX.cpp)
+  // 17. Member Access Expressions (SemaExprMember.cpp)
+  // 18. Initializers (SemaInit.cpp)
+  // 19. C++ Lambda Expressions (SemaLambda.cpp)
+  // 20. Name Lookup (SemaLookup.cpp)
+  // 21. Modules (SemaModule.cpp)
+  // 22. C++ Overloading (SemaOverload.cpp)
+  // 23. Statements (SemaStmt.cpp)
+  // 24. `inline asm` Statement (SemaStmtAsm.cpp)
+  // 25. Statement Attribute Handling (SemaStmtAttr.cpp)
+  // 26. C++ Templates (SemaTemplate.cpp)
+  // 27. C++ Template Argument Deduction (SemaTemplateDeduction.cpp)
+  // 28. C++ Template Deduction Guide (SemaTemplateDeductionGuide.cpp)
+  // 29. C++ Template Instantiation (SemaTemplateInstantiate.cpp)
+  // 30. C++ Template Declaration Instantiation
   //     (SemaTemplateInstantiateDecl.cpp)
-  // 29. C++ Variadic Templates (SemaTemplateVariadic.cpp)
-  // 30. Constraints and Concepts (SemaConcept.cpp)
-  // 31. Types (SemaType.cpp)
-  // 32. FixIt Helpers (SemaFixItUtils.cpp)
+  // 31. C++ Variadic Templates (SemaTemplateVariadic.cpp)
+  // 32. Constraints and Concepts (SemaConcept.cpp)
+  // 33. Types (SemaType.cpp)
+  // 34. FixIt Helpers (SemaFixItUtils.cpp)
 
   /// \name Semantic Analysis
   /// Implementations are in Sema.cpp
@@ -1325,6 +1327,25 @@ class Sema final : public SemaBase {
   //
   //
 
+  /// \name API Notes
+  /// Implementations are in SemaAPINotes.cpp
+  ///@{
+
+public:
+  /// Map any API notes provided for this declaration to attributes on the
+  /// declaration.
+  ///
+  /// Triggered by declaration-attribute processing.
+  void ProcessAPINotes(Decl *D);
+
+  ///@}
+
+  //
+  //
+  // -------------------------------------------------------------------------
+  //
+  //
+
   /// \name C++ Access Control
   /// Implementations are in SemaAccess.cpp
   ///@{
@@ -2095,6 +2116,39 @@ class Sema final : public SemaBase {
   //
   //
 
+  /// \name Bounds Safety
+  /// Implementations are in SemaBoundsSafety.cpp
+  ///@{
+public:
+  /// Check if applying the specified attribute variant from the "counted by"
+  /// family of attributes to FieldDecl \p FD is semantically valid. If
+  /// semantically invalid diagnostics will be emitted explaining the problems.
+  ///
+  /// \param FD The FieldDecl to apply the attribute to
+  /// \param E The count expression on the attribute
+  /// \param CountInBytes If true the attribute is from the "sized_by" family of
+  ///                     attributes. If the false the attribute is from
+  ///                     "counted_by" family of attributes.
+  /// \param OrNull If true the attribute is from the "_or_null" suffixed family
+  ///               of attributes. If false the attribute does not have the
+  ///               suffix.
+  ///
+  /// Together \p CountInBytes and \p OrNull decide the attribute variant. E.g.
+  /// \p CountInBytes and \p OrNull both being true indicates the
+  /// `counted_by_or_null` attribute.
+  ///
+  /// \returns false iff semantically valid.
+  bool CheckCountedByAttrOnField(FieldDecl *FD, Expr *E, bool CountInBytes,
+                                 bool OrNull);
+
+  ///@}
+
+  //
+  //
+  // -------------------------------------------------------------------------
+  //
+  //
+
   /// \name Casts
   /// Implementations are in SemaCast.cpp
   ///@{
@@ -15045,58 +15099,6 @@ class Sema final : public SemaBase {
   std::string getFixItZeroLiteralForType(QualType T, SourceLocation Loc) const;
 
   ///@}
-
-  //
-  //
-  // -------------------------------------------------------------------------
-  //
-  //
-
-  /// \name API Notes
-  /// Implementations are in SemaAPINotes.cpp
-  ///@{
-
-public:
-  /// Map any API notes provided for this declaration to attributes on the
-  /// declaration.
-  ///
-  /// Triggered by declaration-attribute processing.
-  void ProcessAPINotes(Decl *D);
-
-  ///@}
-
-  //
-  //
-  // -------------------------------------------------------------------------
-  //
-  //
-
-  /// \name Bounds Safety
-  /// Implementations are in SemaBoundsSafety.cpp
-  ///@{
-public:
-  /// Check if applying the specified attribute variant from the "counted by"
-  /// family of attributes to FieldDecl \p FD is semantically valid. If
-  /// semantically invalid diagnostics will be emitted explaining the problems.
-  ///
-  /// \param FD The FieldDecl to apply the attribute to
-  /// \param E The count expression on the attribute
-  /// \param CountInBytes If true the attribute is from the "sized_by" family of
-  ///                     attributes. If the false the attribute is from
-  ///                     "counted_by" family of attributes.
-  /// \param OrNull If true the attribute is from the "_or_null" suffixed family
-  ///               of attributes. If false the attribute does not have the
-  ///               suffix.
-  ///
-  /// Together \p CountInBytes and \p OrNull decide the attribute variant. E.g.
-  /// \p CountInBytes and \p OrNull both being true indicates the
-  /// `counted_by_or_null` attribute.
-  ///
-  /// \returns false iff semantically valid.
-  bool CheckCountedByAttrOnField(FieldDecl *FD, Expr *E, bool CountInBytes,
-                                 bool OrNull);
-
-  ///@}
 };
 
 DeductionFailureInfo

From c458e9e034b2c49a4b35eb732f08a30b338a9ae2 Mon Sep 17 00:00:00 2001
From: Sirraide <aeternalmail@gmail.com>
Date: Thu, 15 Aug 2024 23:42:23 +0200
Subject: [PATCH 089/441] [Clang] Add target triple to fix failing test
 (#104513)

#102757 erroneously removed the target triple required for the test to
pass.
---
 clang/test/SemaCXX/constexpr-vectors.cpp | 42 ++++++++++++------------
 1 file changed, 21 insertions(+), 21 deletions(-)

diff --git a/clang/test/SemaCXX/constexpr-vectors.cpp b/clang/test/SemaCXX/constexpr-vectors.cpp
index 6cd305e57325773..7492116f44d70f3 100644
--- a/clang/test/SemaCXX/constexpr-vectors.cpp
+++ b/clang/test/SemaCXX/constexpr-vectors.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 %s -Wno-uninitialized -std=c++14 -fsyntax-only -verify
+// RUN: %clang_cc1 %s -triple x86_64-linux-gnu -Wno-uninitialized -std=c++14 -fsyntax-only -verify
 
 // expected-no-diagnostics
 
@@ -156,11 +156,11 @@ void CharUsage() {
   constexpr auto a = FourCharsVecSize{6, 3, 2, 1} +
             FourCharsVecSize{12, 15, 5, 7};
   static_assert(a[0] == 18 && a[1] == 18 && a[2] == 7 && a[3] == 8, "");
-  
+
   constexpr auto b = FourCharsVecSize{19, 15, 13, 12} -
                      FourCharsVecSize{13, 14, 5, 3};
   static_assert(b[0] == 6 && b[1] == 1 && b[2] == 8 && b[3] == 9, "");
-  
+
   constexpr auto c = FourCharsVecSize{8, 4, 2, 1} *
                      FourCharsVecSize{3, 4, 5, 6};
   static_assert(c[0] == 24 && c[1] == 16 && c[2] == 10 && c[3] == 6, "");
@@ -175,13 +175,13 @@ void CharUsage() {
 
   constexpr auto f = FourCharsVecSize{6, 3, 2, 1} + 3;
   static_assert(f[0] == 9 && f[1] == 6 && f[2] == 5 && f[3] == 4, "");
-  
+
   constexpr auto g = FourCharsVecSize{19, 15, 12, 10} - 3;
-  static_assert(g[0] == 16 && g[1] == 12 && g[2] == 9 && g[3] == 7, "");  
+  static_assert(g[0] == 16 && g[1] == 12 && g[2] == 9 && g[3] == 7, "");
 
   constexpr auto h = FourCharsVecSize{8, 4, 2, 1} * 3;
   static_assert(h[0] == 24 && h[1] == 12 && h[2] == 6 && h[3] == 3, "");
- 
+
   constexpr auto j = FourCharsVecSize{12, 15, 18, 21} / 3;
   static_assert(j[0] == 4 && j[1] == 5 && j[2] == 6 && j[3] == 7, "");
 
@@ -193,7 +193,7 @@ void CharUsage() {
 
   constexpr auto m = 20 - FourCharsVecSize{19, 15, 12, 10};
   static_assert(m[0] == 1 && m[1] == 5 && m[2] == 8 && m[3] == 10, "");
- 
+
   constexpr auto n = 3 * FourCharsVecSize{8, 4, 2, 1};
   static_assert(n[0] == 24 && n[1] == 12 && n[2] == 6 && n[3] == 3, "");
 
@@ -254,7 +254,7 @@ void CharUsage() {
 
   constexpr auto E = FourCharsVecSize{1, 2, 3, 4} <= 3;
   static_assert(E[0] == -1 && E[1] == -1 && E[2] == -1 && E[3] == 0, "");
-    
+
   constexpr auto F = FourCharsVecSize{1, 2, 3, 4} >= 3;
   static_assert(F[0] == 0 && F[1] == 0 && F[2] == -1 && F[3] == -1, "");
 
@@ -267,7 +267,7 @@ void CharUsage() {
   constexpr auto I = FourCharsVecSize{1, 2, 3, 4} &
                      FourCharsVecSize{4, 3, 2, 1};
   static_assert(I[0] == 0 && I[1] == 2 && I[2] == 2 && I[3] == 0, "");
- 
+
   constexpr auto J = FourCharsVecSize{1, 2, 3, 4} ^
                      FourCharsVecSize { 4, 3, 2, 1 };
   static_assert(J[0] == 5 && J[1] == 1 && J[2] == 1 && J[3] == 5, "");
@@ -288,7 +288,7 @@ void CharUsage() {
   constexpr auto O = FourCharsVecSize{5, 0, 6, 0} &&
                      FourCharsVecSize{5, 5, 0, 0};
   static_assert(O[0] == 1 && O[1] == 0 && O[2] == 0 && O[3] == 0, "");
-  
+
   constexpr auto P = FourCharsVecSize{5, 0, 6, 0} ||
                      FourCharsVecSize{5, 5, 0, 0};
   static_assert(P[0] == 1 && P[1] == 1 && P[2] == 1 && P[3] == 0, "");
@@ -330,7 +330,7 @@ void CharUsage() {
   static_assert(ac[0] == 20 && ac[1] == 19 && ac[2] == 15 && ac[3] == 1, "");
 
   constexpr auto ad = CmpBinOr(a, b);
-  static_assert(ad[0] == 22 && ad[1] == 19 && ad[2] == 15 && ad[3] == 9, ""); 
+  static_assert(ad[0] == 22 && ad[1] == 19 && ad[2] == 15 && ad[3] == 9, "");
 
   constexpr auto ae = ~FourCharsVecSize{1, 2, 10, 20};
   static_assert(ae[0] == -2 && ae[1] == -3 && ae[2] == -11 && ae[3] == -21, "");
@@ -395,7 +395,7 @@ void CharExtVecUsage() {
 
   constexpr auto r = FourCharsExtVec{19, 15, 12, 10} >>
                      FourCharsExtVec{1, 1, 2, 2};
-  static_assert(r[0] == 9 && r[1] == 7 && r[2] == 3 && r[3] == 2, ""); 
+  static_assert(r[0] == 9 && r[1] == 7 && r[2] == 3 && r[3] == 2, "");
 
   constexpr auto s = FourCharsExtVec{6, 3, 5, 10} << 1;
   static_assert(s[0] == 12 && s[1] == 6 && s[2] == 10 && s[3] == 20, "");
@@ -441,7 +441,7 @@ void CharExtVecUsage() {
 
   constexpr auto E = FourCharsExtVec{1, 2, 3, 4} <= 3;
   static_assert(E[0] == -1 && E[1] == -1 && E[2] == -1 && E[3] == 0, "");
-  
+
   constexpr auto F = FourCharsExtVec{1, 2, 3, 4} >= 3;
   static_assert(F[0] == 0 && F[1] == 0 && F[2] == -1 && F[3] == -1, "");
 
@@ -462,7 +462,7 @@ void CharExtVecUsage() {
   constexpr auto K = FourCharsExtVec{1, 2, 3, 4} |
                      FourCharsExtVec{4, 3, 2, 1};
   static_assert(K[0] == 5 && K[1] == 3 && K[2] == 3 && K[3] == 5, "");
-  
+
   constexpr auto L = FourCharsExtVec{1, 2, 3, 4} & 3;
   static_assert(L[0] == 1 && L[1] == 2 && L[2] == 3 && L[3] == 0, "");
 
@@ -549,7 +549,7 @@ void FloatUsage() {
   constexpr auto g = FourFloatsVecSize{19, 15, 12, 10} - 3;
   static_assert(g[0] == 1.600000e+01 && g[1] == 1.200000e+01 && g[2] == 9.000000e+00 && g[3] == 7.000000e+00, "");
 
-  constexpr auto h = FourFloatsVecSize{8, 4, 2, 1} * 3;  
+  constexpr auto h = FourFloatsVecSize{8, 4, 2, 1} * 3;
   static_assert(h[0] == 2.400000e+01 && h[1] == 1.200000e+01 && h[2] == 6.000000e+00 && h[3] == 3.000000e+00, "");
 
   constexpr auto j = FourFloatsVecSize{12, 15, 18, 21} / 3;
@@ -593,7 +593,7 @@ void FloatUsage() {
 
   constexpr auto C = FourFloatsVecSize{1, 2, 3, 4} < 3;
   static_assert(C[0] == -1 && C[1] == -1 && C[2] == 0 && C[3] == 0, "");
-     
+
   constexpr auto D = FourFloatsVecSize{1, 2, 3, 4} > 3;
   static_assert(D[0] == 0 && D[1] == 0 && D[2] == 0 && D[3] == -1, "");
 
@@ -607,7 +607,7 @@ void FloatUsage() {
   static_assert(G[0] == 0 && G[1] == 0 && G[2] == -1 && G[3] == 0, "");
 
   constexpr auto H = FourFloatsVecSize{1, 2, 3, 4} != 3;
-  static_assert(H[0] == -1 && H[1] == -1 && H[2] == 0 && H[3] == -1, "");  
+  static_assert(H[0] == -1 && H[1] == -1 && H[2] == 0 && H[3] == -1, "");
 
   constexpr auto O = FourFloatsVecSize{5, 0, 6, 0} &&
                      FourFloatsVecSize{5, 5, 0, 0};
@@ -655,7 +655,7 @@ void FloatVecUsage() {
   constexpr auto c = FourFloatsVecSize{8, 4, 2, 1} *
                      FourFloatsVecSize{3, 4, 5, 6};
   static_assert(c[0] == 2.400000e+01 && c[1] == 1.600000e+01 && c[2] == 1.000000e+01 && c[3] == 6.000000e+00, "");
-                      
+
   constexpr auto d = FourFloatsVecSize{12, 12, 10, 10} /
                      FourFloatsVecSize{6, 4, 5, 2};
   static_assert(d[0] == 2.000000e+00 && d[1] == 3.000000e+00 && d[2] == 2.000000e+00 && d[3] == 5.000000e+00, "");
@@ -741,14 +741,14 @@ void FloatVecUsage() {
   static_assert(R[0] == 1 && R[1] == 1 && R[2] == 1 && R[3] == 1, "");
 
   constexpr auto T = CmpMul(a, b);
-  static_assert(T[0] == 1.080000e+02 && T[1] == 1.800000e+01 && T[2] == 5.600000e+01 && T[3] == 7.200000e+01, ""); 
+  static_assert(T[0] == 1.080000e+02 && T[1] == 1.800000e+01 && T[2] == 5.600000e+01 && T[3] == 7.200000e+01, "");
 
   constexpr auto U = CmpDiv(a, b);
   static_assert(CmpF(U[0], a[0] / b[0]) && CmpF(U[1], a[1] / b[1]) && CmpF(U[2], a[2] / b[2]) && CmpF(U[3], a[3] / b[3]), "");
 
   constexpr auto X = CmpAdd(a, b);
   static_assert(X[0] == 2.400000e+01 && X[1] == 1.900000e+01 && X[2] == 1.500000e+01 && X[3] == 1.700000e+01, "");
- 
+
   constexpr auto Y = CmpSub(a, b);
   static_assert(Y[0] == 1.200000e+01 && Y[1] == 1.700000e+01 && Y[2] == -1.000000e+00 && Y[3] == -1.000000e+00, "");
 
@@ -826,7 +826,7 @@ void BoolVecUsage() {
 
   constexpr auto i = FourBoolsExtVec{true, false, true, false} ^
                      FourBoolsExtVec { false, false, true, true };
-  static_assert(i[0] == true && i[1] == false && i[2] == false && i[3] == true, "");  
+  static_assert(i[0] == true && i[1] == false && i[2] == false && i[3] == true, "");
 
   constexpr auto j = !FourBoolsExtVec{true, false, true, false};
   static_assert(j[0] == false && j[1] == true && j[2] == false && j[3] == true, "");

From c5b611a419ca8acab041ca52a94c6338bdcd1756 Mon Sep 17 00:00:00 2001
From: Jannick Kremer <jannick.kremer@mailbox.org>
Date: Fri, 16 Aug 2024 00:32:58 +0200
Subject: [PATCH 090/441] [libclang/python] Expose
 `clang_isBeforeInTranslationUnit` for `SourceRange.__contains__`

Add libclang function `clang_isBeforeInTranslationUnit` to allow checking the order between two source locations.
Simplify the `SourceRange.__contains__` implementation using this new function.
Add tests for `SourceRange.__contains__` and the newly added functionality.

Fixes #22617
Fixes #52827
---
 clang/bindings/python/clang/cindex.py         | 28 ++----
 .../python/tests/cindex/test_location.py      | 20 +++++
 .../python/tests/cindex/test_source_range.py  | 85 +++++++++++++++++++
 clang/docs/ReleaseNotes.rst                   |  5 ++
 clang/include/clang-c/CXSourceLocation.h      | 10 +++
 clang/lib/Basic/SourceManager.cpp             |  3 +-
 clang/tools/libclang/CXSourceLocation.cpp     | 12 +++
 clang/tools/libclang/libclang.map             |  5 ++
 8 files changed, 146 insertions(+), 22 deletions(-)
 create mode 100644 clang/bindings/python/tests/cindex/test_source_range.py

diff --git a/clang/bindings/python/clang/cindex.py b/clang/bindings/python/clang/cindex.py
index c251c46a04adf43..4da99e899e7f7ce 100644
--- a/clang/bindings/python/clang/cindex.py
+++ b/clang/bindings/python/clang/cindex.py
@@ -319,6 +319,12 @@ def __eq__(self, other):
     def __ne__(self, other):
         return not self.__eq__(other)
 
+    def __lt__(self, other: SourceLocation) -> bool:
+        return conf.lib.clang_isBeforeInTranslationUnit(self, other)  # type: ignore [no-any-return]
+
+    def __le__(self, other: SourceLocation) -> bool:
+        return self < other or self == other
+
     def __repr__(self):
         if self.file:
             filename = self.file.name
@@ -375,26 +381,7 @@ def __contains__(self, other):
         """Useful to detect the Token/Lexer bug"""
         if not isinstance(other, SourceLocation):
             return False
-        if other.file is None and self.start.file is None:
-            pass
-        elif (
-            self.start.file.name != other.file.name
-            or other.file.name != self.end.file.name
-        ):
-            # same file name
-            return False
-        # same file, in between lines
-        if self.start.line < other.line < self.end.line:
-            return True
-        elif self.start.line == other.line:
-            # same file first line
-            if self.start.column <= other.column:
-                return True
-        elif other.line == self.end.line:
-            # same file last line
-            if other.column <= self.end.column:
-                return True
-        return False
+        return self.start <= other <= self.end
 
     def __repr__(self):
         return "<SourceRange start %r, end %r>" % (self.start, self.end)
@@ -3908,6 +3895,7 @@ def write_main_file_to_stdout(self):
     ("clang_isUnexposed", [CursorKind], bool),
     ("clang_isVirtualBase", [Cursor], bool),
     ("clang_isVolatileQualifiedType", [Type], bool),
+    ("clang_isBeforeInTranslationUnit", [SourceLocation, SourceLocation], bool),
     (
         "clang_parseTranslationUnit",
         [Index, c_interop_string, c_void_p, c_int, c_void_p, c_int, c_int],
diff --git a/clang/bindings/python/tests/cindex/test_location.py b/clang/bindings/python/tests/cindex/test_location.py
index e23677a9be3c095..27854a312e6721d 100644
--- a/clang/bindings/python/tests/cindex/test_location.py
+++ b/clang/bindings/python/tests/cindex/test_location.py
@@ -130,3 +130,23 @@ def test_is_system_location(self):
         two = get_cursor(tu, "two")
         self.assertFalse(one.location.is_in_system_header)
         self.assertTrue(two.location.is_in_system_header)
+
+    def test_operator_lt(self):
+        tu = get_tu("aaaaa")
+        t1_f = tu.get_file(tu.spelling)
+        tu2 = get_tu("aaaaa")
+
+        l_t1_12 = SourceLocation.from_position(tu, t1_f, 1, 2)
+        l_t1_13 = SourceLocation.from_position(tu, t1_f, 1, 3)
+        l_t1_14 = SourceLocation.from_position(tu, t1_f, 1, 4)
+
+        l_t2_13 = SourceLocation.from_position(tu2, tu2.get_file(tu2.spelling), 1, 3)
+
+        # In same file
+        assert l_t1_12 < l_t1_13 < l_t1_14
+        assert not l_t1_13 < l_t1_12
+
+        # In same file, different TU
+        assert l_t1_12 < l_t2_13 < l_t1_14
+        assert not l_t2_13 < l_t1_12
+        assert not l_t1_14 < l_t2_13
diff --git a/clang/bindings/python/tests/cindex/test_source_range.py b/clang/bindings/python/tests/cindex/test_source_range.py
new file mode 100644
index 000000000000000..47d8960fcafb352
--- /dev/null
+++ b/clang/bindings/python/tests/cindex/test_source_range.py
@@ -0,0 +1,85 @@
+import os
+from clang.cindex import Config
+
+if "CLANG_LIBRARY_PATH" in os.environ:
+    Config.set_library_path(os.environ["CLANG_LIBRARY_PATH"])
+
+import unittest
+from clang.cindex import SourceLocation, SourceRange, TranslationUnit
+
+from .util import get_tu
+
+
+def create_range(tu, line1, column1, line2, column2):
+    return SourceRange.from_locations(
+        SourceLocation.from_position(tu, tu.get_file(tu.spelling), line1, column1),
+        SourceLocation.from_position(tu, tu.get_file(tu.spelling), line2, column2),
+    )
+
+
+class TestSourceRange(unittest.TestCase):
+    def test_contains(self):
+        tu = get_tu(
+            """aaaaa
+aaaaa
+aaaaa
+aaaaa"""
+        )
+        file = tu.get_file(tu.spelling)
+
+        l13 = SourceLocation.from_position(tu, file, 1, 3)
+        l21 = SourceLocation.from_position(tu, file, 2, 1)
+        l22 = SourceLocation.from_position(tu, file, 2, 2)
+        l23 = SourceLocation.from_position(tu, file, 2, 3)
+        l24 = SourceLocation.from_position(tu, file, 2, 4)
+        l25 = SourceLocation.from_position(tu, file, 2, 5)
+        l33 = SourceLocation.from_position(tu, file, 3, 3)
+        l31 = SourceLocation.from_position(tu, file, 3, 1)
+        r22_24 = create_range(tu, 2, 2, 2, 4)
+        r23_23 = create_range(tu, 2, 3, 2, 3)
+        r24_32 = create_range(tu, 2, 4, 3, 2)
+        r14_32 = create_range(tu, 1, 4, 3, 2)
+
+        assert l13 not in r22_24  # Line before start
+        assert l21 not in r22_24  # Column before start
+        assert l22 in r22_24  # Colum on start
+        assert l23 in r22_24  # Column in range
+        assert l24 in r22_24  # Column on end
+        assert l25 not in r22_24  # Column after end
+        assert l33 not in r22_24  # Line after end
+
+        assert l23 in r23_23  # In one-column range
+
+        assert l23 not in r24_32  # Outside range in first line
+        assert l33 not in r24_32  # Outside range in last line
+        assert l25 in r24_32  # In range in first line
+        assert l31 in r24_32  # In range in last line
+
+        assert l21 in r14_32  # In range at start of center line
+        assert l25 in r14_32  # In range at end of center line
+
+        # In range within included file
+        tu2 = TranslationUnit.from_source(
+            "main.c",
+            unsaved_files=[
+                (
+                    "main.c",
+                    """int a[] = {
+#include "numbers.inc"
+};
+""",
+                ),
+                (
+                    "./numbers.inc",
+                    """1,
+2,
+3,
+4
+                 """,
+                ),
+            ],
+        )
+
+        r_curly = create_range(tu2, 1, 11, 3, 1)
+        l_f2 = SourceLocation.from_position(tu2, tu2.get_file("./numbers.inc"), 4, 1)
+        assert l_f2 in r_curly
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 14f1eecc5748ed0..598fccfe59bff1b 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -90,6 +90,9 @@ Clang Python Bindings Potentially Breaking Changes
 - Calling a property on the ``CompletionChunk`` or ``CompletionString`` class
   statically now leads to an error, instead of returning a ``CachedProperty`` object
   that is used internally. Properties are only available on instances.
+- For a single-line ``SourceRange`` and a ``SourceLocation`` in the same line,
+  but after the end of the ``SourceRange``, ``SourceRange.__contains__``
+  used to incorrectly return ``True``. (#GH22617), (#GH52827)
 
 What's New in Clang |release|?
 ==============================
@@ -364,6 +367,8 @@ clang-format
 
 libclang
 --------
+- Add ``clang_isBeforeInTranslationUnit``. Given two source locations, it determines
+  whether the first one comes strictly before the second in the source code.
 
 Static Analyzer
 ---------------
diff --git a/clang/include/clang-c/CXSourceLocation.h b/clang/include/clang-c/CXSourceLocation.h
index dcb13ba273173fd..421802151d02ab8 100644
--- a/clang/include/clang-c/CXSourceLocation.h
+++ b/clang/include/clang-c/CXSourceLocation.h
@@ -74,6 +74,16 @@ CINDEX_LINKAGE CXSourceLocation clang_getNullLocation(void);
 CINDEX_LINKAGE unsigned clang_equalLocations(CXSourceLocation loc1,
                                              CXSourceLocation loc2);
 
+/**
+ * Determine for two source locations if the first comes
+ * strictly before the second one in the source code.
+ *
+ * \returns non-zero if the first source location comes
+ * strictly before the second one, zero otherwise.
+ */
+CINDEX_LINKAGE unsigned clang_isBeforeInTranslationUnit(CXSourceLocation loc1,
+                                                        CXSourceLocation loc2);
+
 /**
  * Returns non-zero if the given source location is in a system header.
  */
diff --git a/clang/lib/Basic/SourceManager.cpp b/clang/lib/Basic/SourceManager.cpp
index 533a9fe88a21508..b0256a8ce9ed042 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -2038,8 +2038,7 @@ bool SourceManager::isBeforeInTranslationUnit(SourceLocation LHS,
   std::pair<bool, bool> InSameTU = isInTheSameTranslationUnit(LOffs, ROffs);
   if (InSameTU.first)
     return InSameTU.second;
-  // TODO: This should be unreachable, but some clients are calling this
-  //       function before making sure LHS and RHS are in the same TU.
+  // This case is used by libclang: clang_isBeforeInTranslationUnit
   return LOffs.first < ROffs.first;
 }
 
diff --git a/clang/tools/libclang/CXSourceLocation.cpp b/clang/tools/libclang/CXSourceLocation.cpp
index 53cb71f7276f299..cd41e9f0d64031b 100644
--- a/clang/tools/libclang/CXSourceLocation.cpp
+++ b/clang/tools/libclang/CXSourceLocation.cpp
@@ -50,6 +50,18 @@ unsigned clang_equalLocations(CXSourceLocation loc1, CXSourceLocation loc2) {
           loc1.int_data == loc2.int_data);
 }
 
+unsigned clang_isBeforeInTranslationUnit(CXSourceLocation loc1,
+                                         CXSourceLocation loc2) {
+  const SourceLocation Loc1 = SourceLocation::getFromRawEncoding(loc1.int_data);
+  const SourceLocation Loc2 = SourceLocation::getFromRawEncoding(loc2.int_data);
+
+  const SourceManager &SM =
+      *static_cast<const SourceManager *>(loc1.ptr_data[0]);
+  // Use the appropriate SourceManager method here rather than operator< because
+  // ordering is meaningful only if LHS and RHS have the same FileID.
+  return SM.isBeforeInTranslationUnit(Loc1, Loc2);
+}
+
 CXSourceRange clang_getNullRange() {
   CXSourceRange Result = { { nullptr, nullptr }, 0, 0 };
   return Result;
diff --git a/clang/tools/libclang/libclang.map b/clang/tools/libclang/libclang.map
index 371fe512ce71c1f..25d8ba57d325149 100644
--- a/clang/tools/libclang/libclang.map
+++ b/clang/tools/libclang/libclang.map
@@ -434,6 +434,11 @@ LLVM_19 {
     clang_Cursor_getBinaryOpcodeStr;
 };
 
+LLVM_20 {
+  global:
+    clang_isBeforeInTranslationUnit;
+};
+
 # Example of how to add a new symbol version entry.  If you do add a new symbol
 # version, please update the example to depend on the version you added.
 # LLVM_X {

From 7156bcf2867f4ca98c8c9166e1ecb77daab08aa5 Mon Sep 17 00:00:00 2001
From: Johannes Doerfert <johannes@jdoerfert.de>
Date: Thu, 15 Aug 2024 15:45:36 -0700
Subject: [PATCH 091/441] [Attributor][FIX] Ensure we do not use stale
 references (#104495)

When copying map entries, we might run into resizing and invalidate the
RHS of the assignment. We dealt with this before and now use the proper
helper to avoid the problem in another place.

Fixes: https://github.com/llvm/llvm-project/issues/104397
---
 .../Transforms/IPO/AttributorAttributes.cpp   |   7 +-
 .../aapointer_info_map_invalidation.ll        | 588 ++++++++++++++++++
 2 files changed, 593 insertions(+), 2 deletions(-)
 create mode 100644 llvm/test/Transforms/Attributor/reduced/aapointer_info_map_invalidation.ll

diff --git a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
index 41a7fc0870cf0ad..9f033028347f0d2 100644
--- a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
+++ b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
@@ -1628,6 +1628,8 @@ ChangeStatus AAPointerInfoFloating::updateImpl(Attributor &A) {
                       << "\n");
     assert(OffsetInfoMap.count(CurPtr) &&
            "The current pointer offset should have been seeded!");
+    assert(!OffsetInfoMap[CurPtr].isUnassigned() &&
+           "Current pointer should be assigned");
 
     if (ConstantExpr *CE = dyn_cast<ConstantExpr>(Usr)) {
       if (CE->isCast())
@@ -1906,6 +1908,7 @@ ChangeStatus AAPointerInfoFloating::updateImpl(Attributor &A) {
   };
   auto EquivalentUseCB = [&](const Use &OldU, const Use &NewU) {
     assert(OffsetInfoMap.count(OldU) && "Old use should be known already!");
+    assert(!OffsetInfoMap[OldU].isUnassigned() && "Old use should be assinged");
     if (OffsetInfoMap.count(NewU)) {
       LLVM_DEBUG({
         if (!(OffsetInfoMap[NewU] == OffsetInfoMap[OldU])) {
@@ -1916,8 +1919,8 @@ ChangeStatus AAPointerInfoFloating::updateImpl(Attributor &A) {
       });
       return OffsetInfoMap[NewU] == OffsetInfoMap[OldU];
     }
-    OffsetInfoMap[NewU] = OffsetInfoMap[OldU];
-    return true;
+    bool Unused;
+    return HandlePassthroughUser(NewU.get(), OldU.get(), Unused);
   };
   if (!A.checkForAllUses(UsePred, *this, AssociatedValue,
                          /* CheckBBLivenessOnly */ true, DepClassTy::OPTIONAL,
diff --git a/llvm/test/Transforms/Attributor/reduced/aapointer_info_map_invalidation.ll b/llvm/test/Transforms/Attributor/reduced/aapointer_info_map_invalidation.ll
new file mode 100644
index 000000000000000..a7622143c2e7366
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/reduced/aapointer_info_map_invalidation.ll
@@ -0,0 +1,588 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --scrub-attributes --check-attributes
+; RUN: opt -passes=attributor -S < %s | FileCheck %s --check-prefixes=CHECK
+
+define amdgpu_kernel void @__omp_offloading_fd00_2c00523__ZN11qmcplusplus7ompBLAS9gemv_implIfEEiRiciiT_PKS3_iS5_iS3_PS3_i_l383() {
+; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn
+; CHECK-LABEL: define {{[^@]+}}@__omp_offloading_fd00_2c00523__ZN11qmcplusplus7ompBLAS9gemv_implIfEEiRiciiT_PKS3_iS5_iS3_PS3_i_l383
+; CHECK-SAME: () #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca [0 x [0 x float]], i32 0, align 8, addrspace(5)
+; CHECK-NEXT:    [[TMP2:%.*]] = addrspacecast ptr addrspace(5) [[TMP1]] to ptr
+; CHECK-NEXT:    store ptr [[TMP2]], ptr addrspace(5) [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = call fastcc i32 @__kmpc_nvptx_parallel_reduce_nowait_v2(ptr nocapture nofree noundef readonly align 8 dereferenceable_or_null(8) [[TMP2]], i1 noundef false)
+; CHECK-NEXT:    ret void
+;
+  %1 = alloca [0 x [0 x float]], i32 0, align 8, addrspace(5)
+  %2 = addrspacecast ptr addrspace(5) %1 to ptr
+  store ptr %2, ptr addrspace(5) %1, align 8
+  %3 = call fastcc i32 @__kmpc_nvptx_parallel_reduce_nowait_v2(ptr %2, i1 false)
+  ret void
+}
+
+define fastcc i32 @__kmpc_nvptx_parallel_reduce_nowait_v2(ptr %0, i1 %1) {
+; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn
+; CHECK-LABEL: define {{[^@]+}}@__kmpc_nvptx_parallel_reduce_nowait_v2
+; CHECK-SAME: (ptr nocapture nofree nonnull readonly align 8 dereferenceable(8) [[TMP0:%.*]], i1 noundef [[TMP1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP3:%.*]], label [[TMP79:%.*]]
+; CHECK:       3:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP4:%.*]], label [[TMP6:%.*]]
+; CHECK:       4:
+; CHECK-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i64 0, ptr [[TMP5]], align 8
+; CHECK-NEXT:    br label [[TMP77:%.*]]
+; CHECK:       6:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP7:%.*]], label [[TMP9:%.*]]
+; CHECK:       7:
+; CHECK-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[TMP8]], align 4
+; CHECK-NEXT:    br label [[TMP77]]
+; CHECK:       9:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP10:%.*]], label [[TMP12:%.*]]
+; CHECK:       10:
+; CHECK-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[TMP11]], align 4
+; CHECK-NEXT:    br label [[TMP77]]
+; CHECK:       12:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP13:%.*]], label [[TMP15:%.*]]
+; CHECK:       13:
+; CHECK-NEXT:    [[TMP14:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[TMP14]], align 4
+; CHECK-NEXT:    br label [[TMP77]]
+; CHECK:       15:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP16:%.*]], label [[TMP18:%.*]]
+; CHECK:       16:
+; CHECK-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[TMP17]], align 4
+; CHECK-NEXT:    br label [[TMP77]]
+; CHECK:       18:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP19:%.*]], label [[TMP21:%.*]]
+; CHECK:       19:
+; CHECK-NEXT:    [[TMP20:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[TMP20]], align 4
+; CHECK-NEXT:    br label [[TMP77]]
+; CHECK:       21:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP22:%.*]], label [[TMP24:%.*]]
+; CHECK:       22:
+; CHECK-NEXT:    [[TMP23:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[TMP23]], align 4
+; CHECK-NEXT:    br label [[TMP77]]
+; CHECK:       24:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP25:%.*]], label [[TMP27:%.*]]
+; CHECK:       25:
+; CHECK-NEXT:    [[TMP26:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[TMP26]], align 8
+; CHECK-NEXT:    br label [[TMP77]]
+; CHECK:       27:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP28:%.*]], label [[TMP30:%.*]]
+; CHECK:       28:
+; CHECK-NEXT:    [[TMP29:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[TMP29]], align 4
+; CHECK-NEXT:    br label [[TMP77]]
+; CHECK:       30:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP31:%.*]], label [[TMP33:%.*]]
+; CHECK:       31:
+; CHECK-NEXT:    [[TMP32:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[TMP32]], align 4
+; CHECK-NEXT:    br label [[TMP77]]
+; CHECK:       33:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP34:%.*]], label [[TMP36:%.*]]
+; CHECK:       34:
+; CHECK-NEXT:    [[TMP35:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[TMP35]], align 4
+; CHECK-NEXT:    br label [[TMP77]]
+; CHECK:       36:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP37:%.*]], label [[TMP39:%.*]]
+; CHECK:       37:
+; CHECK-NEXT:    [[TMP38:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i64 0, ptr [[TMP38]], align 8
+; CHECK-NEXT:    br label [[TMP77]]
+; CHECK:       39:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP40:%.*]], label [[TMP42:%.*]]
+; CHECK:       40:
+; CHECK-NEXT:    [[TMP41:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i64 0, ptr [[TMP41]], align 8
+; CHECK-NEXT:    br label [[TMP77]]
+; CHECK:       42:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP43:%.*]], label [[TMP45:%.*]]
+; CHECK:       43:
+; CHECK-NEXT:    [[TMP44:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[TMP44]], align 4
+; CHECK-NEXT:    br label [[TMP77]]
+; CHECK:       45:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP46:%.*]], label [[TMP48:%.*]]
+; CHECK:       46:
+; CHECK-NEXT:    [[TMP47:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[TMP47]], align 4
+; CHECK-NEXT:    br label [[TMP77]]
+; CHECK:       48:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP49:%.*]], label [[TMP51:%.*]]
+; CHECK:       49:
+; CHECK-NEXT:    [[TMP50:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i64 0, ptr [[TMP50]], align 8
+; CHECK-NEXT:    br label [[TMP77]]
+; CHECK:       51:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP52:%.*]], label [[TMP54:%.*]]
+; CHECK:       52:
+; CHECK-NEXT:    [[TMP53:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i64 0, ptr [[TMP53]], align 8
+; CHECK-NEXT:    br label [[TMP77]]
+; CHECK:       54:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP55:%.*]], label [[TMP57:%.*]]
+; CHECK:       55:
+; CHECK-NEXT:    [[TMP56:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[TMP56]], align 4
+; CHECK-NEXT:    br label [[TMP77]]
+; CHECK:       57:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP58:%.*]], label [[TMP60:%.*]]
+; CHECK:       58:
+; CHECK-NEXT:    [[TMP59:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[TMP59]], align 4
+; CHECK-NEXT:    br label [[TMP77]]
+; CHECK:       60:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP61:%.*]], label [[TMP63:%.*]]
+; CHECK:       61:
+; CHECK-NEXT:    [[TMP62:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i64 0, ptr [[TMP62]], align 8
+; CHECK-NEXT:    br label [[TMP77]]
+; CHECK:       63:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP64:%.*]], label [[TMP66:%.*]]
+; CHECK:       64:
+; CHECK-NEXT:    [[TMP65:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i64 0, ptr [[TMP65]], align 8
+; CHECK-NEXT:    br label [[TMP77]]
+; CHECK:       66:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP67:%.*]], label [[TMP69:%.*]]
+; CHECK:       67:
+; CHECK-NEXT:    [[TMP68:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[TMP68]], align 4
+; CHECK-NEXT:    br label [[TMP77]]
+; CHECK:       69:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP70:%.*]], label [[TMP72:%.*]]
+; CHECK:       70:
+; CHECK-NEXT:    [[TMP71:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[TMP71]], align 4
+; CHECK-NEXT:    br label [[TMP77]]
+; CHECK:       72:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP73:%.*]], label [[TMP75:%.*]]
+; CHECK:       73:
+; CHECK-NEXT:    [[TMP74:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[TMP74]], align 4
+; CHECK-NEXT:    br label [[TMP77]]
+; CHECK:       75:
+; CHECK-NEXT:    [[TMP76:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[TMP76]], align 4
+; CHECK-NEXT:    br label [[TMP77]]
+; CHECK:       77:
+; CHECK-NEXT:    [[TMP78:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[TMP78]], align 4
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP80:%.*]], label [[TMP82:%.*]]
+; CHECK:       79:
+; CHECK-NEXT:    ret i32 0
+; CHECK:       80:
+; CHECK-NEXT:    [[TMP81:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i64 0, ptr [[TMP81]], align 8
+; CHECK-NEXT:    ret i32 0
+; CHECK:       82:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP83:%.*]], label [[TMP85:%.*]]
+; CHECK:       83:
+; CHECK-NEXT:    [[TMP84:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[TMP84]], align 4
+; CHECK-NEXT:    ret i32 0
+; CHECK:       85:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP86:%.*]], label [[TMP88:%.*]]
+; CHECK:       86:
+; CHECK-NEXT:    [[TMP87:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[TMP87]], align 4
+; CHECK-NEXT:    ret i32 0
+; CHECK:       88:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP89:%.*]], label [[TMP91:%.*]]
+; CHECK:       89:
+; CHECK-NEXT:    [[TMP90:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[TMP90]], align 4
+; CHECK-NEXT:    ret i32 0
+; CHECK:       91:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP92:%.*]], label [[TMP94:%.*]]
+; CHECK:       92:
+; CHECK-NEXT:    [[TMP93:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[TMP93]], align 4
+; CHECK-NEXT:    ret i32 0
+; CHECK:       94:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP95:%.*]], label [[TMP97:%.*]]
+; CHECK:       95:
+; CHECK-NEXT:    [[TMP96:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[TMP96]], align 4
+; CHECK-NEXT:    ret i32 0
+; CHECK:       97:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP98:%.*]], label [[TMP100:%.*]]
+; CHECK:       98:
+; CHECK-NEXT:    [[TMP99:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[TMP99]], align 4
+; CHECK-NEXT:    ret i32 0
+; CHECK:       100:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP101:%.*]], label [[TMP103:%.*]]
+; CHECK:       101:
+; CHECK-NEXT:    [[TMP102:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[TMP102]], align 8
+; CHECK-NEXT:    ret i32 0
+; CHECK:       103:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP104:%.*]], label [[TMP106:%.*]]
+; CHECK:       104:
+; CHECK-NEXT:    [[TMP105:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[TMP105]], align 4
+; CHECK-NEXT:    ret i32 0
+; CHECK:       106:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP107:%.*]], label [[TMP109:%.*]]
+; CHECK:       107:
+; CHECK-NEXT:    [[TMP108:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[TMP108]], align 4
+; CHECK-NEXT:    ret i32 0
+; CHECK:       109:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP110:%.*]], label [[TMP112:%.*]]
+; CHECK:       110:
+; CHECK-NEXT:    [[TMP111:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[TMP111]], align 4
+; CHECK-NEXT:    ret i32 0
+; CHECK:       112:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP113:%.*]], label [[TMP115:%.*]]
+; CHECK:       113:
+; CHECK-NEXT:    [[TMP114:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i64 0, ptr [[TMP114]], align 8
+; CHECK-NEXT:    ret i32 0
+; CHECK:       115:
+; CHECK-NEXT:    br i1 [[TMP1]], label [[TMP116:%.*]], label [[TMP118:%.*]]
+; CHECK:       116:
+; CHECK-NEXT:    [[TMP117:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i64 0, ptr [[TMP117]], align 8
+; CHECK-NEXT:    ret i32 0
+; CHECK:       118:
+; CHECK-NEXT:    [[TMP119:%.*]] = load ptr, ptr [[TMP0]], align 8
+; CHECK-NEXT:    store i32 0, ptr [[TMP119]], align 4
+; CHECK-NEXT:    ret i32 0
+;
+  br i1 %1, label %3, label %88
+
+3:                                                ; preds = %2
+  %4 = load ptr, ptr %0, align 8
+  br i1 %1, label %5, label %7
+
+5:                                                ; preds = %3
+  %6 = load ptr, ptr %0, align 8
+  store i64 0, ptr %6, align 8
+  br label %79
+
+7:                                                ; preds = %3
+  br i1 %1, label %8, label %11
+
+8:                                                ; preds = %7
+  %9 = load i32, ptr %4, align 4
+  %10 = load ptr, ptr %0, align 8
+  store i32 0, ptr %10, align 4
+  br label %79
+
+11:                                               ; preds = %7
+  br i1 %1, label %12, label %14
+
+12:                                               ; preds = %11
+  %13 = load ptr, ptr %0, align 8
+  store i32 0, ptr %13, align 4
+  br label %79
+
+14:                                               ; preds = %11
+  br i1 %1, label %15, label %17
+
+15:                                               ; preds = %14
+  %16 = load ptr, ptr %0, align 8
+  store i32 0, ptr %16, align 4
+  br label %79
+
+17:                                               ; preds = %14
+  br i1 %1, label %18, label %20
+
+18:                                               ; preds = %17
+  %19 = load ptr, ptr %0, align 8
+  store i32 0, ptr %19, align 4
+  br label %79
+
+20:                                               ; preds = %17
+  br i1 %1, label %21, label %23
+
+21:                                               ; preds = %20
+  %22 = load ptr, ptr %0, align 8
+  store i32 0, ptr %22, align 4
+  br label %79
+
+23:                                               ; preds = %20
+  br i1 %1, label %24, label %26
+
+24:                                               ; preds = %23
+  %25 = load ptr, ptr %0, align 8
+  store i32 0, ptr %25, align 4
+  br label %79
+
+26:                                               ; preds = %23
+  br i1 %1, label %27, label %29
+
+27:                                               ; preds = %26
+  %28 = load ptr, ptr %0, align 8
+  store i32 0, ptr %28, align 8
+  br label %79
+
+29:                                               ; preds = %26
+  br i1 %1, label %30, label %32
+
+30:                                               ; preds = %29
+  %31 = load ptr, ptr %0, align 8
+  store i32 0, ptr %31, align 4
+  br label %79
+
+32:                                               ; preds = %29
+  br i1 %1, label %33, label %35
+
+33:                                               ; preds = %32
+  %34 = load ptr, ptr %0, align 8
+  store i32 0, ptr %34, align 4
+  br label %79
+
+35:                                               ; preds = %32
+  br i1 %1, label %36, label %38
+
+36:                                               ; preds = %35
+  %37 = load ptr, ptr %0, align 8
+  store i32 0, ptr %37, align 4
+  br label %79
+
+38:                                               ; preds = %35
+  br i1 %1, label %39, label %41
+
+39:                                               ; preds = %38
+  %40 = load ptr, ptr %0, align 8
+  store i64 0, ptr %40, align 8
+  br label %79
+
+41:                                               ; preds = %38
+  br i1 %1, label %42, label %44
+
+42:                                               ; preds = %41
+  %43 = load ptr, ptr %0, align 8
+  store i64 0, ptr %43, align 8
+  br label %79
+
+44:                                               ; preds = %41
+  br i1 %1, label %45, label %47
+
+45:                                               ; preds = %44
+  %46 = load ptr, ptr %0, align 8
+  store i32 0, ptr %46, align 4
+  br label %79
+
+47:                                               ; preds = %44
+  br i1 %1, label %48, label %50
+
+48:                                               ; preds = %47
+  %49 = load ptr, ptr %0, align 8
+  store i32 0, ptr %49, align 4
+  br label %79
+
+50:                                               ; preds = %47
+  br i1 %1, label %51, label %53
+
+51:                                               ; preds = %50
+  %52 = load ptr, ptr %0, align 8
+  store i64 0, ptr %52, align 8
+  br label %79
+
+53:                                               ; preds = %50
+  br i1 %1, label %54, label %56
+
+54:                                               ; preds = %53
+  %55 = load ptr, ptr %0, align 8
+  store i64 0, ptr %55, align 8
+  br label %79
+
+56:                                               ; preds = %53
+  br i1 %1, label %57, label %59
+
+57:                                               ; preds = %56
+  %58 = load ptr, ptr %0, align 8
+  store i32 0, ptr %58, align 4
+  br label %79
+
+59:                                               ; preds = %56
+  br i1 %1, label %60, label %62
+
+60:                                               ; preds = %59
+  %61 = load ptr, ptr %0, align 8
+  store i32 0, ptr %61, align 4
+  br label %79
+
+62:                                               ; preds = %59
+  br i1 %1, label %63, label %65
+
+63:                                               ; preds = %62
+  %64 = load ptr, ptr %0, align 8
+  store i64 0, ptr %64, align 8
+  br label %79
+
+65:                                               ; preds = %62
+  br i1 %1, label %66, label %68
+
+66:                                               ; preds = %65
+  %67 = load ptr, ptr %0, align 8
+  store i64 0, ptr %67, align 8
+  br label %79
+
+68:                                               ; preds = %65
+  br i1 %1, label %69, label %71
+
+69:                                               ; preds = %68
+  %70 = load ptr, ptr %0, align 8
+  store i32 0, ptr %70, align 4
+  br label %79
+
+71:                                               ; preds = %68
+  br i1 %1, label %72, label %74
+
+72:                                               ; preds = %71
+  %73 = load ptr, ptr %0, align 8
+  store i32 0, ptr %73, align 4
+  br label %79
+
+74:                                               ; preds = %71
+  br i1 %1, label %75, label %77
+
+75:                                               ; preds = %74
+  %76 = load ptr, ptr %0, align 8
+  store i32 0, ptr %76, align 4
+  br label %79
+
+77:                                               ; preds = %74
+  %78 = load ptr, ptr %0, align 8
+  store i32 0, ptr %78, align 4
+  br label %79
+
+79:                                               ; preds = %77, %75, %72, %69, %66, %63, %60, %57, %54, %51, %48, %45, %42, %39, %36, %33, %30, %27, %24, %21, %18, %15, %12, %8, %5
+  %80 = load ptr, ptr %0, align 8
+  %81 = load i32, ptr %80, align 4
+  %82 = load ptr, ptr %0, align 8
+  store i32 0, ptr %82, align 4
+  %83 = load ptr, ptr %0, align 8
+  %84 = getelementptr i8, ptr %83, i64 4
+  %85 = load ptr, ptr %0, align 8
+  %86 = getelementptr i8, ptr %85, i64 4
+  %87 = load ptr, ptr %0, align 8
+  br i1 %1, label %91, label %93
+
+88:                                               ; preds = %2
+  %89 = load ptr, ptr %0, align 8
+  %90 = load i64, ptr %89, align 8
+  ret i32 0
+
+91:                                               ; preds = %79
+  %92 = load ptr, ptr %0, align 8
+  store i64 0, ptr %92, align 8
+  ret i32 0
+
+93:                                               ; preds = %79
+  br i1 %1, label %94, label %96
+
+94:                                               ; preds = %93
+  %95 = load ptr, ptr %0, align 8
+  store i32 0, ptr %95, align 4
+  ret i32 0
+
+96:                                               ; preds = %93
+  br i1 %1, label %97, label %99
+
+97:                                               ; preds = %96
+  %98 = load ptr, ptr %0, align 8
+  store i32 0, ptr %98, align 4
+  ret i32 0
+
+99:                                               ; preds = %96
+  br i1 %1, label %100, label %102
+
+100:                                              ; preds = %99
+  %101 = load ptr, ptr %0, align 8
+  store i32 0, ptr %101, align 4
+  ret i32 0
+
+102:                                              ; preds = %99
+  br i1 %1, label %103, label %105
+
+103:                                              ; preds = %102
+  %104 = load ptr, ptr %0, align 8
+  store i32 0, ptr %104, align 4
+  ret i32 0
+
+105:                                              ; preds = %102
+  br i1 %1, label %106, label %108
+
+106:                                              ; preds = %105
+  %107 = load ptr, ptr %0, align 8
+  store i32 0, ptr %107, align 4
+  ret i32 0
+
+108:                                              ; preds = %105
+  br i1 %1, label %109, label %111
+
+109:                                              ; preds = %108
+  %110 = load ptr, ptr %0, align 8
+  store i32 0, ptr %110, align 4
+  ret i32 0
+
+111:                                              ; preds = %108
+  br i1 %1, label %112, label %114
+
+112:                                              ; preds = %111
+  %113 = load ptr, ptr %0, align 8
+  store i32 0, ptr %113, align 8
+  ret i32 0
+
+114:                                              ; preds = %111
+  br i1 %1, label %115, label %117
+
+115:                                              ; preds = %114
+  %116 = load ptr, ptr %0, align 8
+  store i32 0, ptr %116, align 4
+  ret i32 0
+
+117:                                              ; preds = %114
+  br i1 %1, label %118, label %121
+
+118:                                              ; preds = %117
+  %119 = load i32, ptr %87, align 4
+  %120 = load ptr, ptr %0, align 8
+  store i32 0, ptr %120, align 4
+  ret i32 0
+
+121:                                              ; preds = %117
+  br i1 %1, label %122, label %124
+
+122:                                              ; preds = %121
+  %123 = load ptr, ptr %0, align 8
+  store i32 0, ptr %123, align 4
+  ret i32 0
+
+124:                                              ; preds = %121
+  br i1 %1, label %125, label %127
+
+125:                                              ; preds = %124
+  %126 = load ptr, ptr %0, align 8
+  store i64 0, ptr %126, align 8
+  ret i32 0
+
+127:                                              ; preds = %124
+  br i1 %1, label %128, label %130
+
+128:                                              ; preds = %127
+  %129 = load ptr, ptr %0, align 8
+  store i64 0, ptr %129, align 8
+  ret i32 0
+
+130:                                              ; preds = %127
+  %131 = load ptr, ptr %0, align 8
+  store i32 0, ptr %131, align 4
+  ret i32 0
+}
+
+; uselistorder directives
+uselistorder i32 0, { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 0 }

From 3333ec1183803fa5a2353e710b8b22db6a72e292 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Thu, 15 Aug 2024 15:52:20 -0700
Subject: [PATCH 092/441] [Driver] Make CodeGenOptions name match
 MCTargetOptions names

* Initialize `X86RelaxRelocations`.
* Fix #96860 test to actually test -Wa,-msse2avx for non-x86.
---
 clang/include/clang/Basic/CodeGenOptions.def |  4 ++--
 clang/include/clang/Driver/Options.td        |  4 ++--
 clang/lib/CodeGen/BackendUtil.cpp            |  2 +-
 clang/test/Driver/msse2avx.c                 |  8 ++++----
 clang/tools/driver/cc1as_main.cpp            | 20 +++++++++++---------
 5 files changed, 20 insertions(+), 18 deletions(-)

diff --git a/clang/include/clang/Basic/CodeGenOptions.def b/clang/include/clang/Basic/CodeGenOptions.def
index 5728ab8d86702be..09e892d6d4defea 100644
--- a/clang/include/clang/Basic/CodeGenOptions.def
+++ b/clang/include/clang/Basic/CodeGenOptions.def
@@ -37,8 +37,6 @@ VALUE_CODEGENOPT(Name, Bits, Default)
 
 CODEGENOPT(DisableIntegratedAS, 1, 0) ///< -no-integrated-as
 CODEGENOPT(Crel, 1, 0) ///< -Wa,--crel
-CODEGENOPT(RelaxELFRelocations, 1, 1) ///< -Wa,-mrelax-relocations={yes,no}
-CODEGENOPT(SSE2AVX          , 1, 0)   ///< -msse2avx
 CODEGENOPT(AsmVerbose        , 1, 0) ///< -dA, -fverbose-asm.
 CODEGENOPT(PreserveAsmComments, 1, 1) ///< -dA, -fno-preserve-as-comments.
 CODEGENOPT(AssumeSaneOperatorNew , 1, 1) ///< implicit __attribute__((malloc)) operator new
@@ -194,6 +192,8 @@ CODEGENOPT(HIPSaveKernelArgName, 1, 0) ///< Set when -fhip-kernel-arg-name is en
 CODEGENOPT(UniqueInternalLinkageNames, 1, 0) ///< Internal Linkage symbols get unique names.
 CODEGENOPT(SplitMachineFunctions, 1, 0) ///< Split machine functions using profile information.
 CODEGENOPT(PPCUseFullRegisterNames, 1, 0) ///< Print full register names in assembly
+CODEGENOPT(X86RelaxRelocations, 1, 1) ///< -Wa,-mrelax-relocations={yes,no}
+CODEGENOPT(X86Sse2Avx        , 1, 0)   ///< -Wa,-msse2avx
 
 /// When false, this attempts to generate code as if the result of an
 /// overflowing conversion matches the overflowing behavior of a target's native
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 40df91dc3fe0e3d..c9ed08c20fc04f9 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -5231,7 +5231,7 @@ let Flags = [TargetSpecific] in {
 def msse2avx : Flag<["-"], "msse2avx">, Group<m_Group>,
   Visibility<[ClangOption, CC1Option, CC1AsOption]>,
   HelpText<"Specify that the assembler should encode SSE instructions with VEX prefix">,
-  MarshallingInfoFlag<CodeGenOpts<"SSE2AVX">>;
+  MarshallingInfoFlag<CodeGenOpts<"X86Sse2Avx">>;
 } // let Flags = [TargetSpecific]
 
 defm zvector : BoolFOption<"zvector",
@@ -7133,7 +7133,7 @@ def crel : Flag<["--"], "crel">,
   MarshallingInfoFlag<CodeGenOpts<"Crel">>;
 def mrelax_relocations_no : Flag<["-"], "mrelax-relocations=no">,
     HelpText<"Disable x86 relax relocations">,
-    MarshallingInfoNegativeFlag<CodeGenOpts<"RelaxELFRelocations">>;
+    MarshallingInfoNegativeFlag<CodeGenOpts<"X86RelaxRelocations">>;
 def msave_temp_labels : Flag<["-"], "msave-temp-labels">,
   HelpText<"Save temporary labels in the symbol table. "
            "Note this may change .s semantics and shouldn't generally be used "
diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp
index 81e6702d5de6661..34c08818dbb9ade 100644
--- a/clang/lib/CodeGen/BackendUtil.cpp
+++ b/clang/lib/CodeGen/BackendUtil.cpp
@@ -471,7 +471,7 @@ static bool initTargetOptions(DiagnosticsEngine &Diags,
   Options.MCOptions.Dwarf64 = CodeGenOpts.Dwarf64;
   Options.MCOptions.PreserveAsmComments = CodeGenOpts.PreserveAsmComments;
   Options.MCOptions.Crel = CodeGenOpts.Crel;
-  Options.MCOptions.X86RelaxRelocations = CodeGenOpts.RelaxELFRelocations;
+  Options.MCOptions.X86RelaxRelocations = CodeGenOpts.X86RelaxRelocations;
   Options.MCOptions.CompressDebugSections =
       CodeGenOpts.getCompressDebugSections();
   Options.MCOptions.ABIName = TargetOpts.ABI;
diff --git a/clang/test/Driver/msse2avx.c b/clang/test/Driver/msse2avx.c
index a63ac9a6c866815..829b1b2b79360d4 100644
--- a/clang/test/Driver/msse2avx.c
+++ b/clang/test/Driver/msse2avx.c
@@ -1,7 +1,7 @@
-// RUN: %clang -### -c -target x86_64 -march=x86-64 -Xassembler -msse2avx %s 2>&1 | FileCheck  %s
-// RUN: %clang -### -c -target x86_64 -march=x86-64 -x assembler -Xassembler -msse2avx %s 2>&1 | FileCheck %s
+// RUN: %clang -### -c --target=x86_64 -march=x86-64 -Wa,-msse2avx %s 2>&1 | FileCheck  %s
+// RUN: %clang -### -c --target=x86_64 -march=x86-64 -x assembler -Xassembler -msse2avx %s 2>&1 | FileCheck %s
 
 // CHECK: "-msse2avx"
 
-// RUN: not %clang -### -c -target aarch64 -march=armv8a -msse2avx %s 2>&1 | FileCheck --check-prefix=ERR %s
-// ERR:   error: unsupported option '-msse2avx' for target 'aarch64'
+// RUN: not %clang -### -c --target=aarch64 -march=armv8a -Wa,-msse2avx %s 2>&1 | FileCheck --check-prefix=ERR %s
+// ERR:   error: unsupported argument '-msse2avx' to option '-Wa,'
diff --git a/clang/tools/driver/cc1as_main.cpp b/clang/tools/driver/cc1as_main.cpp
index 78019d39742e64d..070cf8b44e8eb63 100644
--- a/clang/tools/driver/cc1as_main.cpp
+++ b/clang/tools/driver/cc1as_main.cpp
@@ -96,10 +96,6 @@ struct AssemblerInvocation {
   LLVM_PREFERRED_TYPE(bool)
   unsigned GenDwarfForAssembly : 1;
   LLVM_PREFERRED_TYPE(bool)
-  unsigned RelaxELFRelocations : 1;
-  LLVM_PREFERRED_TYPE(bool)
-  unsigned SSE2AVX : 1;
-  LLVM_PREFERRED_TYPE(bool)
   unsigned Dwarf64 : 1;
   unsigned DwarfVersion;
   std::string DwarfDebugFlags;
@@ -169,6 +165,11 @@ struct AssemblerInvocation {
   LLVM_PREFERRED_TYPE(bool)
   unsigned Crel : 1;
 
+  LLVM_PREFERRED_TYPE(bool)
+  unsigned X86RelaxRelocations : 1;
+  LLVM_PREFERRED_TYPE(bool)
+  unsigned X86Sse2Avx : 1;
+
   /// The name of the relocation model to use.
   std::string RelocationModel;
 
@@ -199,7 +200,6 @@ struct AssemblerInvocation {
     ShowInst = 0;
     ShowEncoding = 0;
     RelaxAll = 0;
-    SSE2AVX = 0;
     NoExecStack = 0;
     FatalWarnings = 0;
     NoWarn = 0;
@@ -211,6 +211,8 @@ struct AssemblerInvocation {
     EmitDwarfUnwind = EmitDwarfUnwindType::Default;
     EmitCompactUnwindNonCanonical = false;
     Crel = false;
+    X86RelaxRelocations = 0;
+    X86Sse2Avx = 0;
   }
 
   static bool CreateFromArgs(AssemblerInvocation &Res,
@@ -290,8 +292,6 @@ bool AssemblerInvocation::CreateFromArgs(AssemblerInvocation &Opts,
             .Default(llvm::DebugCompressionType::None);
   }
 
-  Opts.RelaxELFRelocations = !Args.hasArg(OPT_mrelax_relocations_no);
-  Opts.SSE2AVX = Args.hasArg(OPT_msse2avx);
   if (auto *DwarfFormatArg = Args.getLastArg(OPT_gdwarf64, OPT_gdwarf32))
     Opts.Dwarf64 = DwarfFormatArg->getOption().matches(OPT_gdwarf64);
   Opts.DwarfVersion = getLastArgIntValue(Args, OPT_dwarf_version_EQ, 2, Diags);
@@ -382,6 +382,8 @@ bool AssemblerInvocation::CreateFromArgs(AssemblerInvocation &Opts,
   Opts.EmitCompactUnwindNonCanonical =
       Args.hasArg(OPT_femit_compact_unwind_non_canonical);
   Opts.Crel = Args.hasArg(OPT_crel);
+  Opts.X86RelaxRelocations = !Args.hasArg(OPT_mrelax_relocations_no);
+  Opts.X86Sse2Avx = Args.hasArg(OPT_msse2avx);
 
   Opts.AsSecureLogFile = Args.getLastArgValue(OPT_as_secure_log_file);
 
@@ -440,8 +442,8 @@ static bool ExecuteAssemblerImpl(AssemblerInvocation &Opts,
   MCOptions.EmitCompactUnwindNonCanonical = Opts.EmitCompactUnwindNonCanonical;
   MCOptions.MCSaveTempLabels = Opts.SaveTemporaryLabels;
   MCOptions.Crel = Opts.Crel;
-  MCOptions.X86RelaxRelocations = Opts.RelaxELFRelocations;
-  MCOptions.X86Sse2Avx = Opts.SSE2AVX;
+  MCOptions.X86RelaxRelocations = Opts.X86RelaxRelocations;
+  MCOptions.X86Sse2Avx = Opts.X86Sse2Avx;
   MCOptions.CompressDebugSections = Opts.CompressDebugSections;
   MCOptions.AsSecureLogFile = Opts.AsSecureLogFile;
 

From 157c3fb8c794eaade5fb2a9b08a27c912603b085 Mon Sep 17 00:00:00 2001
From: Petr Hosek <phosek@google.com>
Date: Thu, 15 Aug 2024 15:53:35 -0700
Subject: [PATCH 093/441] [libc] Make sure we have RISC-V f or d extension
 before using it (#104476)

This matches what we do for other architectures.
---
 libc/src/__support/FPUtil/FEnvImpl.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libc/src/__support/FPUtil/FEnvImpl.h b/libc/src/__support/FPUtil/FEnvImpl.h
index 4bb31a6f4c426cd..1c5a1108ff9e0be 100644
--- a/libc/src/__support/FPUtil/FEnvImpl.h
+++ b/libc/src/__support/FPUtil/FEnvImpl.h
@@ -31,7 +31,7 @@
 #include "x86_64/FEnvImpl.h"
 #elif defined(LIBC_TARGET_ARCH_IS_ARM) && defined(__ARM_FP)
 #include "arm/FEnvImpl.h"
-#elif defined(LIBC_TARGET_ARCH_IS_ANY_RISCV)
+#elif defined(LIBC_TARGET_ARCH_IS_ANY_RISCV) && defined(__riscv_flen)
 #include "riscv/FEnvImpl.h"
 #else
 

From 3c0a4f2645288e600e31c6753e9d237b3edb4542 Mon Sep 17 00:00:00 2001
From: Vitaly Buka <vitalybuka@google.com>
Date: Thu, 15 Aug 2024 16:02:23 -0700
Subject: [PATCH 094/441] [asan] Reduce priority of "contiguous_container:"
 VPrintf (#104402)

They are quite noisy in used in test application.
---
 compiler-rt/lib/asan/asan_poisoning.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/compiler-rt/lib/asan/asan_poisoning.cpp b/compiler-rt/lib/asan/asan_poisoning.cpp
index 746ad61813c65dc..d600b1a0c241fa2 100644
--- a/compiler-rt/lib/asan/asan_poisoning.cpp
+++ b/compiler-rt/lib/asan/asan_poisoning.cpp
@@ -410,7 +410,7 @@ void __sanitizer_annotate_contiguous_container(const void *beg_p,
                                                const void *new_mid_p) {
   if (!flags()->detect_container_overflow)
     return;
-  VPrintf(2, "contiguous_container: %p %p %p %p\n", beg_p, end_p, old_mid_p,
+  VPrintf(3, "contiguous_container: %p %p %p %p\n", beg_p, end_p, old_mid_p,
           new_mid_p);
   uptr storage_beg = reinterpret_cast<uptr>(beg_p);
   uptr storage_end = reinterpret_cast<uptr>(end_p);
@@ -479,7 +479,7 @@ void __sanitizer_annotate_double_ended_contiguous_container(
   if (!flags()->detect_container_overflow)
     return;
 
-  VPrintf(2, "contiguous_container: %p %p %p %p %p %p\n", storage_beg_p,
+  VPrintf(3, "contiguous_container: %p %p %p %p %p %p\n", storage_beg_p,
           storage_end_p, old_container_beg_p, old_container_end_p,
           new_container_beg_p, new_container_end_p);
 

From 1f0834ad7192d8a73bbf15bb4ad8aa71f9a92ec6 Mon Sep 17 00:00:00 2001
From: Vitaly Buka <vitalybuka@google.com>
Date: Thu, 15 Aug 2024 16:05:03 -0700
Subject: [PATCH 095/441] [NFC][sanitizer] Use `UNLIKELY` in VReport/VPrintf
 (#104403)

---
 .../lib/sanitizer_common/sanitizer_common.h        | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_common.h b/compiler-rt/lib/sanitizer_common/sanitizer_common.h
index 2428a8cd1479473..182dc8f26c88fd5 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_common.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_common.h
@@ -239,13 +239,15 @@ void RemoveANSIEscapeSequencesFromString(char *buffer);
 void Printf(const char *format, ...) FORMAT(1, 2);
 void Report(const char *format, ...) FORMAT(1, 2);
 void SetPrintfAndReportCallback(void (*callback)(const char *));
-#define VReport(level, ...)                                              \
-  do {                                                                   \
-    if ((uptr)Verbosity() >= (level)) Report(__VA_ARGS__); \
+#define VReport(level, ...)                     \
+  do {                                          \
+    if (UNLIKELY((uptr)Verbosity() >= (level))) \
+      Report(__VA_ARGS__);                      \
   } while (0)
-#define VPrintf(level, ...)                                              \
-  do {                                                                   \
-    if ((uptr)Verbosity() >= (level)) Printf(__VA_ARGS__); \
+#define VPrintf(level, ...)                     \
+  do {                                          \
+    if (UNLIKELY((uptr)Verbosity() >= (level))) \
+      Printf(__VA_ARGS__);                      \
   } while (0)
 
 // Lock sanitizer error reporting and protects against nested errors.

From 6dcfc84e903ca6b0fb652962bdbd054a05befc2b Mon Sep 17 00:00:00 2001
From: Mircea Trofin <mtrofin@google.com>
Date: Thu, 15 Aug 2024 16:21:18 -0700
Subject: [PATCH 096/441] [docs] Stress out the branch naming scheme for
 Graphite. (#104499)

This should make this nuance more discoverable, if the user's first instinct is to search for "Graphite" rather than "stacked reviews"
---
 llvm/docs/GitHub.rst | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/llvm/docs/GitHub.rst b/llvm/docs/GitHub.rst
index 14424f722abf63d..a33211bfef41def 100644
--- a/llvm/docs/GitHub.rst
+++ b/llvm/docs/GitHub.rst
@@ -23,6 +23,23 @@ intended to be able to support "stacked" pull-request. Do not create any branche
 llvm/llvm-project repository otherwise, please use a fork (see below). User branches that
 aren't associated with a pull-request **will be deleted**.
 
+Using Graphite for stacked Pull Requests
+========================================
+
+[Graphite](https://app.graphite.dev/) is a stacked pull request tool supported
+by the LLVM repo (the other being [reviewable.io](https://reviewable.io)).
+
+Graphite will want to create branches under `llvm/llvm-project` rather than your
+private fork, so the guidance above, about branch naming, is critical, otherwise
+`gt submit` (i.e. publish your PRs for review) will fail.
+
+Use `gt config` then `Branch naming settings` and `Set a prefix for branch names`.
+Include the last `/`.
+
+If you didn't do the above and Graphite created non-prefixed branches, a simple way to
+unblock is to rename (`git -m <old name> <new name>`), and then checkout the branch
+and `gt track`.
+
 Pull Requests
 =============
 The LLVM project is using GitHub Pull Requests for Code Reviews. This document

From d156a5a1cb3b3405a6c1e941d112f262c29e15cf Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Thu, 15 Aug 2024 16:28:42 -0700
Subject: [PATCH 097/441] [Driver] Reject -Wa,-mrelax-relocations= for non-x86

Similar to other target-specific -Wa, options.
---
 clang/lib/Driver/ToolChains/Clang.cpp | 19 ++++++++++---------
 clang/test/Driver/ohos.c              |  1 -
 clang/test/Driver/relax.c             |  8 +++++++-
 3 files changed, 17 insertions(+), 11 deletions(-)

diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index 96aa930ea286129..c26ebd1ba56bab6 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -2620,11 +2620,18 @@ static void CollectArgsForIntegratedAssembler(Compilation &C,
           Value == "-mbig-obj")
         continue; // LLVM handles bigobj automatically
 
+      auto Equal = Value.split('=');
       switch (C.getDefaultToolChain().getArch()) {
       default:
         break;
       case llvm::Triple::x86:
       case llvm::Triple::x86_64:
+        if (Equal.first == "-mrelax-relocations" ||
+            Equal.first == "--mrelax-relocations") {
+          UseRelaxRelocations = Equal.second == "yes";
+          if (llvm::is_contained({"yes", "no"}, Equal.second))
+            continue;
+        }
         if (Value == "-msse2avx") {
           CmdArgs.push_back("-msse2avx");
           continue;
@@ -2641,10 +2648,10 @@ static void CollectArgsForIntegratedAssembler(Compilation &C,
       case llvm::Triple::thumbeb:
       case llvm::Triple::arm:
       case llvm::Triple::armeb:
-        if (Value.starts_with("-mimplicit-it=")) {
+        if (Equal.first == "-mimplicit-it") {
           // Only store the value; the last value set takes effect.
-          ImplicitIt = Value.split("=").second;
-          if (CheckARMImplicitITArg(ImplicitIt))
+          ImplicitIt = Equal.second;
+          if (CheckARMImplicitITArg(Equal.second))
             continue;
         }
         if (Value == "-mthumb")
@@ -2719,12 +2726,6 @@ static void CollectArgsForIntegratedAssembler(Compilation &C,
         Crel = false;
       } else if (Value == "--allow-experimental-crel") {
         ExperimentalCrel = true;
-      } else if (Value == "-mrelax-relocations=yes" ||
-                 Value == "--mrelax-relocations=yes") {
-        UseRelaxRelocations = true;
-      } else if (Value == "-mrelax-relocations=no" ||
-                 Value == "--mrelax-relocations=no") {
-        UseRelaxRelocations = false;
       } else if (Value.starts_with("-I")) {
         CmdArgs.push_back(Value.data());
         // We need to consume the next argument if the current arg is a plain
diff --git a/clang/test/Driver/ohos.c b/clang/test/Driver/ohos.c
index 8de4e6de57f7ff8..af8218fb232e601 100644
--- a/clang/test/Driver/ohos.c
+++ b/clang/test/Driver/ohos.c
@@ -7,7 +7,6 @@
 // RUN:     --sysroot=%S/Inputs/ohos_native_tree/sysroot -fuse-ld=ld -march=armv7-a -mcpu=cortex-a7 -mfloat-abi=soft 2>&1 \
 // RUN:     | FileCheck -check-prefixes=CHECK,CHECK-ARM-A7-SOFT %s
 // CHECK: {{.*}}clang{{.*}}" "-cc1"
-// CHECK-NOT: "--mrelax-relocations"
 // CHECK-NOT: "-munwind-tables"
 // CHECK: "-resource-dir" "[[RESOURCE_DIR:[^"]+]]"
 // CHECK: "-isysroot" "[[SYSROOT:[^"]+]]"
diff --git a/clang/test/Driver/relax.c b/clang/test/Driver/relax.c
index 48d22fede12d761..4771b9e3af11163 100644
--- a/clang/test/Driver/relax.c
+++ b/clang/test/Driver/relax.c
@@ -1,4 +1,10 @@
-// RUN: %clang -### -c -integrated-as -Wa,--mrelax-relocations=no %s 2>&1 | FileCheck  %s
+// RUN: %clang -### --target=x86_64 -c -Wa,--mrelax-relocations=no %s 2>&1 | FileCheck  %s
 
 // CHECK: "-cc1"
 // CHECK: "-mrelax-relocations=no"
+
+// RUN: not %clang -### --target=x86_64 -c -Wa,-mrelax-relocations=x %s 2>&1 | FileCheck %s --check-prefix=ERR
+// ERR: error: unsupported argument '-mrelax-relocations=x' to option '-Wa,'
+
+// RUN: not %clang -### --target=aarch64 -c -Wa,-mrelax-relocations=no %s 2>&1 | FileCheck %s --check-prefix=ERR2
+// ERR2: error: unsupported argument '-mrelax-relocations=no' to option '-Wa,'

From 572943e79080962cd70d30fe58a17a8bd0579fd7 Mon Sep 17 00:00:00 2001
From: Jacob Lalonde <jalalonde@fb.com>
Date: Thu, 15 Aug 2024 16:29:59 -0700
Subject: [PATCH 098/441] [LLDB] Reapply #100443 SBSaveCore Thread list
 (#104497)

Reapply #100443 and #101770. These were originally reverted due to a
test failure and an MSAN failure. I changed the test attribute to
restrict to x86 (following the other existing tests). I could not
reproduce the test or the MSAN failure and no repo steps were provided.
---
 lldb/include/lldb/API/SBProcess.h             |   1 +
 lldb/include/lldb/API/SBSaveCoreOptions.h     |  27 +++++
 lldb/include/lldb/API/SBThread.h              |   3 +
 lldb/include/lldb/Core/PluginManager.h        |   2 +-
 lldb/include/lldb/Symbol/SaveCoreOptions.h    |  15 ++-
 lldb/include/lldb/Target/Process.h            |   8 +-
 lldb/include/lldb/lldb-private-interfaces.h   |   2 +-
 lldb/source/API/SBSaveCoreOptions.cpp         |  17 ++-
 lldb/source/API/SBThread.cpp                  |   2 +
 lldb/source/Core/PluginManager.cpp            |   6 +-
 .../ObjectFile/Mach-O/ObjectFileMachO.cpp     |  33 +++---
 .../ObjectFile/Mach-O/ObjectFileMachO.h       |   2 +-
 .../Minidump/MinidumpFileBuilder.cpp          |  72 ++++++-------
 .../ObjectFile/Minidump/MinidumpFileBuilder.h |  10 +-
 .../Minidump/ObjectFileMinidump.cpp           |  12 +--
 .../ObjectFile/Minidump/ObjectFileMinidump.h  |   2 +-
 .../ObjectFile/PECOFF/ObjectFilePECOFF.cpp    |   2 +-
 .../ObjectFile/PECOFF/ObjectFilePECOFF.h      |   2 +-
 .../ObjectFile/PECOFF/WindowsMiniDump.cpp     |   3 +-
 .../ObjectFile/PECOFF/WindowsMiniDump.h       |   3 +-
 lldb/source/Symbol/SaveCoreOptions.cpp        |  80 ++++++++++++++
 lldb/source/Target/Process.cpp                |  30 +++++-
 .../TestProcessSaveCoreMinidump.py            | 101 ++++++++++++++++++
 .../TestSBSaveCoreOptions.py                  |  55 +++++++++-
 .../sbsavecoreoptions/basic_minidump.yaml     |  26 +++++
 .../basic_minidump_different_pid.yaml         |  26 +++++
 26 files changed, 458 insertions(+), 84 deletions(-)
 create mode 100644 lldb/test/API/python_api/sbsavecoreoptions/basic_minidump.yaml
 create mode 100644 lldb/test/API/python_api/sbsavecoreoptions/basic_minidump_different_pid.yaml

diff --git a/lldb/include/lldb/API/SBProcess.h b/lldb/include/lldb/API/SBProcess.h
index 778be7958399010..1624e02070b1b21 100644
--- a/lldb/include/lldb/API/SBProcess.h
+++ b/lldb/include/lldb/API/SBProcess.h
@@ -586,6 +586,7 @@ class LLDB_API SBProcess {
   friend class SBBreakpointCallbackBaton;
   friend class SBBreakpointLocation;
   friend class SBCommandInterpreter;
+  friend class SBSaveCoreOptions;
   friend class SBDebugger;
   friend class SBExecutionContext;
   friend class SBFunction;
diff --git a/lldb/include/lldb/API/SBSaveCoreOptions.h b/lldb/include/lldb/API/SBSaveCoreOptions.h
index 75506fd752e7621..ba48ba5eaea5a0b 100644
--- a/lldb/include/lldb/API/SBSaveCoreOptions.h
+++ b/lldb/include/lldb/API/SBSaveCoreOptions.h
@@ -10,6 +10,10 @@
 #define LLDB_API_SBSAVECOREOPTIONS_H
 
 #include "lldb/API/SBDefines.h"
+#include "lldb/API/SBError.h"
+#include "lldb/API/SBFileSpec.h"
+#include "lldb/API/SBProcess.h"
+#include "lldb/API/SBThread.h"
 
 namespace lldb {
 
@@ -53,6 +57,29 @@ class LLDB_API SBSaveCoreOptions {
   /// \return The output file spec.
   SBFileSpec GetOutputFile() const;
 
+  /// Set the process to save, or unset if supplied with a default constructed
+  /// process.
+  ///
+  /// \param process The process to save.
+  /// \return Success if process was set, otherwise an error
+  /// \note This will clear all process specific options if a different process
+  /// is specified than the current set process, either explicitly from this
+  /// api, or implicitly from any function that requires a process.
+  SBError SetProcess(lldb::SBProcess process);
+
+  /// Add a thread to save in the core file.
+  ///
+  /// \param thread The thread to save.
+  /// \note This will set the process if it is not already set, or return
+  /// and error if the SBThread is not from the set process.
+  SBError AddThread(lldb::SBThread thread);
+
+  /// Remove a thread from the list of threads to save.
+  ///
+  /// \param thread The thread to remove.
+  /// \return True if the thread was removed, false if it was not in the list.
+  bool RemoveThread(lldb::SBThread thread);
+
   /// Reset all options.
   void Clear();
 
diff --git a/lldb/include/lldb/API/SBThread.h b/lldb/include/lldb/API/SBThread.h
index dcf6aa9d5424e85..f8ae627da5acee0 100644
--- a/lldb/include/lldb/API/SBThread.h
+++ b/lldb/include/lldb/API/SBThread.h
@@ -233,6 +233,7 @@ class LLDB_API SBThread {
   friend class SBBreakpoint;
   friend class SBBreakpointLocation;
   friend class SBBreakpointCallbackBaton;
+  friend class SBSaveCoreOptions;
   friend class SBExecutionContext;
   friend class SBFrame;
   friend class SBProcess;
@@ -253,6 +254,8 @@ class LLDB_API SBThread {
   SBError ResumeNewPlan(lldb_private::ExecutionContext &exe_ctx,
                         lldb_private::ThreadPlan *new_plan);
 
+  lldb::ThreadSP GetSP() const;
+
   lldb::ExecutionContextRefSP m_opaque_sp;
 
   lldb_private::Thread *operator->();
diff --git a/lldb/include/lldb/Core/PluginManager.h b/lldb/include/lldb/Core/PluginManager.h
index a23f834f471fb34..e4e0c3eea67f8c0 100644
--- a/lldb/include/lldb/Core/PluginManager.h
+++ b/lldb/include/lldb/Core/PluginManager.h
@@ -194,7 +194,7 @@ class PluginManager {
   GetObjectFileCreateMemoryCallbackForPluginName(llvm::StringRef name);
 
   static Status SaveCore(const lldb::ProcessSP &process_sp,
-                         const lldb_private::SaveCoreOptions &core_options);
+                         lldb_private::SaveCoreOptions &core_options);
 
   // ObjectContainer
   static bool RegisterPlugin(
diff --git a/lldb/include/lldb/Symbol/SaveCoreOptions.h b/lldb/include/lldb/Symbol/SaveCoreOptions.h
index 583bc1f483d043a..f4fed4676fa4ae1 100644
--- a/lldb/include/lldb/Symbol/SaveCoreOptions.h
+++ b/lldb/include/lldb/Symbol/SaveCoreOptions.h
@@ -15,6 +15,7 @@
 
 #include <optional>
 #include <string>
+#include <unordered_set>
 
 namespace lldb_private {
 
@@ -32,13 +33,25 @@ class SaveCoreOptions {
   void SetOutputFile(lldb_private::FileSpec file);
   const std::optional<lldb_private::FileSpec> GetOutputFile() const;
 
+  Status SetProcess(lldb::ProcessSP process_sp);
+
+  Status AddThread(lldb::ThreadSP thread_sp);
+  bool RemoveThread(lldb::ThreadSP thread_sp);
+  bool ShouldThreadBeSaved(lldb::tid_t tid) const;
+
+  Status EnsureValidConfiguration(lldb::ProcessSP process_sp) const;
+
   void Clear();
 
 private:
+  void ClearProcessSpecificData();
+
   std::optional<std::string> m_plugin_name;
   std::optional<lldb_private::FileSpec> m_file;
   std::optional<lldb::SaveCoreStyle> m_style;
+  lldb::ProcessSP m_process_sp;
+  std::unordered_set<lldb::tid_t> m_threads_to_save;
 };
 } // namespace lldb_private
 
-#endif // LLDB_SOURCE_PLUGINS_OBJECTFILE_SaveCoreOPTIONS_H
+#endif // LLDB_SOURCE_PLUGINS_OBJECTFILE_SAVECOREOPTIONS_H
diff --git a/lldb/include/lldb/Target/Process.h b/lldb/include/lldb/Target/Process.h
index cf16fbc812aa48d..9cf6f50558570bc 100644
--- a/lldb/include/lldb/Target/Process.h
+++ b/lldb/include/lldb/Target/Process.h
@@ -738,9 +738,15 @@ class Process : public std::enable_shared_from_this<Process>,
   /// Helper function for Process::SaveCore(...) that calculates the address
   /// ranges that should be saved. This allows all core file plug-ins to save
   /// consistent memory ranges given a \a core_style.
-  Status CalculateCoreFileSaveRanges(lldb::SaveCoreStyle core_style,
+  Status CalculateCoreFileSaveRanges(const SaveCoreOptions &core_options,
                                      CoreFileMemoryRanges &ranges);
 
+  /// Helper function for Process::SaveCore(...) that calculates the thread list
+  /// based upon options set within a given \a core_options object.
+  /// \note If there is no thread list defined, all threads will be saved.
+  std::vector<lldb::ThreadSP>
+  CalculateCoreFileThreadList(const SaveCoreOptions &core_options);
+
 protected:
   virtual JITLoaderList &GetJITLoaders();
 
diff --git a/lldb/include/lldb/lldb-private-interfaces.h b/lldb/include/lldb/lldb-private-interfaces.h
index 87c5ff8d22fb682..b3c8cda899b95ef 100644
--- a/lldb/include/lldb/lldb-private-interfaces.h
+++ b/lldb/include/lldb/lldb-private-interfaces.h
@@ -57,7 +57,7 @@ typedef ObjectFile *(*ObjectFileCreateMemoryInstance)(
     const lldb::ModuleSP &module_sp, lldb::WritableDataBufferSP data_sp,
     const lldb::ProcessSP &process_sp, lldb::addr_t offset);
 typedef bool (*ObjectFileSaveCore)(const lldb::ProcessSP &process_sp,
-                                   const lldb_private::SaveCoreOptions &options,
+                                   lldb_private::SaveCoreOptions &options,
                                    Status &error);
 typedef EmulateInstruction *(*EmulateInstructionCreateInstance)(
     const ArchSpec &arch, InstructionType inst_type);
diff --git a/lldb/source/API/SBSaveCoreOptions.cpp b/lldb/source/API/SBSaveCoreOptions.cpp
index 19ca83f932bcf14..2cd431611ef558c 100644
--- a/lldb/source/API/SBSaveCoreOptions.cpp
+++ b/lldb/source/API/SBSaveCoreOptions.cpp
@@ -7,8 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "lldb/API/SBSaveCoreOptions.h"
-#include "lldb/API/SBError.h"
-#include "lldb/API/SBFileSpec.h"
 #include "lldb/Host/FileSystem.h"
 #include "lldb/Symbol/SaveCoreOptions.h"
 #include "lldb/Utility/Instrumentation.h"
@@ -77,6 +75,21 @@ lldb::SaveCoreStyle SBSaveCoreOptions::GetStyle() const {
   return m_opaque_up->GetStyle();
 }
 
+SBError SBSaveCoreOptions::SetProcess(lldb::SBProcess process) {
+  LLDB_INSTRUMENT_VA(this, process);
+  return m_opaque_up->SetProcess(process.GetSP());
+}
+
+SBError SBSaveCoreOptions::AddThread(lldb::SBThread thread) {
+  LLDB_INSTRUMENT_VA(this, thread);
+  return m_opaque_up->AddThread(thread.GetSP());
+}
+
+bool SBSaveCoreOptions::RemoveThread(lldb::SBThread thread) {
+  LLDB_INSTRUMENT_VA(this, thread);
+  return m_opaque_up->RemoveThread(thread.GetSP());
+}
+
 void SBSaveCoreOptions::Clear() {
   LLDB_INSTRUMENT_VA(this);
   m_opaque_up->Clear();
diff --git a/lldb/source/API/SBThread.cpp b/lldb/source/API/SBThread.cpp
index bda981041064ff7..786f62bd66d520a 100644
--- a/lldb/source/API/SBThread.cpp
+++ b/lldb/source/API/SBThread.cpp
@@ -1337,6 +1337,8 @@ bool SBThread::SafeToCallFunctions() {
   return true;
 }
 
+lldb::ThreadSP SBThread::GetSP() const { return m_opaque_sp->GetThreadSP(); }
+
 lldb_private::Thread *SBThread::operator->() {
   return get();
 }
diff --git a/lldb/source/Core/PluginManager.cpp b/lldb/source/Core/PluginManager.cpp
index 01bee8680b7ba51..f243807df509efd 100644
--- a/lldb/source/Core/PluginManager.cpp
+++ b/lldb/source/Core/PluginManager.cpp
@@ -702,7 +702,7 @@ PluginManager::GetObjectFileCreateMemoryCallbackForPluginName(
 }
 
 Status PluginManager::SaveCore(const lldb::ProcessSP &process_sp,
-                               const lldb_private::SaveCoreOptions &options) {
+                               lldb_private::SaveCoreOptions &options) {
   Status error;
   if (!options.GetOutputFile()) {
     error.SetErrorString("No output file specified");
@@ -714,6 +714,10 @@ Status PluginManager::SaveCore(const lldb::ProcessSP &process_sp,
     return error;
   }
 
+  error = options.EnsureValidConfiguration(process_sp);
+  if (error.Fail())
+    return error;
+
   if (!options.GetPluginName().has_value()) {
     // Try saving core directly from the process plugin first.
     llvm::Expected<bool> ret =
diff --git a/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp b/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp
index ce095bcc48374b2..22ece4f4dacf790 100644
--- a/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp
+++ b/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp
@@ -6347,22 +6347,24 @@ struct segment_vmaddr {
 // are some multiple passes over the image list while calculating
 // everything.
 
-static offset_t CreateAllImageInfosPayload(
-    const lldb::ProcessSP &process_sp, offset_t initial_file_offset,
-    StreamString &all_image_infos_payload, SaveCoreStyle core_style) {
+static offset_t
+CreateAllImageInfosPayload(const lldb::ProcessSP &process_sp,
+                           offset_t initial_file_offset,
+                           StreamString &all_image_infos_payload,
+                           lldb_private::SaveCoreOptions &options) {
   Target &target = process_sp->GetTarget();
   ModuleList modules = target.GetImages();
 
   // stack-only corefiles have no reason to include binaries that
   // are not executing; we're trying to make the smallest corefile
   // we can, so leave the rest out.
-  if (core_style == SaveCoreStyle::eSaveCoreStackOnly)
+  if (options.GetStyle() == SaveCoreStyle::eSaveCoreStackOnly)
     modules.Clear();
 
   std::set<std::string> executing_uuids;
-  ThreadList &thread_list(process_sp->GetThreadList());
-  for (uint32_t i = 0; i < thread_list.GetSize(); i++) {
-    ThreadSP thread_sp = thread_list.GetThreadAtIndex(i);
+  std::vector<ThreadSP> thread_list =
+      process_sp->CalculateCoreFileThreadList(options);
+  for (const ThreadSP &thread_sp : thread_list) {
     uint32_t stack_frame_count = thread_sp->GetStackFrameCount();
     for (uint32_t j = 0; j < stack_frame_count; j++) {
       StackFrameSP stack_frame_sp = thread_sp->GetStackFrameAtIndex(j);
@@ -6520,16 +6522,17 @@ struct page_object {
 };
 
 bool ObjectFileMachO::SaveCore(const lldb::ProcessSP &process_sp,
-                               const lldb_private::SaveCoreOptions &options,
+                               lldb_private::SaveCoreOptions &options,
                                Status &error) {
-  auto core_style = options.GetStyle();
-  if (core_style == SaveCoreStyle::eSaveCoreUnspecified)
-    core_style = SaveCoreStyle::eSaveCoreDirtyOnly;
   // The FileSpec and Process are already checked in PluginManager::SaveCore.
   assert(options.GetOutputFile().has_value());
   assert(process_sp);
   const FileSpec outfile = options.GetOutputFile().value();
 
+  // MachO defaults to dirty pages
+  if (options.GetStyle() == SaveCoreStyle::eSaveCoreUnspecified)
+    options.SetStyle(eSaveCoreDirtyOnly);
+
   Target &target = process_sp->GetTarget();
   const ArchSpec target_arch = target.GetArchitecture();
   const llvm::Triple &target_triple = target_arch.GetTriple();
@@ -6559,7 +6562,7 @@ bool ObjectFileMachO::SaveCore(const lldb::ProcessSP &process_sp,
 
     if (make_core) {
       Process::CoreFileMemoryRanges core_ranges;
-      error = process_sp->CalculateCoreFileSaveRanges(core_style, core_ranges);
+      error = process_sp->CalculateCoreFileSaveRanges(options, core_ranges);
       if (error.Success()) {
         const uint32_t addr_byte_size = target_arch.GetAddressByteSize();
         const ByteOrder byte_order = target_arch.GetByteOrder();
@@ -6730,8 +6733,8 @@ bool ObjectFileMachO::SaveCore(const lldb::ProcessSP &process_sp,
             std::make_shared<StructuredData::Dictionary>());
         StructuredData::ArraySP threads(
             std::make_shared<StructuredData::Array>());
-        for (uint32_t thread_idx = 0; thread_idx < num_threads; ++thread_idx) {
-          ThreadSP thread_sp(thread_list.GetThreadAtIndex(thread_idx));
+        for (const ThreadSP &thread_sp :
+             process_sp->CalculateCoreFileThreadList(options)) {
           StructuredData::DictionarySP thread(
               std::make_shared<StructuredData::Dictionary>());
           thread->AddIntegerItem("thread_id", thread_sp->GetID());
@@ -6754,7 +6757,7 @@ bool ObjectFileMachO::SaveCore(const lldb::ProcessSP &process_sp,
         all_image_infos_lcnote_up->payload_file_offset = file_offset;
         file_offset = CreateAllImageInfosPayload(
             process_sp, file_offset, all_image_infos_lcnote_up->payload,
-            core_style);
+            options);
         lc_notes.push_back(std::move(all_image_infos_lcnote_up));
 
         // Add LC_NOTE load commands
diff --git a/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.h b/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.h
index e7af90e28bc4b7e..27bc237aaac48d8 100644
--- a/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.h
+++ b/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.h
@@ -62,7 +62,7 @@ class ObjectFileMachO : public lldb_private::ObjectFile {
                                         lldb_private::ModuleSpecList &specs);
 
   static bool SaveCore(const lldb::ProcessSP &process_sp,
-                       const lldb_private::SaveCoreOptions &options,
+                       lldb_private::SaveCoreOptions &options,
                        lldb_private::Status &error);
 
   static bool MagicBytesMatch(lldb::DataBufferSP data_sp, lldb::addr_t offset,
diff --git a/lldb/source/Plugins/ObjectFile/Minidump/MinidumpFileBuilder.cpp b/lldb/source/Plugins/ObjectFile/Minidump/MinidumpFileBuilder.cpp
index de212c6b20da7eb..c0cc3af638a7772 100644
--- a/lldb/source/Plugins/ObjectFile/Minidump/MinidumpFileBuilder.cpp
+++ b/lldb/source/Plugins/ObjectFile/Minidump/MinidumpFileBuilder.cpp
@@ -69,10 +69,9 @@ Status MinidumpFileBuilder::AddHeaderAndCalculateDirectories() {
     m_expected_directories += 9;
 
   // Go through all of the threads and check for exceptions.
-  lldb_private::ThreadList thread_list = m_process_sp->GetThreadList();
-  const uint32_t num_threads = thread_list.GetSize();
-  for (uint32_t thread_idx = 0; thread_idx < num_threads; ++thread_idx) {
-    ThreadSP thread_sp(thread_list.GetThreadAtIndex(thread_idx));
+  std::vector<lldb::ThreadSP> threads =
+      m_process_sp->CalculateCoreFileThreadList(m_save_core_options);
+  for (const ThreadSP &thread_sp : threads) {
     StopInfoSP stop_info_sp = thread_sp->GetStopInfo();
     if (stop_info_sp) {
       const StopReason &stop_reason = stop_info_sp->GetStopReason();
@@ -588,12 +587,13 @@ Status MinidumpFileBuilder::FixThreadStacks() {
 
 Status MinidumpFileBuilder::AddThreadList() {
   constexpr size_t minidump_thread_size = sizeof(llvm::minidump::Thread);
-  lldb_private::ThreadList thread_list = m_process_sp->GetThreadList();
+  std::vector<ThreadSP> thread_list =
+      m_process_sp->CalculateCoreFileThreadList(m_save_core_options);
 
   // size of the entire thread stream consists of:
   // number of threads and threads array
   size_t thread_stream_size = sizeof(llvm::support::ulittle32_t) +
-                              thread_list.GetSize() * minidump_thread_size;
+                              thread_list.size() * minidump_thread_size;
   // save for the ability to set up RVA
   size_t size_before = GetCurrentDataEndOffset();
   Status error;
@@ -602,17 +602,15 @@ Status MinidumpFileBuilder::AddThreadList() {
     return error;
 
   llvm::support::ulittle32_t thread_count =
-      static_cast<llvm::support::ulittle32_t>(thread_list.GetSize());
+      static_cast<llvm::support::ulittle32_t>(thread_list.size());
   m_data.AppendData(&thread_count, sizeof(llvm::support::ulittle32_t));
 
   // Take the offset after the thread count.
   m_thread_list_start = GetCurrentDataEndOffset();
   DataBufferHeap helper_data;
 
-  const uint32_t num_threads = thread_list.GetSize();
   Log *log = GetLog(LLDBLog::Object);
-  for (uint32_t thread_idx = 0; thread_idx < num_threads; ++thread_idx) {
-    ThreadSP thread_sp(thread_list.GetThreadAtIndex(thread_idx));
+  for (const ThreadSP &thread_sp : thread_list) {
     RegisterContextSP reg_ctx_sp(thread_sp->GetRegisterContext());
 
     if (!reg_ctx_sp) {
@@ -650,7 +648,7 @@ Status MinidumpFileBuilder::AddThreadList() {
     m_tid_to_reg_ctx[thread_sp->GetID()] = thread_context_memory_locator;
 
     LLDB_LOGF(log, "AddThreadList for thread %d: thread_context %zu bytes",
-              thread_idx, thread_context.size());
+              thread_sp->GetIndexID(), thread_context.size());
     helper_data.AppendData(thread_context.data(), thread_context.size());
 
     llvm::minidump::Thread t;
@@ -674,11 +672,10 @@ Status MinidumpFileBuilder::AddThreadList() {
 }
 
 Status MinidumpFileBuilder::AddExceptions() {
-  lldb_private::ThreadList thread_list = m_process_sp->GetThreadList();
+  std::vector<ThreadSP> thread_list =
+      m_process_sp->CalculateCoreFileThreadList(m_save_core_options);
   Status error;
-  const uint32_t num_threads = thread_list.GetSize();
-  for (uint32_t thread_idx = 0; thread_idx < num_threads; ++thread_idx) {
-    ThreadSP thread_sp(thread_list.GetThreadAtIndex(thread_idx));
+  for (const ThreadSP &thread_sp : thread_list) {
     StopInfoSP stop_info_sp = thread_sp->GetStopInfo();
     bool add_exception = false;
     if (stop_info_sp) {
@@ -819,7 +816,7 @@ Status MinidumpFileBuilder::AddLinuxFileStreams() {
   return error;
 }
 
-Status MinidumpFileBuilder::AddMemoryList(SaveCoreStyle core_style) {
+Status MinidumpFileBuilder::AddMemoryList() {
   Status error;
 
   // We first save the thread stacks to ensure they fit in the first UINT32_MAX
@@ -828,18 +825,26 @@ Status MinidumpFileBuilder::AddMemoryList(SaveCoreStyle core_style) {
   // in accessible with a 32 bit offset.
   Process::CoreFileMemoryRanges ranges_32;
   Process::CoreFileMemoryRanges ranges_64;
-  error = m_process_sp->CalculateCoreFileSaveRanges(
-      SaveCoreStyle::eSaveCoreStackOnly, ranges_32);
+  Process::CoreFileMemoryRanges all_core_memory_ranges;
+  error = m_process_sp->CalculateCoreFileSaveRanges(m_save_core_options,
+                                                    all_core_memory_ranges);
   if (error.Fail())
     return error;
 
-  // Calculate totalsize including the current offset.
+  // Start by saving all of the stacks and ensuring they fit under the 32b
+  // limit.
   uint64_t total_size = GetCurrentDataEndOffset();
-  total_size += ranges_32.size() * sizeof(llvm::minidump::MemoryDescriptor);
-  std::unordered_set<addr_t> stack_start_addresses;
-  for (const auto &core_range : ranges_32) {
-    stack_start_addresses.insert(core_range.range.start());
-    total_size += core_range.range.size();
+  auto iterator = all_core_memory_ranges.begin();
+  while (iterator != all_core_memory_ranges.end()) {
+    if (m_saved_stack_ranges.count(iterator->range.start()) > 0) {
+      // We don't save stacks twice.
+      ranges_32.push_back(*iterator);
+      total_size +=
+          iterator->range.size() + sizeof(llvm::minidump::MemoryDescriptor);
+      iterator = all_core_memory_ranges.erase(iterator);
+    } else {
+      iterator++;
+    }
   }
 
   if (total_size >= UINT32_MAX) {
@@ -849,14 +854,6 @@ Status MinidumpFileBuilder::AddMemoryList(SaveCoreStyle core_style) {
     return error;
   }
 
-  Process::CoreFileMemoryRanges all_core_memory_ranges;
-  if (core_style != SaveCoreStyle::eSaveCoreStackOnly) {
-    error = m_process_sp->CalculateCoreFileSaveRanges(core_style,
-                                                      all_core_memory_ranges);
-    if (error.Fail())
-      return error;
-  }
-
   // After saving the stacks, we start packing as much as we can into 32b.
   // We apply a generous padding here so that the Directory, MemoryList and
   // Memory64List sections all begin in 32b addressable space.
@@ -864,16 +861,13 @@ Status MinidumpFileBuilder::AddMemoryList(SaveCoreStyle core_style) {
   // All core memeroy ranges will either container nothing on stacks only
   // or all the memory ranges including stacks
   if (!all_core_memory_ranges.empty())
-    total_size +=
-        256 + (all_core_memory_ranges.size() - stack_start_addresses.size()) *
-                  sizeof(llvm::minidump::MemoryDescriptor_64);
+    total_size += 256 + (all_core_memory_ranges.size() *
+                         sizeof(llvm::minidump::MemoryDescriptor_64));
 
   for (const auto &core_range : all_core_memory_ranges) {
     const addr_t range_size = core_range.range.size();
-    if (stack_start_addresses.count(core_range.range.start()) > 0)
-      // Don't double save stacks.
-      continue;
-
+    // We don't need to check for stacks here because we already removed them
+    // from all_core_memory_ranges.
     if (total_size + range_size < UINT32_MAX) {
       ranges_32.push_back(core_range);
       total_size += range_size;
diff --git a/lldb/source/Plugins/ObjectFile/Minidump/MinidumpFileBuilder.h b/lldb/source/Plugins/ObjectFile/Minidump/MinidumpFileBuilder.h
index 20564e0661f2a29..762de83db5a39ce 100644
--- a/lldb/source/Plugins/ObjectFile/Minidump/MinidumpFileBuilder.h
+++ b/lldb/source/Plugins/ObjectFile/Minidump/MinidumpFileBuilder.h
@@ -75,8 +75,10 @@ lldb_private::Status WriteString(const std::string &to_write,
 class MinidumpFileBuilder {
 public:
   MinidumpFileBuilder(lldb::FileUP &&core_file,
-                      const lldb::ProcessSP &process_sp)
-      : m_process_sp(process_sp), m_core_file(std::move(core_file)){};
+                      const lldb::ProcessSP &process_sp,
+                      lldb_private::SaveCoreOptions &save_core_options)
+      : m_process_sp(process_sp), m_core_file(std::move(core_file)),
+        m_save_core_options(save_core_options){}
 
   MinidumpFileBuilder(const MinidumpFileBuilder &) = delete;
   MinidumpFileBuilder &operator=(const MinidumpFileBuilder &) = delete;
@@ -103,7 +105,7 @@ class MinidumpFileBuilder {
   // Add Exception streams for any threads that stopped with exceptions.
   lldb_private::Status AddExceptions();
   // Add MemoryList stream, containing dumps of important memory segments
-  lldb_private::Status AddMemoryList(lldb::SaveCoreStyle core_style);
+  lldb_private::Status AddMemoryList();
   // Add MiscInfo stream, mainly providing ProcessId
   lldb_private::Status AddMiscInfo();
   // Add informative files about a Linux process
@@ -163,7 +165,9 @@ class MinidumpFileBuilder {
   // to duplicate it in the exception data.
   std::unordered_map<lldb::tid_t, llvm::minidump::LocationDescriptor>
       m_tid_to_reg_ctx;
+  std::unordered_set<lldb::addr_t> m_saved_stack_ranges;
   lldb::FileUP m_core_file;
+  lldb_private::SaveCoreOptions m_save_core_options;
 };
 
 #endif // LLDB_SOURCE_PLUGINS_OBJECTFILE_MINIDUMP_MINIDUMPFILEBUILDER_H
diff --git a/lldb/source/Plugins/ObjectFile/Minidump/ObjectFileMinidump.cpp b/lldb/source/Plugins/ObjectFile/Minidump/ObjectFileMinidump.cpp
index faa144bfb5f6a5b..0897895e6bc25da 100644
--- a/lldb/source/Plugins/ObjectFile/Minidump/ObjectFileMinidump.cpp
+++ b/lldb/source/Plugins/ObjectFile/Minidump/ObjectFileMinidump.cpp
@@ -56,16 +56,15 @@ size_t ObjectFileMinidump::GetModuleSpecifications(
 }
 
 bool ObjectFileMinidump::SaveCore(const lldb::ProcessSP &process_sp,
-                                  const lldb_private::SaveCoreOptions &options,
+                                  lldb_private::SaveCoreOptions &options,
                                   lldb_private::Status &error) {
   // Output file and process_sp are both checked in PluginManager::SaveCore.
   assert(options.GetOutputFile().has_value());
   assert(process_sp);
 
   // Minidump defaults to stacks only.
-  SaveCoreStyle core_style = options.GetStyle();
-  if (core_style == SaveCoreStyle::eSaveCoreUnspecified)
-    core_style = SaveCoreStyle::eSaveCoreStackOnly;
+  if (options.GetStyle() == SaveCoreStyle::eSaveCoreUnspecified)
+    options.SetStyle(SaveCoreStyle::eSaveCoreStackOnly);
 
   llvm::Expected<lldb::FileUP> maybe_core_file = FileSystem::Instance().Open(
       options.GetOutputFile().value(),
@@ -74,7 +73,8 @@ bool ObjectFileMinidump::SaveCore(const lldb::ProcessSP &process_sp,
     error = maybe_core_file.takeError();
     return false;
   }
-  MinidumpFileBuilder builder(std::move(maybe_core_file.get()), process_sp);
+  MinidumpFileBuilder builder(std::move(maybe_core_file.get()), process_sp,
+                              options);
 
   Log *log = GetLog(LLDBLog::Object);
   error = builder.AddHeaderAndCalculateDirectories();
@@ -121,7 +121,7 @@ bool ObjectFileMinidump::SaveCore(const lldb::ProcessSP &process_sp,
 
   // Note: add memory HAS to be the last thing we do. It can overflow into 64b
   // land and many RVA's only support 32b
-  error = builder.AddMemoryList(core_style);
+  error = builder.AddMemoryList();
   if (error.Fail()) {
     LLDB_LOGF(log, "AddMemoryList failed: %s", error.AsCString());
     return false;
diff --git a/lldb/source/Plugins/ObjectFile/Minidump/ObjectFileMinidump.h b/lldb/source/Plugins/ObjectFile/Minidump/ObjectFileMinidump.h
index 0cd31a0e482d5f8..b76fcd0052a8a8c 100644
--- a/lldb/source/Plugins/ObjectFile/Minidump/ObjectFileMinidump.h
+++ b/lldb/source/Plugins/ObjectFile/Minidump/ObjectFileMinidump.h
@@ -55,7 +55,7 @@ class ObjectFileMinidump : public lldb_private::PluginInterface {
 
   // Saves dump in Minidump file format
   static bool SaveCore(const lldb::ProcessSP &process_sp,
-                       const lldb_private::SaveCoreOptions &options,
+                       lldb_private::SaveCoreOptions &options,
                        lldb_private::Status &error);
 
 private:
diff --git a/lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.cpp b/lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.cpp
index bda691ade8af025..9d01089745dfc91 100644
--- a/lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.cpp
+++ b/lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.cpp
@@ -355,7 +355,7 @@ size_t ObjectFilePECOFF::GetModuleSpecifications(
 }
 
 bool ObjectFilePECOFF::SaveCore(const lldb::ProcessSP &process_sp,
-                                const lldb_private::SaveCoreOptions &options,
+                                lldb_private::SaveCoreOptions &options,
                                 lldb_private::Status &error) {
   // Outfile and process_sp are validated by PluginManager::SaveCore
   assert(options.GetOutputFile().has_value());
diff --git a/lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.h b/lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.h
index 2eb2b3b7745380d..8bccf3be3e5f636 100644
--- a/lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.h
+++ b/lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.h
@@ -82,7 +82,7 @@ class ObjectFilePECOFF : public lldb_private::ObjectFile {
                                         lldb_private::ModuleSpecList &specs);
 
   static bool SaveCore(const lldb::ProcessSP &process_sp,
-                       const lldb_private::SaveCoreOptions &options,
+                       lldb_private::SaveCoreOptions &options,
                        lldb_private::Status &error);
 
   static bool MagicBytesMatch(lldb::DataBufferSP data_sp);
diff --git a/lldb/source/Plugins/ObjectFile/PECOFF/WindowsMiniDump.cpp b/lldb/source/Plugins/ObjectFile/PECOFF/WindowsMiniDump.cpp
index 6b8488990a97200..24ab881d92d85a7 100644
--- a/lldb/source/Plugins/ObjectFile/PECOFF/WindowsMiniDump.cpp
+++ b/lldb/source/Plugins/ObjectFile/PECOFF/WindowsMiniDump.cpp
@@ -21,8 +21,7 @@
 namespace lldb_private {
 
 bool SaveMiniDump(const lldb::ProcessSP &process_sp,
-                  const SaveCoreOptions &core_options,
-                  lldb_private::Status &error) {
+                  SaveCoreOptions &core_options, lldb_private::Status &error) {
   if (!process_sp)
     return false;
 #ifdef _WIN32
diff --git a/lldb/source/Plugins/ObjectFile/PECOFF/WindowsMiniDump.h b/lldb/source/Plugins/ObjectFile/PECOFF/WindowsMiniDump.h
index 03c0ece30614318..2f8606d82c974b0 100644
--- a/lldb/source/Plugins/ObjectFile/PECOFF/WindowsMiniDump.h
+++ b/lldb/source/Plugins/ObjectFile/PECOFF/WindowsMiniDump.h
@@ -14,8 +14,7 @@
 namespace lldb_private {
 
 bool SaveMiniDump(const lldb::ProcessSP &process_sp,
-                  const SaveCoreOptions &core_options,
-                  lldb_private::Status &error);
+                  SaveCoreOptions &core_options, lldb_private::Status &error);
 
 } // namespace lldb_private
 
diff --git a/lldb/source/Symbol/SaveCoreOptions.cpp b/lldb/source/Symbol/SaveCoreOptions.cpp
index 0f6fdac1ce22e6a..3c4ca2d852b76aa 100644
--- a/lldb/source/Symbol/SaveCoreOptions.cpp
+++ b/lldb/source/Symbol/SaveCoreOptions.cpp
@@ -8,6 +8,8 @@
 
 #include "lldb/Symbol/SaveCoreOptions.h"
 #include "lldb/Core/PluginManager.h"
+#include "lldb/Target/Process.h"
+#include "lldb/Target/Thread.h"
 
 using namespace lldb;
 using namespace lldb_private;
@@ -46,8 +48,86 @@ SaveCoreOptions::GetOutputFile() const {
   return m_file;
 }
 
+Status SaveCoreOptions::SetProcess(lldb::ProcessSP process_sp) {
+  Status error;
+  if (!process_sp) {
+    ClearProcessSpecificData();
+    m_process_sp.reset();
+    return error;
+  }
+
+  if (!process_sp->IsValid()) {
+    error.SetErrorString("Cannot assign an invalid process.");
+    return error;
+  }
+
+  // Don't clear any process specific data if the process is the same.
+  if (m_process_sp == process_sp)
+    return error;
+
+  ClearProcessSpecificData();
+  m_process_sp = process_sp;
+  return error;
+}
+
+Status SaveCoreOptions::AddThread(lldb::ThreadSP thread_sp) {
+  Status error;
+  if (!thread_sp) {
+    error.SetErrorString("invalid thread");
+    return error;
+  }
+
+  if (m_process_sp) {
+    if (m_process_sp != thread_sp->GetProcess()) {
+      error.SetErrorString("Cannot add a thread from a different process.");
+      return error;
+    }
+  } else {
+    m_process_sp = thread_sp->GetProcess();
+  }
+
+  m_threads_to_save.insert(thread_sp->GetID());
+  return error;
+}
+
+bool SaveCoreOptions::RemoveThread(lldb::ThreadSP thread_sp) {
+  return thread_sp && m_threads_to_save.erase(thread_sp->GetID()) > 0;
+}
+
+bool SaveCoreOptions::ShouldThreadBeSaved(lldb::tid_t tid) const {
+  // If the user specified no threads to save, then we save all threads.
+  if (m_threads_to_save.empty())
+    return true;
+  return m_threads_to_save.count(tid) > 0;
+}
+
+Status SaveCoreOptions::EnsureValidConfiguration(
+    lldb::ProcessSP process_sp) const {
+  Status error;
+  std::string error_str;
+  if (!m_threads_to_save.empty() && GetStyle() == lldb::eSaveCoreFull)
+    error_str += "Cannot save a full core with a subset of threads\n";
+
+  if (m_process_sp && m_process_sp != process_sp)
+    error_str += "Cannot save core for process using supplied core options. "
+                 "Options were constructed targeting a different process. \n";
+
+  if (!error_str.empty())
+    error.SetErrorString(error_str);
+
+  return error;
+}
+
+void SaveCoreOptions::ClearProcessSpecificData() { 
+  // Deliberately not following the formatter style here to indicate that
+  // this method will be expanded in the future.
+  m_threads_to_save.clear(); 
+}
+
 void SaveCoreOptions::Clear() {
   m_file = std::nullopt;
   m_plugin_name = std::nullopt;
   m_style = std::nullopt;
+  m_threads_to_save.clear();
+  m_process_sp.reset();
 }
diff --git a/lldb/source/Target/Process.cpp b/lldb/source/Target/Process.cpp
index e3c4f2ee398cc46..3c9247fdbbbc968 100644
--- a/lldb/source/Target/Process.cpp
+++ b/lldb/source/Target/Process.cpp
@@ -6548,8 +6548,9 @@ static void AddRegion(const MemoryRegionInfo &region, bool try_dirty_pages,
 }
 
 static void SaveOffRegionsWithStackPointers(
-    Process &process, const MemoryRegionInfos &regions,
-    Process::CoreFileMemoryRanges &ranges, std::set<addr_t> &stack_ends) {
+    Process &process, const SaveCoreOptions &core_options,
+    const MemoryRegionInfos &regions, Process::CoreFileMemoryRanges &ranges,
+    std::set<addr_t> &stack_ends) {
   const bool try_dirty_pages = true;
 
   // Before we take any dump, we want to save off the used portions of the
@@ -6571,10 +6572,16 @@ static void SaveOffRegionsWithStackPointers(
     if (process.GetMemoryRegionInfo(sp, sp_region).Success()) {
       const size_t stack_head = (sp - red_zone);
       const size_t stack_size = sp_region.GetRange().GetRangeEnd() - stack_head;
+      // Even if the SaveCoreOption doesn't want us to save the stack
+      // we still need to populate the stack_ends set so it doesn't get saved
+      // off in other calls
       sp_region.GetRange().SetRangeBase(stack_head);
       sp_region.GetRange().SetByteSize(stack_size);
       stack_ends.insert(sp_region.GetRange().GetRangeEnd());
-      AddRegion(sp_region, try_dirty_pages, ranges);
+      // This will return true if the threadlist the user specified is empty,
+      // or contains the thread id from thread_sp.
+      if (core_options.ShouldThreadBeSaved(thread_sp->GetID()))
+        AddRegion(sp_region, try_dirty_pages, ranges);
     }
   }
 }
@@ -6643,10 +6650,11 @@ static void GetCoreFileSaveRangesStackOnly(
   }
 }
 
-Status Process::CalculateCoreFileSaveRanges(lldb::SaveCoreStyle core_style,
+Status Process::CalculateCoreFileSaveRanges(const SaveCoreOptions &options,
                                             CoreFileMemoryRanges &ranges) {
   lldb_private::MemoryRegionInfos regions;
   Status err = GetMemoryRegions(regions);
+  SaveCoreStyle core_style = options.GetStyle();
   if (err.Fail())
     return err;
   if (regions.empty())
@@ -6656,7 +6664,7 @@ Status Process::CalculateCoreFileSaveRanges(lldb::SaveCoreStyle core_style,
                   "eSaveCoreUnspecified");
 
   std::set<addr_t> stack_ends;
-  SaveOffRegionsWithStackPointers(*this, regions, ranges, stack_ends);
+  SaveOffRegionsWithStackPointers(*this, options, regions, ranges, stack_ends);
 
   switch (core_style) {
   case eSaveCoreUnspecified:
@@ -6684,6 +6692,18 @@ Status Process::CalculateCoreFileSaveRanges(lldb::SaveCoreStyle core_style,
   return Status(); // Success!
 }
 
+std::vector<ThreadSP>
+Process::CalculateCoreFileThreadList(const SaveCoreOptions &core_options) {
+  std::vector<ThreadSP> thread_list;
+  for (const lldb::ThreadSP &thread_sp : m_thread_list.Threads()) {
+    if (core_options.ShouldThreadBeSaved(thread_sp->GetID())) {
+      thread_list.push_back(thread_sp);
+    }
+  }
+
+  return thread_list;
+}
+
 void Process::SetAddressableBitMasks(AddressableBits bit_masks) {
   uint32_t low_memory_addr_bits = bit_masks.GetLowmemAddressableBits();
   uint32_t high_memory_addr_bits = bit_masks.GetHighmemAddressableBits();
diff --git a/lldb/test/API/functionalities/process_save_core_minidump/TestProcessSaveCoreMinidump.py b/lldb/test/API/functionalities/process_save_core_minidump/TestProcessSaveCoreMinidump.py
index 96511d790271fe2..5abaa05a90f63e6 100644
--- a/lldb/test/API/functionalities/process_save_core_minidump/TestProcessSaveCoreMinidump.py
+++ b/lldb/test/API/functionalities/process_save_core_minidump/TestProcessSaveCoreMinidump.py
@@ -199,3 +199,104 @@ def test_save_linux_mini_dump(self):
                 os.unlink(core_sb_dirty)
             if os.path.isfile(core_sb_full):
                 os.unlink(core_sb_full)
+
+    @skipUnlessArch("x86_64")
+    @skipUnlessPlatform(["linux"])
+    def test_save_linux_mini_dump_thread_options(self):
+        """Test that we can save a Linux mini dump
+        with a subset of threads"""
+
+        self.build()
+        exe = self.getBuildArtifact("a.out")
+        thread_subset_dmp = self.getBuildArtifact("core.thread.subset.dmp")
+        try:
+            target = self.dbg.CreateTarget(exe)
+            process = target.LaunchSimple(
+                None, None, self.get_process_working_directory()
+            )
+            self.assertState(process.GetState(), lldb.eStateStopped)
+
+            thread_to_include = process.GetThreadAtIndex(0)
+            options = lldb.SBSaveCoreOptions()
+            thread_subset_spec = lldb.SBFileSpec(thread_subset_dmp)
+            options.AddThread(thread_to_include)
+            options.SetOutputFile(thread_subset_spec)
+            options.SetPluginName("minidump")
+            options.SetStyle(lldb.eSaveCoreStackOnly)
+            error = process.SaveCore(options)
+            self.assertTrue(error.Success())
+
+            core_target = self.dbg.CreateTarget(None)
+            core_process = core_target.LoadCore(thread_subset_dmp)
+
+            self.assertTrue(core_process, PROCESS_IS_VALID)
+            self.assertEqual(core_process.GetNumThreads(), 1)
+            saved_thread = core_process.GetThreadAtIndex(0)
+            expected_thread = process.GetThreadAtIndex(0)
+            self.assertEqual(expected_thread.GetThreadID(), saved_thread.GetThreadID())
+            expected_sp = expected_thread.GetFrameAtIndex(0).GetSP()
+            saved_sp = saved_thread.GetFrameAtIndex(0).GetSP()
+            self.assertEqual(expected_sp, saved_sp)
+            expected_region = lldb.SBMemoryRegionInfo()
+            saved_region = lldb.SBMemoryRegionInfo()
+            error = core_process.GetMemoryRegionInfo(saved_sp, saved_region)
+            self.assertTrue(error.Success(), error.GetCString())
+            error = process.GetMemoryRegionInfo(expected_sp, expected_region)
+            self.assertTrue(error.Success(), error.GetCString())
+            self.assertEqual(
+                expected_region.GetRegionBase(), saved_region.GetRegionBase()
+            )
+            self.assertEqual(
+                expected_region.GetRegionEnd(), saved_region.GetRegionEnd()
+            )
+
+        finally:
+            self.assertTrue(self.dbg.DeleteTarget(target))
+            if os.path.isfile(thread_subset_dmp):
+                os.unlink(thread_subset_dmp)
+
+    @skipUnlessArch("x86_64")
+    @skipUnlessPlatform(["linux"])
+    def test_save_linux_mini_dump_default_options(self):
+        """Test that we can save a Linux mini dump with default SBSaveCoreOptions"""
+
+        self.build()
+        exe = self.getBuildArtifact("a.out")
+        default_value_file = self.getBuildArtifact("core.defaults.dmp")
+        try:
+            target = self.dbg.CreateTarget(exe)
+            process = target.LaunchSimple(
+                None, None, self.get_process_working_directory()
+            )
+            self.assertState(process.GetState(), lldb.eStateStopped)
+
+            process_info = process.GetProcessInfo()
+            expected_pid = process_info.GetProcessID() if process_info.IsValid() else -1
+            expected_modules = target.modules
+            expected_threads = []
+            stacks_to_sp_map = {}
+            expected_pid = process.GetProcessInfo().GetProcessID()
+
+            for thread_idx in range(process.GetNumThreads()):
+                thread = process.GetThreadAtIndex(thread_idx)
+                thread_id = thread.GetThreadID()
+                expected_threads.append(thread_id)
+                stacks_to_sp_map[thread_id] = thread.GetFrameAtIndex(0).GetSP()
+
+
+            # This is almost identical to the single thread test case because
+            # minidump defaults to stacks only, so we want to see if the
+            # default options work as expected.
+            options = lldb.SBSaveCoreOptions()
+            default_value_spec = lldb.SBFileSpec(default_value_file)
+            options.SetOutputFile(default_value_spec)
+            options.SetPluginName("minidump")
+            error = process.SaveCore(options)
+            self.assertTrue(error.Success())
+
+            self.verify_core_file(default_value_file, expected_pid, expected_modules, expected_threads, stacks_to_sp_map)
+
+        finally:
+            self.assertTrue(self.dbg.DeleteTarget(target))
+            if os.path.isfile(default_value_file):
+                os.unlink(default_value_file)
diff --git a/lldb/test/API/python_api/sbsavecoreoptions/TestSBSaveCoreOptions.py b/lldb/test/API/python_api/sbsavecoreoptions/TestSBSaveCoreOptions.py
index c509e81d8951a8f..40d0cc7e96ff489 100644
--- a/lldb/test/API/python_api/sbsavecoreoptions/TestSBSaveCoreOptions.py
+++ b/lldb/test/API/python_api/sbsavecoreoptions/TestSBSaveCoreOptions.py
@@ -4,8 +4,26 @@
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
 
-
 class SBSaveCoreOptionsAPICase(TestBase):
+    basic_minidump = "basic_minidump.yaml"
+    basic_minidump_different_pid = "basic_minidump_different_pid.yaml"
+
+    def get_process_from_yaml(self, yaml_file):
+        minidump_path = self.getBuildArtifact(os.path.basename(yaml_file) + ".dmp")
+        print ("minidump_path: " + minidump_path)
+        self.yaml2obj(yaml_file, minidump_path)
+        self.assertTrue(os.path.exists(minidump_path), "yaml2obj did not emit a minidump file")
+        target = self.dbg.CreateTarget(None)
+        process = target.LoadCore(minidump_path)
+        self.assertTrue(process.IsValid(), "Process is not valid")
+        return process
+
+    def get_basic_process(self):
+        return self.get_process_from_yaml(self.basic_minidump)
+
+    def get_basic_process_different_pid(self):
+        return self.get_process_from_yaml(self.basic_minidump_different_pid)
+
     def test_plugin_name_assignment(self):
         """Test assignment ensuring valid plugin names only."""
         options = lldb.SBSaveCoreOptions()
@@ -26,3 +44,38 @@ def test_default_corestyle_behavior(self):
         """Test that the default core style is unspecified."""
         options = lldb.SBSaveCoreOptions()
         self.assertEqual(options.GetStyle(), lldb.eSaveCoreUnspecified)
+
+    def test_adding_and_removing_thread(self):
+        """Test adding and removing a thread from save core options."""
+        self.assertTrue(self.dbg)
+        options = lldb.SBSaveCoreOptions()
+        process = self.get_basic_process()
+        self.assertTrue(process.IsValid(), "Process is not valid")
+        thread = process.GetThreadAtIndex(0)
+        error = options.AddThread(thread)
+        self.assertTrue(error.Success(), error.GetCString())
+        removed_success = options.RemoveThread(thread)
+        self.assertTrue(removed_success)
+        removed_success = options.RemoveThread(thread)
+        self.assertFalse(removed_success)
+
+
+    def test_adding_thread_different_process(self):
+        """Test adding and removing a thread from save core options."""
+        options = lldb.SBSaveCoreOptions()
+        process = self.get_basic_process()
+        process_2 = self.get_basic_process_different_pid()
+        thread = process.GetThreadAtIndex(0)
+        error = options.AddThread(thread)
+        self.assertTrue(error.Success())
+        thread_2 = process_2.GetThreadAtIndex(0)
+        error = options.AddThread(thread_2)
+        self.assertTrue(error.Fail())
+        options.Clear()
+        error = options.AddThread(thread_2)
+        self.assertTrue(error.Success())
+        options.SetProcess(process)
+        error = options.AddThread(thread_2)
+        self.assertTrue(error.Fail())
+        error = options.AddThread(thread)
+        self.assertTrue(error.Success())
diff --git a/lldb/test/API/python_api/sbsavecoreoptions/basic_minidump.yaml b/lldb/test/API/python_api/sbsavecoreoptions/basic_minidump.yaml
new file mode 100644
index 000000000000000..993c7da21225a98
--- /dev/null
+++ b/lldb/test/API/python_api/sbsavecoreoptions/basic_minidump.yaml
@@ -0,0 +1,26 @@
+--- !minidump
+Streams:
+  - Type:            SystemInfo
+    Processor Arch:  AMD64
+    Processor Level: 6
+    Processor Revision: 15876
+    Number of Processors: 40
+    Platform ID:     Linux
+    CSD Version:     'Linux 3.13.0-91-generic #138-Ubuntu SMP Fri Jun 24 17:00:34 UTC 2016 x86_64'
+    CPU:
+      Vendor ID:       GenuineIntel
+      Version Info:    0x00000000
+      Feature Info:    0x00000000
+  - Type:            LinuxProcStatus
+    Text:             |
+      Name:	test-yaml
+      Umask:	0002
+      State:	t (tracing stop)
+      Pid:	8567
+  - Type:            ThreadList
+    Threads:
+      - Thread Id:       0x000074DD
+        Context:         0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000B0010000000000033000000000000000000000002020100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000040109600000000000100000000000000000000000000000068E7D0C8FF7F000068E7D0C8FF7F000097E6D0C8FF7F000010109600000000000000000000000000020000000000000088E4D0C8FF7F0000603FFF85C77F0000F00340000000000080E7D0C8FF7F000000000000000000000000000000000000E0034000000000007F0300000000000000000000000000000000000000000000801F0000FFFF00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF252525252525252525252525252525250000000000000000000000000000000000000000000000000000000000000000FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000FF00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
+        Stack:
+          Start of Memory Range: 0x00007FFFC8D0E000
+          Content:               'DEADBEEF'
diff --git a/lldb/test/API/python_api/sbsavecoreoptions/basic_minidump_different_pid.yaml b/lldb/test/API/python_api/sbsavecoreoptions/basic_minidump_different_pid.yaml
new file mode 100644
index 000000000000000..7393f1fe62af3d1
--- /dev/null
+++ b/lldb/test/API/python_api/sbsavecoreoptions/basic_minidump_different_pid.yaml
@@ -0,0 +1,26 @@
+--- !minidump
+Streams:
+  - Type:            SystemInfo
+    Processor Arch:  AMD64
+    Processor Level: 6
+    Processor Revision: 15876
+    Number of Processors: 40
+    Platform ID:     Linux
+    CSD Version:     'Linux 3.13.0-91-generic #138-Ubuntu SMP Fri Jun 24 17:00:34 UTC 2016 x86_64'
+    CPU:
+      Vendor ID:       GenuineIntel
+      Version Info:    0x00000000
+      Feature Info:    0x00000000
+  - Type:            LinuxProcStatus
+    Text:             |
+      Name:	test-yaml
+      Umask:	0002
+      State:	t (tracing stop)
+      Pid:	1967
+  - Type:            ThreadList
+    Threads:
+      - Thread Id:       0x000074DD
+        Context:         0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000B0010000000000033000000000000000000000002020100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000040109600000000000100000000000000000000000000000068E7D0C8FF7F000068E7D0C8FF7F000097E6D0C8FF7F000010109600000000000000000000000000020000000000000088E4D0C8FF7F0000603FFF85C77F0000F00340000000000080E7D0C8FF7F000000000000000000000000000000000000E0034000000000007F0300000000000000000000000000000000000000000000801F0000FFFF00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF252525252525252525252525252525250000000000000000000000000000000000000000000000000000000000000000FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000FF00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
+        Stack:
+          Start of Memory Range: 0x00007FFFC8D0E000
+          Content:               'DEADBEEF'

From 039a86d057b144c963f7a30f9b4b8d3da200576d Mon Sep 17 00:00:00 2001
From: Philip Reames <preames@rivosinc.com>
Date: Thu, 15 Aug 2024 16:46:19 -0700
Subject: [PATCH 099/441] [RISCV] Use significant bits helpers in narrowing of
 build vectors [nfc] (#104511)

---
 llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 18 ++++++++----------
 1 file changed, 8 insertions(+), 10 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index e16dd132ed71bb0..515d7dbcd47bd93 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -3813,16 +3813,14 @@ static SDValue lowerBuildVectorOfConstants(SDValue Op, SelectionDAG &DAG,
   // TODO: We really should be costing the smaller vector.  There are
   // profitable cases this misses.
   if (EltBitSize > 8 && VT.isInteger() &&
-      (NumElts <= 4 || VT.getSizeInBits() > Subtarget.getRealMinVLen())) {
-    unsigned SignBits = DAG.ComputeNumSignBits(Op);
-    if (EltBitSize - SignBits < 8) {
-      SDValue Source = DAG.getBuildVector(VT.changeVectorElementType(MVT::i8),
-                                          DL, Op->ops());
-      Source = convertToScalableVector(ContainerVT.changeVectorElementType(MVT::i8),
-                                       Source, DAG, Subtarget);
-      SDValue Res = DAG.getNode(RISCVISD::VSEXT_VL, DL, ContainerVT, Source, Mask, VL);
-      return convertFromScalableVector(VT, Res, DAG, Subtarget);
-    }
+      (NumElts <= 4 || VT.getSizeInBits() > Subtarget.getRealMinVLen()) &&
+      DAG.ComputeMaxSignificantBits(Op) <= 8) {
+    SDValue Source = DAG.getBuildVector(VT.changeVectorElementType(MVT::i8),
+                                        DL, Op->ops());
+    Source = convertToScalableVector(ContainerVT.changeVectorElementType(MVT::i8),
+                                     Source, DAG, Subtarget);
+    SDValue Res = DAG.getNode(RISCVISD::VSEXT_VL, DL, ContainerVT, Source, Mask, VL);
+    return convertFromScalableVector(VT, Res, DAG, Subtarget);
   }
 
   if (SDValue Res = lowerBuildVectorViaDominantValues(Op, DAG, Subtarget))

From 3e1d4ec671c59204e3e556cc58fd948894af1e1b Mon Sep 17 00:00:00 2001
From: Lang Hames <lhames@gmail.com>
Date: Fri, 16 Aug 2024 09:52:36 +1000
Subject: [PATCH 100/441] [ORC] loadRelocatableObject: universal binary
 support, clearer errors (#104406)

ORC supports loading relocatable object files into a JIT'd process. The
raw "add object file" API (ObjectLayer::add) accepts plain relocatable
object files as llvm::MemoryBuffers only and does not check that the
object file's format or architecture are compatible with the process
that it will be linked in to. This API is flexible, but places the
burden of error checking and universal binary support on clients.

This commit introduces a new utility, loadRelocatableObject, that takes
a path to load and a target triple and then:
1. If the path does not exist, returns a FileError containing the
invalid path.
2. If the path points to a MachO universal binary, identifies and
returns MemoryBuffer covering the slice that matches the given triple
(checking that the slice really does contains a valid MachO relocatable
object with a compatible arch).
3. If the path points to a regular relocatable object file, verifies
that the format and architecture are compatible with the triple.

Clients can use loadRelocatableObject in the common case of loading
object files from disk to simplify their code.

Note: Error checking for ELF and COFF is left as a FIXME.

rdar://133653290
---
 .../llvm/ExecutionEngine/Orc/ExecutionUtils.h |   3 -
 .../Orc/LoadRelocatableObject.h               |  36 ++++
 llvm/include/llvm/ExecutionEngine/Orc/MachO.h |  63 +++++++
 llvm/lib/ExecutionEngine/Orc/CMakeLists.txt   |   2 +
 .../ExecutionEngine/Orc/ExecutionUtils.cpp    |  26 +--
 .../Orc/LoadRelocatableObject.cpp             |  69 ++++++++
 llvm/lib/ExecutionEngine/Orc/MachO.cpp        | 164 ++++++++++++++++++
 .../Generic/MachO_universal_binaries.test     |   9 +
 llvm/tools/llvm-jitlink/llvm-jitlink.cpp      |  39 ++++-
 9 files changed, 379 insertions(+), 32 deletions(-)
 create mode 100644 llvm/include/llvm/ExecutionEngine/Orc/LoadRelocatableObject.h
 create mode 100644 llvm/include/llvm/ExecutionEngine/Orc/MachO.h
 create mode 100644 llvm/lib/ExecutionEngine/Orc/LoadRelocatableObject.cpp
 create mode 100644 llvm/lib/ExecutionEngine/Orc/MachO.cpp
 create mode 100644 llvm/test/ExecutionEngine/JITLink/Generic/MachO_universal_binaries.test

diff --git a/llvm/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h b/llvm/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h
index f997faf1ebcb056..44db455f218a475 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h
@@ -319,9 +319,6 @@ class StaticLibraryDefinitionGenerator : public DefinitionGenerator {
                                    Error &Err);
   Error buildObjectFilesMap();
 
-  static Expected<std::pair<size_t, size_t>>
-  getSliceRangeForArch(object::MachOUniversalBinary &UB, const Triple &TT);
-
   ObjectLayer &L;
   GetObjectFileInterface GetObjFileInterface;
   std::set<std::string> ImportedDynamicLibraries;
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/LoadRelocatableObject.h b/llvm/include/llvm/ExecutionEngine/Orc/LoadRelocatableObject.h
new file mode 100644
index 000000000000000..a6a2f41dcff7d61
--- /dev/null
+++ b/llvm/include/llvm/ExecutionEngine/Orc/LoadRelocatableObject.h
@@ -0,0 +1,36 @@
+//===---- LoadRelocatableObject.h - Load relocatable objects ----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// A wrapper for `MemoryBuffer::getFile` / `MemoryBuffer::getFileSlice` that:
+//
+//   1. Adds file paths to errors by default.
+//   2. Checks architecture compatibility up-front.
+//   3. Handles MachO universal binaries, returning the MemoryBuffer for the
+//      requested slice only.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_EXECUTIONENGINE_ORC_LOADRELOCATABLEOBJECT_H
+#define LLVM_EXECUTIONENGINE_ORC_LOADRELOCATABLEOBJECT_H
+
+#include "llvm/Support/Error.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/TargetParser/Triple.h"
+
+namespace llvm {
+namespace orc {
+
+// Load an object file compatible with the given triple (if given) from the
+// given path. May return a file slice if the path contains a universal binary.
+Expected<std::unique_ptr<MemoryBuffer>> loadRelocatableObject(StringRef Path,
+                                                              const Triple &TT);
+
+} // End namespace orc
+} // End namespace llvm
+
+#endif // LLVM_EXECUTIONENGINE_ORC_LOADRELOCATABLEOBJECT_H
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/MachO.h b/llvm/include/llvm/ExecutionEngine/Orc/MachO.h
new file mode 100644
index 000000000000000..58c04a9121c87b9
--- /dev/null
+++ b/llvm/include/llvm/ExecutionEngine/Orc/MachO.h
@@ -0,0 +1,63 @@
+//===------------- MachO.h - MachO format utilities -------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Contains utilities for load MachO relocatable object files.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_EXECUTIONENGINE_ORC_MACHO_H
+#define LLVM_EXECUTIONENGINE_ORC_MACHO_H
+
+#include "llvm/Support/Error.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/TargetParser/Triple.h"
+
+namespace llvm {
+
+namespace object {
+
+class MachOUniversalBinary;
+
+} // namespace object
+
+namespace orc {
+
+/// Check that the given buffer contains a MachO object file compatible with the
+/// given triple.
+/// ObjIsSlice should be set to true if Obj is a slice of a universal binary
+/// (that fact will then be reported in the error messages).
+Expected<std::unique_ptr<MemoryBuffer>>
+checkMachORelocatableObject(std::unique_ptr<MemoryBuffer> Obj, const Triple &TT,
+                            bool ObjIsSlice);
+
+/// Load a relocatable object compatible with TT from Path.
+/// If Path is a universal binary, this function will return a buffer for the
+/// slice compatible with Triple (if one is present).
+Expected<std::unique_ptr<MemoryBuffer>>
+loadMachORelocatableObject(StringRef Path, const Triple &TT);
+
+/// Load a compatible relocatable object (if available) from a MachO universal
+/// binary.
+Expected<std::unique_ptr<MemoryBuffer>>
+loadMachORelocatableObjectFromUniversalBinary(
+    StringRef UBPath, std::unique_ptr<MemoryBuffer> UBBuf, const Triple &TT);
+
+/// Utility for identifying the file-slice compatible with TT in a universal
+/// binary.
+Expected<std::pair<size_t, size_t>>
+getMachOSliceRangeForTriple(object::MachOUniversalBinary &UB, const Triple &TT);
+
+/// Utility for identifying the file-slice compatible with TT in a universal
+/// binary.
+Expected<std::pair<size_t, size_t>>
+getMachOSliceRangeForTriple(MemoryBufferRef UBBuf, const Triple &TT);
+
+} // namespace orc
+} // namespace llvm
+
+#endif // LLVM_EXECUTIONENGINE_ORC_MACHO_H
diff --git a/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt b/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt
index f36605bbf03baf7..f8bc82ba8d6f9b8 100644
--- a/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt
+++ b/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt
@@ -29,8 +29,10 @@ add_llvm_component_library(LLVMOrcJIT
   JITTargetMachineBuilder.cpp
   LazyReexports.cpp
   Layer.cpp
+  LoadRelocatableObject.cpp
   LookupAndRecordAddrs.cpp
   LLJIT.cpp
+  MachO.cpp
   MachOPlatform.cpp
   MapperJITLinkMemoryManager.cpp
   MemoryMapper.cpp
diff --git a/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp b/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp
index c1a193f6a280248..36bd425beab8de8 100644
--- a/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp
@@ -9,6 +9,7 @@
 #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
 #include "llvm/ExecutionEngine/JITLink/x86_64.h"
 #include "llvm/ExecutionEngine/Orc/Layer.h"
+#include "llvm/ExecutionEngine/Orc/MachO.h"
 #include "llvm/ExecutionEngine/Orc/ObjectFileInterface.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/Function.h"
@@ -295,7 +296,7 @@ StaticLibraryDefinitionGenerator::Load(
 
     const auto &TT = L.getExecutionSession().getTargetTriple();
 
-    auto SliceRange = getSliceRangeForArch(*UB, TT);
+    auto SliceRange = getMachOSliceRangeForTriple(*UB, TT);
     if (!SliceRange)
       return SliceRange.takeError();
 
@@ -358,7 +359,7 @@ StaticLibraryDefinitionGenerator::Create(
 
     const auto &TT = L.getExecutionSession().getTargetTriple();
 
-    auto SliceRange = getSliceRangeForArch(*UB, TT);
+    auto SliceRange = getMachOSliceRangeForTriple(*UB, TT);
     if (!SliceRange)
       return SliceRange.takeError();
 
@@ -460,27 +461,6 @@ Error StaticLibraryDefinitionGenerator::buildObjectFilesMap() {
   return Error::success();
 }
 
-Expected<std::pair<size_t, size_t>>
-StaticLibraryDefinitionGenerator::getSliceRangeForArch(
-    object::MachOUniversalBinary &UB, const Triple &TT) {
-
-  for (const auto &Obj : UB.objects()) {
-    auto ObjTT = Obj.getTriple();
-    if (ObjTT.getArch() == TT.getArch() &&
-        ObjTT.getSubArch() == TT.getSubArch() &&
-        (TT.getVendor() == Triple::UnknownVendor ||
-         ObjTT.getVendor() == TT.getVendor())) {
-      // We found a match. Return the range for the slice.
-      return std::make_pair(Obj.getOffset(), Obj.getSize());
-    }
-  }
-
-  return make_error<StringError>(Twine("Universal binary ") + UB.getFileName() +
-                                     " does not contain a slice for " +
-                                     TT.str(),
-                                 inconvertibleErrorCode());
-}
-
 StaticLibraryDefinitionGenerator::StaticLibraryDefinitionGenerator(
     ObjectLayer &L, std::unique_ptr<MemoryBuffer> ArchiveBuffer,
     std::unique_ptr<object::Archive> Archive,
diff --git a/llvm/lib/ExecutionEngine/Orc/LoadRelocatableObject.cpp b/llvm/lib/ExecutionEngine/Orc/LoadRelocatableObject.cpp
new file mode 100644
index 000000000000000..5fde1b5fb4da927
--- /dev/null
+++ b/llvm/lib/ExecutionEngine/Orc/LoadRelocatableObject.cpp
@@ -0,0 +1,69 @@
+//===----- LoadRelocatableObject.cpp -- Load relocatable object files -----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ExecutionEngine/Orc/LoadRelocatableObject.h"
+#include "llvm/BinaryFormat/Magic.h"
+#include "llvm/ExecutionEngine/Orc/MachO.h"
+
+#define DEBUG_TYPE "orc"
+
+namespace llvm {
+namespace orc {
+
+static Expected<std::unique_ptr<MemoryBuffer>>
+checkCOFFRelocatableObject(std::unique_ptr<MemoryBuffer> Obj,
+                           const Triple &TT) {
+  // TODO: Actually check the architecture of the file.
+  return std::move(Obj);
+}
+
+static Expected<std::unique_ptr<MemoryBuffer>>
+checkELFRelocatableObject(std::unique_ptr<MemoryBuffer> Obj, const Triple &TT) {
+  // TODO: Actually check the architecture of the file.
+  return std::move(Obj);
+}
+
+Expected<std::unique_ptr<MemoryBuffer>>
+loadRelocatableObject(StringRef Path, const Triple &TT) {
+  auto Buf = MemoryBuffer::getFile(Path);
+  if (!Buf)
+    return createFileError(Path, Buf.getError());
+
+  std::optional<Triple::ObjectFormatType> RequireFormat;
+  if (TT.getObjectFormat() != Triple::UnknownObjectFormat)
+    RequireFormat = TT.getObjectFormat();
+
+  switch (identify_magic((*Buf)->getBuffer())) {
+  case file_magic::coff_object:
+    if (!RequireFormat || *RequireFormat == Triple::COFF)
+      return checkCOFFRelocatableObject(std::move(*Buf), TT);
+    break;
+  case file_magic::elf_relocatable:
+    if (!RequireFormat || *RequireFormat == Triple::ELF)
+      return checkELFRelocatableObject(std::move(*Buf), TT);
+    break;
+  case file_magic::macho_object:
+    if (!RequireFormat || *RequireFormat == Triple::MachO)
+      return checkMachORelocatableObject(std::move(*Buf), TT, false);
+    break;
+  case file_magic::macho_universal_binary:
+    if (!RequireFormat || *RequireFormat == Triple::MachO)
+      return loadMachORelocatableObjectFromUniversalBinary(Path,
+                                                           std::move(*Buf), TT);
+    break;
+  default:
+    break;
+  }
+  return make_error<StringError>(
+      Path + " does not contain a relocatable object file compatible with " +
+          TT.str(),
+      inconvertibleErrorCode());
+}
+
+} // End namespace orc.
+} // End namespace llvm.
diff --git a/llvm/lib/ExecutionEngine/Orc/MachO.cpp b/llvm/lib/ExecutionEngine/Orc/MachO.cpp
new file mode 100644
index 000000000000000..fe5a1dbbb9f8a1a
--- /dev/null
+++ b/llvm/lib/ExecutionEngine/Orc/MachO.cpp
@@ -0,0 +1,164 @@
+//===----------------- MachO.cpp - MachO format utilities -----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ExecutionEngine/Orc/MachO.h"
+
+#include "llvm/BinaryFormat/MachO.h"
+#include "llvm/Object/MachOUniversal.h"
+
+#define DEBUG_TYPE "orc"
+
+namespace llvm {
+namespace orc {
+
+static std::string objDesc(MemoryBuffer &Obj, const Triple &TT,
+                           bool ObjIsSlice) {
+  std::string Desc;
+  if (ObjIsSlice)
+    Desc += (TT.getArchName() + " slice of universal binary").str();
+  Desc += Obj.getBufferIdentifier();
+  return Desc;
+}
+
+template <typename HeaderType>
+static Expected<std::unique_ptr<MemoryBuffer>>
+checkMachORelocatableObject(std::unique_ptr<MemoryBuffer> Obj,
+                            bool SwapEndianness, const Triple &TT,
+                            bool ObjIsSlice) {
+  StringRef Data = Obj->getBuffer();
+
+  HeaderType Hdr;
+  memcpy(&Hdr, Data.data(), sizeof(HeaderType));
+
+  if (SwapEndianness)
+    swapStruct(Hdr);
+
+  if (Hdr.filetype != MachO::MH_OBJECT)
+    return make_error<StringError>(objDesc(*Obj, TT, ObjIsSlice) +
+                                       " is not a MachO relocatable object",
+                                   inconvertibleErrorCode());
+
+  auto ObjArch = object::MachOObjectFile::getArch(Hdr.cputype, Hdr.cpusubtype);
+  if (ObjArch != TT.getArch())
+    return make_error<StringError>(
+        objDesc(*Obj, TT, ObjIsSlice) + Triple::getArchTypeName(ObjArch) +
+            ", cannot be loaded into " + TT.str() + " process",
+        inconvertibleErrorCode());
+
+  return std::move(Obj);
+}
+
+Expected<std::unique_ptr<MemoryBuffer>>
+checkMachORelocatableObject(std::unique_ptr<MemoryBuffer> Obj, const Triple &TT,
+                            bool ObjIsSlice) {
+  StringRef Data = Obj->getBuffer();
+
+  if (Data.size() < 4)
+    return make_error<StringError>(
+        objDesc(*Obj, TT, ObjIsSlice) +
+            " is not a valid MachO relocatable object file (truncated header)",
+        inconvertibleErrorCode());
+
+  uint32_t Magic;
+  memcpy(&Magic, Data.data(), sizeof(uint32_t));
+
+  switch (Magic) {
+  case MachO::MH_MAGIC:
+  case MachO::MH_CIGAM:
+    return checkMachORelocatableObject<MachO::mach_header>(
+        std::move(Obj), Magic == MachO::MH_CIGAM, TT, ObjIsSlice);
+  case MachO::MH_MAGIC_64:
+  case MachO::MH_CIGAM_64:
+    return checkMachORelocatableObject<MachO::mach_header_64>(
+        std::move(Obj), Magic == MachO::MH_CIGAM_64, TT, ObjIsSlice);
+  default:
+    return make_error<StringError>(
+        objDesc(*Obj, TT, ObjIsSlice) +
+            " is not a valid MachO relocatable object (bad magic value)",
+        inconvertibleErrorCode());
+  }
+}
+
+Expected<std::unique_ptr<MemoryBuffer>>
+loadMachORelocatableObject(StringRef Path, const Triple &TT) {
+  assert((TT.getObjectFormat() == Triple::UnknownObjectFormat ||
+          TT.getObjectFormat() == Triple::MachO) &&
+         "TT must specify MachO or Unknown object format");
+
+  auto Buf = MemoryBuffer::getFile(Path);
+  if (!Buf)
+    return createFileError(Path, Buf.getError());
+
+  switch (identify_magic((*Buf)->getBuffer())) {
+  case file_magic::macho_object:
+    return checkMachORelocatableObject(std::move(*Buf), TT, false);
+  case file_magic::macho_universal_binary:
+    return loadMachORelocatableObjectFromUniversalBinary(Path, std::move(*Buf),
+                                                         TT);
+  default:
+    return make_error<StringError>(
+        Path + " does not contain a relocatable object file compatible with " +
+            TT.str(),
+        inconvertibleErrorCode());
+  }
+}
+
+Expected<std::unique_ptr<MemoryBuffer>>
+loadMachORelocatableObjectFromUniversalBinary(
+    StringRef UBPath, std::unique_ptr<MemoryBuffer> UBBuf, const Triple &TT) {
+
+  auto UniversalBin =
+      object::MachOUniversalBinary::create(UBBuf->getMemBufferRef());
+  if (!UniversalBin)
+    return UniversalBin.takeError();
+
+  auto SliceRange = getMachOSliceRangeForTriple(**UniversalBin, TT);
+  if (!SliceRange)
+    return SliceRange.takeError();
+
+  auto ObjBuf = errorOrToExpected(MemoryBuffer::getFileSlice(
+      UBPath, SliceRange->second, SliceRange->first, false));
+  if (!ObjBuf)
+    return createFileError(UBPath, ObjBuf.takeError());
+
+  return checkMachORelocatableObject(std::move(*ObjBuf), TT, true);
+}
+
+Expected<std::pair<size_t, size_t>>
+getMachOSliceRangeForTriple(object::MachOUniversalBinary &UB,
+                            const Triple &TT) {
+
+  for (const auto &Obj : UB.objects()) {
+    auto ObjTT = Obj.getTriple();
+    if (ObjTT.getArch() == TT.getArch() &&
+        ObjTT.getSubArch() == TT.getSubArch() &&
+        (TT.getVendor() == Triple::UnknownVendor ||
+         ObjTT.getVendor() == TT.getVendor())) {
+      // We found a match. Return the range for the slice.
+      return std::make_pair(Obj.getOffset(), Obj.getSize());
+    }
+  }
+
+  return make_error<StringError>(Twine("Universal binary ") + UB.getFileName() +
+                                     " does not contain a slice for " +
+                                     TT.str(),
+                                 inconvertibleErrorCode());
+}
+
+Expected<std::pair<size_t, size_t>>
+getMachOSliceRangeForTriple(MemoryBufferRef UBBuf, const Triple &TT) {
+
+  auto UB = object::MachOUniversalBinary::create(UBBuf);
+  if (!UB)
+    return UB.takeError();
+
+  return getMachOSliceRangeForTriple(**UB, TT);
+}
+
+} // End namespace orc.
+} // End namespace llvm.
diff --git a/llvm/test/ExecutionEngine/JITLink/Generic/MachO_universal_binaries.test b/llvm/test/ExecutionEngine/JITLink/Generic/MachO_universal_binaries.test
new file mode 100644
index 000000000000000..684068807f01ee9
--- /dev/null
+++ b/llvm/test/ExecutionEngine/JITLink/Generic/MachO_universal_binaries.test
@@ -0,0 +1,9 @@
+# RUN: llc -filetype=obj -mtriple x86_64-apple-macosx -o %t.x86-64.o \
+# RUN:   %S/Inputs/main-ret-0.ll
+# RUN: llc -filetype=obj -mtriple arm64-apple-macosx -o %t.arm64.o \
+# RUN:   %S/Inputs/main-ret-0.ll
+# RUN: llvm-lipo -create -output %t.universal.o -arch x86_64 %t.x86-64.o \
+# RUN:   -arch arm64 %t.arm64.o
+# RUN: llvm-jitlink -noexec -triple arm64-apple-macosx %t.universal.o
+#
+# Check that universal binaries don't raise "not a valid object file" errors.
diff --git a/llvm/tools/llvm-jitlink/llvm-jitlink.cpp b/llvm/tools/llvm-jitlink/llvm-jitlink.cpp
index bff05b9ca4bebc9..01f2033c397fb57 100644
--- a/llvm/tools/llvm-jitlink/llvm-jitlink.cpp
+++ b/llvm/tools/llvm-jitlink/llvm-jitlink.cpp
@@ -28,6 +28,8 @@
 #include "llvm/ExecutionEngine/Orc/EPCEHFrameRegistrar.h"
 #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
 #include "llvm/ExecutionEngine/Orc/IndirectionUtils.h"
+#include "llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h"
+#include "llvm/ExecutionEngine/Orc/LoadRelocatableObject.h"
 #include "llvm/ExecutionEngine/Orc/MachOPlatform.h"
 #include "llvm/ExecutionEngine/Orc/MapperJITLinkMemoryManager.h"
 #include "llvm/ExecutionEngine/Orc/ObjectFileInterface.h"
@@ -260,6 +262,10 @@ static cl::opt<bool> UseSharedMemory(
     cl::desc("Use shared memory to transfer generated code and data"),
     cl::init(false), cl::cat(JITLinkCategory));
 
+static cl::opt<std::string>
+    OverrideTriple("triple", cl::desc("Override target triple detection"),
+                   cl::init(""), cl::cat(JITLinkCategory));
+
 static ExitOnError ExitOnErr;
 
 static LLVM_ATTRIBUTE_USED void linkComponents() {
@@ -1101,7 +1107,8 @@ Session::Session(std::unique_ptr<ExecutorProcessControl> EPC, Error &Err)
   for (auto &HarnessFile : TestHarnesses) {
     HarnessFiles.insert(HarnessFile);
 
-    auto ObjBuffer = ExitOnErr(getFile(HarnessFile));
+    auto ObjBuffer =
+        ExitOnErr(loadRelocatableObject(HarnessFile, ES.getTargetTriple()));
 
     auto ObjInterface =
         ExitOnErr(getObjectFileInterface(ES, ObjBuffer->getMemBufferRef()));
@@ -1418,6 +1425,14 @@ Session::findSymbolInfo(StringRef SymbolName, Twine ErrorMsgStem) {
 static std::pair<Triple, SubtargetFeatures> getFirstFileTripleAndFeatures() {
   static std::pair<Triple, SubtargetFeatures> FirstTTAndFeatures = []() {
     assert(!InputFiles.empty() && "InputFiles can not be empty");
+
+    if (!OverrideTriple.empty()) {
+      LLVM_DEBUG({
+        dbgs() << "Triple from -triple override: " << OverrideTriple << "\n";
+      });
+      return std::make_pair(Triple(OverrideTriple), SubtargetFeatures());
+    }
+
     for (auto InputFile : InputFiles) {
       auto ObjBuffer = ExitOnErr(getFile(InputFile));
       file_magic Magic = identify_magic(ObjBuffer->getBuffer());
@@ -1436,13 +1451,25 @@ static std::pair<Triple, SubtargetFeatures> getFirstFileTripleAndFeatures() {
         SubtargetFeatures Features;
         if (auto ObjFeatures = Obj->getFeatures())
           Features = std::move(*ObjFeatures);
+
+        LLVM_DEBUG({
+          dbgs() << "Triple from " << InputFile << ": " << TT.str() << "\n";
+        });
         return std::make_pair(TT, Features);
       }
       default:
         break;
       }
     }
-    return std::make_pair(Triple(), SubtargetFeatures());
+
+    // If no plain object file inputs exist to pin down the triple then detect
+    // the host triple and default to that.
+    auto JTMB = ExitOnErr(JITTargetMachineBuilder::detectHost());
+    LLVM_DEBUG({
+      dbgs() << "Triple from host-detection: " << JTMB.getTargetTriple().str()
+             << "\n";
+    });
+    return std::make_pair(JTMB.getTargetTriple(), JTMB.getFeatures());
   }();
 
   return FirstTTAndFeatures;
@@ -1711,9 +1738,9 @@ static Error addSectCreates(Session &S,
                                          ", filename component cannot be empty",
                                      inconvertibleErrorCode());
 
-    auto Content = MemoryBuffer::getFile(FileName);
+    auto Content = getFile(FileName);
     if (!Content)
-      return createFileError(FileName, errorCodeToError(Content.getError()));
+      return Content.takeError();
 
     SectCreateMaterializationUnit::ExtraSymbolsMap ExtraSymbols;
     while (!ExtraSymbolsString.empty()) {
@@ -1745,7 +1772,7 @@ static Error addTestHarnesses(Session &S) {
   LLVM_DEBUG(dbgs() << "Adding test harness objects...\n");
   for (auto HarnessFile : TestHarnesses) {
     LLVM_DEBUG(dbgs() << "  " << HarnessFile << "\n");
-    auto ObjBuffer = getFile(HarnessFile);
+    auto ObjBuffer = loadRelocatableObject(HarnessFile, S.ES.getTargetTriple());
     if (!ObjBuffer)
       return ObjBuffer.takeError();
     if (auto Err = S.ObjLayer.add(*S.MainJD, std::move(*ObjBuffer)))
@@ -1770,7 +1797,7 @@ static Error addObjects(Session &S,
     auto &JD = *std::prev(IdxToJD.lower_bound(InputFileArgIdx))->second;
     LLVM_DEBUG(dbgs() << "  " << InputFileArgIdx << ": \"" << InputFile
                       << "\" to " << JD.getName() << "\n";);
-    auto ObjBuffer = getFile(InputFile);
+    auto ObjBuffer = loadRelocatableObject(InputFile, S.ES.getTargetTriple());
     if (!ObjBuffer)
       return ObjBuffer.takeError();
 

From ec29660c44e5e73d3b78f4884f9178036563fb25 Mon Sep 17 00:00:00 2001
From: vporpo <vporpodas@google.com>
Date: Thu, 15 Aug 2024 16:55:01 -0700
Subject: [PATCH 101/441] [SandboxIR] Implement UnaryOperator (#104509)

This patch implements sandboxir::UnaryOperator mirroring
llvm::UnaryOperator.
---
 llvm/include/llvm/SandboxIR/SandboxIR.h       |  46 +++++++
 .../llvm/SandboxIR/SandboxIRValues.def        |   5 +-
 llvm/lib/SandboxIR/SandboxIR.cpp              |  75 +++++++++++
 llvm/unittests/SandboxIR/SandboxIRTest.cpp    | 126 ++++++++++++++++++
 4 files changed, 251 insertions(+), 1 deletion(-)

diff --git a/llvm/include/llvm/SandboxIR/SandboxIR.h b/llvm/include/llvm/SandboxIR/SandboxIR.h
index b1af769f29af540..423dad854a91cbf 100644
--- a/llvm/include/llvm/SandboxIR/SandboxIR.h
+++ b/llvm/include/llvm/SandboxIR/SandboxIR.h
@@ -130,6 +130,7 @@ class CastInst;
 class PtrToIntInst;
 class BitCastInst;
 class AllocaInst;
+class UnaryOperator;
 class BinaryOperator;
 class AtomicCmpXchgInst;
 
@@ -250,6 +251,7 @@ class Value {
   friend class InvokeInst;         // For getting `Val`.
   friend class CallBrInst;         // For getting `Val`.
   friend class GetElementPtrInst;  // For getting `Val`.
+  friend class UnaryOperator;      // For getting `Val`.
   friend class BinaryOperator;     // For getting `Val`.
   friend class AtomicCmpXchgInst;  // For getting `Val`.
   friend class AllocaInst;         // For getting `Val`.
@@ -632,6 +634,7 @@ class Instruction : public sandboxir::User {
   friend class InvokeInst;         // For getTopmostLLVMInstruction().
   friend class CallBrInst;         // For getTopmostLLVMInstruction().
   friend class GetElementPtrInst;  // For getTopmostLLVMInstruction().
+  friend class UnaryOperator;      // For getTopmostLLVMInstruction().
   friend class BinaryOperator;     // For getTopmostLLVMInstruction().
   friend class AtomicCmpXchgInst;  // For getTopmostLLVMInstruction().
   friend class AllocaInst;         // For getTopmostLLVMInstruction().
@@ -1435,6 +1438,47 @@ class GetElementPtrInst final
   // TODO: Add missing member functions.
 };
 
+class UnaryOperator : public UnaryInstruction {
+  static Opcode getUnaryOpcode(llvm::Instruction::UnaryOps UnOp) {
+    switch (UnOp) {
+    case llvm::Instruction::FNeg:
+      return Opcode::FNeg;
+    case llvm::Instruction::UnaryOpsEnd:
+      llvm_unreachable("Bad UnOp!");
+    }
+    llvm_unreachable("Unhandled UnOp!");
+  }
+  UnaryOperator(llvm::UnaryOperator *UO, Context &Ctx)
+      : UnaryInstruction(ClassID::UnOp, getUnaryOpcode(UO->getOpcode()), UO,
+                         Ctx) {}
+  friend Context; // for constructor.
+public:
+  static Value *create(Instruction::Opcode Op, Value *OpV, BBIterator WhereIt,
+                       BasicBlock *WhereBB, Context &Ctx,
+                       const Twine &Name = "");
+  static Value *create(Instruction::Opcode Op, Value *OpV,
+                       Instruction *InsertBefore, Context &Ctx,
+                       const Twine &Name = "");
+  static Value *create(Instruction::Opcode Op, Value *OpV,
+                       BasicBlock *InsertAtEnd, Context &Ctx,
+                       const Twine &Name = "");
+  static Value *createWithCopiedFlags(Instruction::Opcode Op, Value *OpV,
+                                      Value *CopyFrom, BBIterator WhereIt,
+                                      BasicBlock *WhereBB, Context &Ctx,
+                                      const Twine &Name = "");
+  static Value *createWithCopiedFlags(Instruction::Opcode Op, Value *OpV,
+                                      Value *CopyFrom,
+                                      Instruction *InsertBefore, Context &Ctx,
+                                      const Twine &Name = "");
+  static Value *createWithCopiedFlags(Instruction::Opcode Op, Value *OpV,
+                                      Value *CopyFrom, BasicBlock *InsertAtEnd,
+                                      Context &Ctx, const Twine &Name = "");
+  /// For isa/dyn_cast.
+  static bool classof(const Value *From) {
+    return From->getSubclassID() == ClassID::UnOp;
+  }
+};
+
 class BinaryOperator : public SingleLLVMInstructionImpl<llvm::BinaryOperator> {
   static Opcode getBinOpOpcode(llvm::Instruction::BinaryOps BinOp) {
     switch (BinOp) {
@@ -1959,6 +2003,8 @@ class Context {
   friend CallBrInst; // For createCallBrInst()
   GetElementPtrInst *createGetElementPtrInst(llvm::GetElementPtrInst *I);
   friend GetElementPtrInst; // For createGetElementPtrInst()
+  UnaryOperator *createUnaryOperator(llvm::UnaryOperator *I);
+  friend UnaryOperator; // For createUnaryOperator()
   BinaryOperator *createBinaryOperator(llvm::BinaryOperator *I);
   friend BinaryOperator; // For createBinaryOperator()
   AtomicCmpXchgInst *createAtomicCmpXchgInst(llvm::AtomicCmpXchgInst *I);
diff --git a/llvm/include/llvm/SandboxIR/SandboxIRValues.def b/llvm/include/llvm/SandboxIR/SandboxIRValues.def
index 43441ba26ec2cbe..7332316c85026c3 100644
--- a/llvm/include/llvm/SandboxIR/SandboxIRValues.def
+++ b/llvm/include/llvm/SandboxIR/SandboxIRValues.def
@@ -45,7 +45,10 @@ DEF_INSTR(Call,          OP(Call),          CallInst)
 DEF_INSTR(Invoke,        OP(Invoke),        InvokeInst)
 DEF_INSTR(CallBr,        OP(CallBr),        CallBrInst)
 DEF_INSTR(GetElementPtr, OP(GetElementPtr), GetElementPtrInst)
-DEF_INSTR(BinaryOperator, OPCODES( \
+DEF_INSTR(UnOp,          OPCODES( \
+                         OP(FNeg) \
+                         ),                 UnaryOperator)
+DEF_INSTR(BinaryOperator, OPCODES(\
                          OP(Add)  \
                          OP(FAdd) \
                          OP(Sub)  \
diff --git a/llvm/lib/SandboxIR/SandboxIR.cpp b/llvm/lib/SandboxIR/SandboxIR.cpp
index 52df53892b45061..67262bdd0dea998 100644
--- a/llvm/lib/SandboxIR/SandboxIR.cpp
+++ b/llvm/lib/SandboxIR/SandboxIR.cpp
@@ -1219,6 +1219,71 @@ static llvm::Instruction::CastOps getLLVMCastOp(Instruction::Opcode Opc) {
   }
 }
 
+/// \Returns the LLVM opcode that corresponds to \p Opc.
+static llvm::Instruction::UnaryOps getLLVMUnaryOp(Instruction::Opcode Opc) {
+  switch (Opc) {
+  case Instruction::Opcode::FNeg:
+    return static_cast<llvm::Instruction::UnaryOps>(llvm::Instruction::FNeg);
+  default:
+    llvm_unreachable("Not a unary op!");
+  }
+}
+
+Value *UnaryOperator::create(Instruction::Opcode Op, Value *OpV,
+                             BBIterator WhereIt, BasicBlock *WhereBB,
+                             Context &Ctx, const Twine &Name) {
+  auto &Builder = Ctx.getLLVMIRBuilder();
+  if (WhereIt == WhereBB->end())
+    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
+  else
+    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
+  auto *NewLLVMV = Builder.CreateUnOp(getLLVMUnaryOp(Op), OpV->Val, Name);
+  if (auto *NewUnOpV = dyn_cast<llvm::UnaryOperator>(NewLLVMV)) {
+    return Ctx.createUnaryOperator(NewUnOpV);
+  }
+  assert(isa<llvm::Constant>(NewLLVMV) && "Expected constant");
+  return Ctx.getOrCreateConstant(cast<llvm::Constant>(NewLLVMV));
+}
+
+Value *UnaryOperator::create(Instruction::Opcode Op, Value *OpV,
+                             Instruction *InsertBefore, Context &Ctx,
+                             const Twine &Name) {
+  return create(Op, OpV, InsertBefore->getIterator(), InsertBefore->getParent(),
+                Ctx, Name);
+}
+
+Value *UnaryOperator::create(Instruction::Opcode Op, Value *OpV,
+                             BasicBlock *InsertAfter, Context &Ctx,
+                             const Twine &Name) {
+  return create(Op, OpV, InsertAfter->end(), InsertAfter, Ctx, Name);
+}
+
+Value *UnaryOperator::createWithCopiedFlags(Instruction::Opcode Op, Value *OpV,
+                                            Value *CopyFrom, BBIterator WhereIt,
+                                            BasicBlock *WhereBB, Context &Ctx,
+                                            const Twine &Name) {
+  auto *NewV = create(Op, OpV, WhereIt, WhereBB, Ctx, Name);
+  if (auto *UnI = dyn_cast<llvm::UnaryOperator>(NewV->Val))
+    UnI->copyIRFlags(CopyFrom->Val);
+  return NewV;
+}
+
+Value *UnaryOperator::createWithCopiedFlags(Instruction::Opcode Op, Value *OpV,
+                                            Value *CopyFrom,
+                                            Instruction *InsertBefore,
+                                            Context &Ctx, const Twine &Name) {
+  return createWithCopiedFlags(Op, OpV, CopyFrom, InsertBefore->getIterator(),
+                               InsertBefore->getParent(), Ctx, Name);
+}
+
+Value *UnaryOperator::createWithCopiedFlags(Instruction::Opcode Op, Value *OpV,
+                                            Value *CopyFrom,
+                                            BasicBlock *InsertAtEnd,
+                                            Context &Ctx, const Twine &Name) {
+  return createWithCopiedFlags(Op, OpV, CopyFrom, InsertAtEnd->end(),
+                               InsertAtEnd, Ctx, Name);
+}
+
 /// \Returns the LLVM opcode that corresponds to \p Opc.
 static llvm::Instruction::BinaryOps getLLVMBinaryOp(Instruction::Opcode Opc) {
   switch (Opc) {
@@ -1729,6 +1794,12 @@ Value *Context::getOrCreateValueInternal(llvm::Value *LLVMV, llvm::User *U) {
         new GetElementPtrInst(LLVMGEP, *this));
     return It->second.get();
   }
+  case llvm::Instruction::FNeg: {
+    auto *LLVMUnaryOperator = cast<llvm::UnaryOperator>(LLVMV);
+    It->second = std::unique_ptr<UnaryOperator>(
+        new UnaryOperator(LLVMUnaryOperator, *this));
+    return It->second.get();
+  }
   case llvm::Instruction::Add:
   case llvm::Instruction::FAdd:
   case llvm::Instruction::Sub:
@@ -1875,6 +1946,10 @@ Context::createGetElementPtrInst(llvm::GetElementPtrInst *I) {
       std::unique_ptr<GetElementPtrInst>(new GetElementPtrInst(I, *this));
   return cast<GetElementPtrInst>(registerValue(std::move(NewPtr)));
 }
+UnaryOperator *Context::createUnaryOperator(llvm::UnaryOperator *I) {
+  auto NewPtr = std::unique_ptr<UnaryOperator>(new UnaryOperator(I, *this));
+  return cast<UnaryOperator>(registerValue(std::move(NewPtr)));
+}
 BinaryOperator *Context::createBinaryOperator(llvm::BinaryOperator *I) {
   auto NewPtr = std::unique_ptr<BinaryOperator>(new BinaryOperator(I, *this));
   return cast<BinaryOperator>(registerValue(std::move(NewPtr)));
diff --git a/llvm/unittests/SandboxIR/SandboxIRTest.cpp b/llvm/unittests/SandboxIR/SandboxIRTest.cpp
index f5e555ba73287b5..3df335985aa705a 100644
--- a/llvm/unittests/SandboxIR/SandboxIRTest.cpp
+++ b/llvm/unittests/SandboxIR/SandboxIRTest.cpp
@@ -1620,6 +1620,132 @@ define void @foo(i32 %arg, float %farg) {
   EXPECT_FALSE(FAdd->getFastMathFlags() != LLVMFAdd->getFastMathFlags());
 }
 
+TEST_F(SandboxIRTest, UnaryOperator) {
+  parseIR(C, R"IR(
+define void @foo(float %arg0) {
+  %fneg = fneg float %arg0
+  %copyfrom = fadd reassoc float %arg0, 42.0
+  ret void
+}
+)IR");
+  Function &LLVMF = *M->getFunction("foo");
+  sandboxir::Context Ctx(C);
+
+  auto &F = *Ctx.createFunction(&LLVMF);
+  auto *Arg0 = F.getArg(0);
+  auto *BB = &*F.begin();
+  auto It = BB->begin();
+  auto *I = cast<sandboxir::UnaryOperator>(&*It++);
+  auto *CopyFrom = cast<sandboxir::BinaryOperator>(&*It++);
+  auto *Ret = &*It++;
+  EXPECT_EQ(I->getOpcode(), sandboxir::Instruction::Opcode::FNeg);
+  EXPECT_EQ(I->getOperand(0), Arg0);
+
+  {
+    // Check create() WhereIt, WhereBB.
+    auto *NewI =
+        cast<sandboxir::UnaryOperator>(sandboxir::UnaryOperator::create(
+            sandboxir::Instruction::Opcode::FNeg, Arg0,
+            /*WhereIt=*/Ret->getIterator(), /*WhereBB=*/Ret->getParent(), Ctx,
+            "New1"));
+    EXPECT_EQ(NewI->getOpcode(), sandboxir::Instruction::Opcode::FNeg);
+    EXPECT_EQ(NewI->getOperand(0), Arg0);
+#ifndef NDEBUG
+    EXPECT_EQ(NewI->getName(), "New1");
+#endif // NDEBUG
+    EXPECT_EQ(NewI->getNextNode(), Ret);
+  }
+  {
+    // Check create() InsertBefore.
+    auto *NewI =
+        cast<sandboxir::UnaryOperator>(sandboxir::UnaryOperator::create(
+            sandboxir::Instruction::Opcode::FNeg, Arg0,
+            /*InsertBefore=*/Ret, Ctx, "New2"));
+    EXPECT_EQ(NewI->getOpcode(), sandboxir::Instruction::Opcode::FNeg);
+    EXPECT_EQ(NewI->getOperand(0), Arg0);
+#ifndef NDEBUG
+    EXPECT_EQ(NewI->getName(), "New2");
+#endif // NDEBUG
+    EXPECT_EQ(NewI->getNextNode(), Ret);
+  }
+  {
+    // Check create() InsertAtEnd.
+    auto *NewI =
+        cast<sandboxir::UnaryOperator>(sandboxir::UnaryOperator::create(
+            sandboxir::Instruction::Opcode::FNeg, Arg0,
+            /*InsertAtEnd=*/BB, Ctx, "New3"));
+    EXPECT_EQ(NewI->getOpcode(), sandboxir::Instruction::Opcode::FNeg);
+    EXPECT_EQ(NewI->getOperand(0), Arg0);
+#ifndef NDEBUG
+    EXPECT_EQ(NewI->getName(), "New3");
+#endif // NDEBUG
+    EXPECT_EQ(NewI->getParent(), BB);
+    EXPECT_EQ(NewI->getNextNode(), nullptr);
+  }
+  {
+    // Check create() when it gets folded.
+    auto *FortyTwo = CopyFrom->getOperand(1);
+    auto *NewV = sandboxir::UnaryOperator::create(
+        sandboxir::Instruction::Opcode::FNeg, FortyTwo,
+        /*WhereIt=*/Ret->getIterator(), /*WhereBB=*/Ret->getParent(), Ctx,
+        "Folded");
+    EXPECT_TRUE(isa<sandboxir::Constant>(NewV));
+  }
+
+  {
+    // Check createWithCopiedFlags() WhereIt, WhereBB.
+    auto *NewI = cast<sandboxir::UnaryOperator>(
+        sandboxir::UnaryOperator::createWithCopiedFlags(
+            sandboxir::Instruction::Opcode::FNeg, Arg0, CopyFrom,
+            /*WhereIt=*/Ret->getIterator(), /*WhereBB=*/Ret->getParent(), Ctx,
+            "NewCopyFrom1"));
+    EXPECT_EQ(NewI->hasAllowReassoc(), CopyFrom->hasAllowReassoc());
+    EXPECT_EQ(NewI->getOpcode(), sandboxir::Instruction::Opcode::FNeg);
+    EXPECT_EQ(NewI->getOperand(0), Arg0);
+#ifndef NDEBUG
+    EXPECT_EQ(NewI->getName(), "NewCopyFrom1");
+#endif // NDEBUG
+    EXPECT_EQ(NewI->getNextNode(), Ret);
+  }
+  {
+    // Check createWithCopiedFlags() InsertBefore,
+    auto *NewI = cast<sandboxir::UnaryOperator>(
+        sandboxir::UnaryOperator::createWithCopiedFlags(
+            sandboxir::Instruction::Opcode::FNeg, Arg0, CopyFrom,
+            /*InsertBefore=*/Ret, Ctx, "NewCopyFrom2"));
+    EXPECT_EQ(NewI->hasAllowReassoc(), CopyFrom->hasAllowReassoc());
+    EXPECT_EQ(NewI->getOpcode(), sandboxir::Instruction::Opcode::FNeg);
+    EXPECT_EQ(NewI->getOperand(0), Arg0);
+#ifndef NDEBUG
+    EXPECT_EQ(NewI->getName(), "NewCopyFrom2");
+#endif // NDEBUG
+    EXPECT_EQ(NewI->getNextNode(), Ret);
+  }
+  {
+    // Check createWithCopiedFlags() InsertAtEnd,
+    auto *NewI = cast<sandboxir::UnaryOperator>(
+        sandboxir::UnaryOperator::createWithCopiedFlags(
+            sandboxir::Instruction::Opcode::FNeg, Arg0, CopyFrom,
+            /*InsertAtEnd=*/BB, Ctx, "NewCopyFrom3"));
+    EXPECT_EQ(NewI->hasAllowReassoc(), CopyFrom->hasAllowReassoc());
+    EXPECT_EQ(NewI->getOpcode(), sandboxir::Instruction::Opcode::FNeg);
+    EXPECT_EQ(NewI->getOperand(0), Arg0);
+#ifndef NDEBUG
+    EXPECT_EQ(NewI->getName(), "NewCopyFrom3");
+#endif // NDEBUG
+    EXPECT_EQ(NewI->getParent(), BB);
+    EXPECT_EQ(NewI->getNextNode(), nullptr);
+  }
+  {
+    // Check createWithCopiedFlags() when it gets folded.
+    auto *FortyTwo = CopyFrom->getOperand(1);
+    auto *NewV = sandboxir::UnaryOperator::createWithCopiedFlags(
+        sandboxir::Instruction::Opcode::FNeg, FortyTwo, CopyFrom,
+        /*InsertAtEnd=*/BB, Ctx, "Folded");
+    EXPECT_TRUE(isa<sandboxir::Constant>(NewV));
+  }
+}
+
 TEST_F(SandboxIRTest, BinaryOperator) {
   parseIR(C, R"IR(
 define void @foo(i8 %arg0, i8 %arg1, float %farg0, float %farg1) {

From 9d63a09b452b641e3cc5d88066464b8250bd2bf7 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Thu, 15 Aug 2024 17:04:41 -0700
Subject: [PATCH 102/441] [Driver] Improve error message for -Wa,-x=unknown

---
 clang/lib/Driver/ToolChains/Clang.cpp          | 13 +++++++++----
 clang/test/Driver/arm-target-as-mimplicit-it.s |  5 ++---
 clang/test/Driver/relax.c                      |  2 +-
 3 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index c26ebd1ba56bab6..2b7241964a345da 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -2621,6 +2621,11 @@ static void CollectArgsForIntegratedAssembler(Compilation &C,
         continue; // LLVM handles bigobj automatically
 
       auto Equal = Value.split('=');
+      auto checkArg = [&](std::initializer_list<const char *> Set) {
+        if (!llvm::is_contained(Set, Equal.second))
+          D.Diag(diag::err_drv_unsupported_option_argument)
+              << (Twine("-Wa,") + Equal.first + "=").str() << Equal.second;
+      };
       switch (C.getDefaultToolChain().getArch()) {
       default:
         break;
@@ -2629,8 +2634,8 @@ static void CollectArgsForIntegratedAssembler(Compilation &C,
         if (Equal.first == "-mrelax-relocations" ||
             Equal.first == "--mrelax-relocations") {
           UseRelaxRelocations = Equal.second == "yes";
-          if (llvm::is_contained({"yes", "no"}, Equal.second))
-            continue;
+          checkArg({"yes", "no"});
+          continue;
         }
         if (Value == "-msse2avx") {
           CmdArgs.push_back("-msse2avx");
@@ -2651,8 +2656,8 @@ static void CollectArgsForIntegratedAssembler(Compilation &C,
         if (Equal.first == "-mimplicit-it") {
           // Only store the value; the last value set takes effect.
           ImplicitIt = Equal.second;
-          if (CheckARMImplicitITArg(Equal.second))
-            continue;
+          checkArg({"always", "never", "arm", "thumb"});
+          continue;
         }
         if (Value == "-mthumb")
           // -mthumb has already been processed in ComputeLLVMTriple()
diff --git a/clang/test/Driver/arm-target-as-mimplicit-it.s b/clang/test/Driver/arm-target-as-mimplicit-it.s
index d30433ab374d598..e40206dfc3a2938 100644
--- a/clang/test/Driver/arm-target-as-mimplicit-it.s
+++ b/clang/test/Driver/arm-target-as-mimplicit-it.s
@@ -30,7 +30,7 @@
 
 /// Test invalid input.
 // RUN: not %clang -target arm-linux-gnueabi -### -Wa,-mimplicit-it=foo %s 2>&1 | FileCheck %s --check-prefix=INVALID
-// RUN: not %clang -target arm-linux-gnueabi -### -Xassembler -mimplicit-it=foo %s 2>&1 | FileCheck %s --check-prefix=XINVALID
+// RUN: not %clang -target arm-linux-gnueabi -### -Xassembler -mimplicit-it=foo %s 2>&1 | FileCheck %s --check-prefix=INVALID
 // RUN: not %clang -target arm-linux-gnueabi -### -Wa,-mimplicit-it=always -Wa,-mimplicit-it=foo %s 2>&1 | FileCheck %s --check-prefix=INVALID
 // RUN: not %clang -target arm-linux-gnueabi -### -Wa,-mimplicit-it=always,-mimplicit-it=foo %s 2>&1 | FileCheck %s --check-prefix=INVALID
 
@@ -47,5 +47,4 @@
 // NEVER-NOT: "-arm-implicit-it={{.*}}"
 // ARM: "-mllvm" "-arm-implicit-it=arm"
 // THUMB: "-mllvm" "-arm-implicit-it=thumb"
-// INVALID: error: unsupported argument '-mimplicit-it=foo' to option '-Wa,'
-// XINVALID: error: unsupported argument '-mimplicit-it=foo' to option '-Xassembler'
+// INVALID: error: unsupported argument 'foo' to option '-Wa,-mimplicit-it='
diff --git a/clang/test/Driver/relax.c b/clang/test/Driver/relax.c
index 4771b9e3af11163..9315a0045f1f438 100644
--- a/clang/test/Driver/relax.c
+++ b/clang/test/Driver/relax.c
@@ -4,7 +4,7 @@
 // CHECK: "-mrelax-relocations=no"
 
 // RUN: not %clang -### --target=x86_64 -c -Wa,-mrelax-relocations=x %s 2>&1 | FileCheck %s --check-prefix=ERR
-// ERR: error: unsupported argument '-mrelax-relocations=x' to option '-Wa,'
+// ERR: error: unsupported argument 'x' to option '-Wa,-mrelax-relocations='
 
 // RUN: not %clang -### --target=aarch64 -c -Wa,-mrelax-relocations=no %s 2>&1 | FileCheck %s --check-prefix=ERR2
 // ERR2: error: unsupported argument '-mrelax-relocations=no' to option '-Wa,'

From 84efc8ed463cd0ae90ecd02418442cb62b9a3ad2 Mon Sep 17 00:00:00 2001
From: Tom Stellard <tstellar@redhat.com>
Date: Thu, 15 Aug 2024 17:08:24 -0700
Subject: [PATCH 103/441] [workflows] Add a new workflow for checking commit
 access qualifications (#93301)

This workflow produces a list of users that should be moved to the LLVM
Triagers team. An admin will review the list and then manually move
people to the LLVM Triagers Team.

I've also added some documentation to the Developer Policy about the
Triage Role.

See
https://discourse.llvm.org/t/rfc2-new-criteria-for-commit-access/77110
for more details.
---
 .github/workflows/commit-access-review.py  | 418 +++++++++++++++++++++
 .github/workflows/commit-access-review.yml |  34 ++
 2 files changed, 452 insertions(+)
 create mode 100644 .github/workflows/commit-access-review.py
 create mode 100644 .github/workflows/commit-access-review.yml

diff --git a/.github/workflows/commit-access-review.py b/.github/workflows/commit-access-review.py
new file mode 100644
index 000000000000000..97589c138c0b707
--- /dev/null
+++ b/.github/workflows/commit-access-review.py
@@ -0,0 +1,418 @@
+#!/usr/bin/env python3
+# ===-- commit-access-review.py  --------------------------------------------===#
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# ===------------------------------------------------------------------------===#
+#
+# ===------------------------------------------------------------------------===#
+
+import datetime
+import github
+import re
+import requests
+import time
+import sys
+import re
+
+
+class User:
+    THRESHOLD = 5
+
+    def __init__(self, name, triage_list):
+        self.name = name
+        self.authored = 0
+        self.merged = 0
+        self.reviewed = 0
+        self.triage_list = triage_list
+
+    def add_authored(self, val=1):
+        self.authored += val
+        if self.meets_threshold():
+            print(self.name, "meets the threshold with authored commits")
+            del self.triage_list[self.name]
+
+    def set_authored(self, val):
+        self.authored = 0
+        self.add_authored(val)
+
+    def add_merged(self, val=1):
+        self.merged += val
+        if self.meets_threshold():
+            print(self.name, "meets the threshold with merged commits")
+            del self.triage_list[self.name]
+
+    def add_reviewed(self, val=1):
+        self.reviewed += val
+        if self.meets_threshold():
+            print(self.name, "meets the threshold with reviewed commits")
+            del self.triage_list[self.name]
+
+    def get_total(self):
+        return self.authored + self.merged + self.reviewed
+
+    def meets_threshold(self):
+        return self.get_total() >= self.THRESHOLD
+
+    def __repr__(self):
+        return "{} : a: {} m: {} r: {}".format(
+            self.name, self.authored, self.merged, self.reviewed
+        )
+
+
+def run_graphql_query(
+    query: str, variables: dict, token: str, retry: bool = True
+) -> dict:
+    """
+    This function submits a graphql query and returns the results as a
+    dictionary.
+    """
+    s = requests.Session()
+    retries = requests.adapters.Retry(total=8, backoff_factor=2, status_forcelist=[504])
+    s.mount("https://", requests.adapters.HTTPAdapter(max_retries=retries))
+
+    headers = {
+        "Authorization": "bearer {}".format(token),
+        # See
+        # https://github.blog/2021-11-16-graphql-global-id-migration-update/
+        "X-Github-Next-Global-ID": "1",
+    }
+    request = s.post(
+        url="https://api.github.com/graphql",
+        json={"query": query, "variables": variables},
+        headers=headers,
+    )
+
+    rate_limit = request.headers.get("X-RateLimit-Remaining")
+    print(rate_limit)
+    if rate_limit and int(rate_limit) < 10:
+        reset_time = int(request.headers["X-RateLimit-Reset"])
+        while reset_time - int(time.time()) > 0:
+            time.sleep(60)
+            print(
+                "Waiting until rate limit reset",
+                reset_time - int(time.time()),
+                "seconds remaining",
+            )
+
+    if request.status_code == 200:
+        if "data" not in request.json():
+            print(request.json())
+            sys.exit(1)
+        return request.json()["data"]
+    elif retry:
+        return run_graphql_query(query, variables, token, False)
+    else:
+        raise Exception(
+            "Failed to run graphql query\nquery: {}\nerror: {}".format(
+                query, request.json()
+            )
+        )
+
+
+def check_manual_requests(start_date: datetime.datetime, token: str) -> list[str]:
+    """
+    Return a list of users who have been asked since ``start_date`` if they
+    want to keep their commit access.
+    """
+    query = """
+        query ($query: String!) {
+          search(query: $query, type: ISSUE, first: 100) {
+            nodes {
+              ... on Issue {
+                body
+                comments (first: 100) {
+                  nodes {
+                    author {
+                      login
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+        """
+    formatted_start_date = start_date.strftime("%Y-%m-%dT%H:%M:%S")
+    variables = {
+        "query": f"type:issue created:>{formatted_start_date} org:llvm repo:llvm-project label:infrastructure:commit-access"
+    }
+
+    data = run_graphql_query(query, variables, token)
+    users = []
+    for issue in data["search"]["nodes"]:
+        users.extend([user[1:] for user in re.findall("@[^ ,\n]+", issue["body"])])
+
+    return users
+
+
+def get_num_commits(user: str, start_date: datetime.datetime, token: str) -> int:
+    """
+    Get number of commits that ``user`` has been made since ``start_date`.
+    """
+    variables = {
+        "owner": "llvm",
+        "user": user,
+        "start_date": start_date.strftime("%Y-%m-%dT%H:%M:%S"),
+    }
+
+    user_query = """
+        query ($user: String!) {
+          user(login: $user) {
+            id
+          }
+        }
+    """
+
+    data = run_graphql_query(user_query, variables, token)
+    variables["user_id"] = data["user"]["id"]
+
+    query = """
+        query ($owner: String!, $user_id: ID!, $start_date: GitTimestamp!){
+          organization(login: $owner) {
+            teams(query: "llvm-committers" first:1) {
+              nodes {
+                repositories {
+                  nodes {
+                    ref(qualifiedName: "main") {
+                      target {
+                        ... on Commit {
+                          history(since: $start_date, author: {id: $user_id }) {
+                            totalCount
+                          }
+                        }
+                      }
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+     """
+    count = 0
+    data = run_graphql_query(query, variables, token)
+    for repo in data["organization"]["teams"]["nodes"][0]["repositories"]["nodes"]:
+        count += int(repo["ref"]["target"]["history"]["totalCount"])
+        if count >= User.THRESHOLD:
+            break
+    return count
+
+
+def is_new_committer_query_repo(
+    user: str, start_date: datetime.datetime, token: str
+) -> bool:
+    """
+    Determine if ``user`` is a new committer.  A new committer can keep their
+    commit access even if they don't meet the criteria.
+    """
+    variables = {
+        "user": user,
+    }
+
+    user_query = """
+        query ($user: String!) {
+          user(login: $user) {
+            id
+          }
+        }
+    """
+
+    data = run_graphql_query(user_query, variables, token)
+    variables["owner"] = "llvm"
+    variables["user_id"] = data["user"]["id"]
+    variables["start_date"] = start_date.strftime("%Y-%m-%dT%H:%M:%S")
+
+    query = """
+        query ($owner: String!, $user_id: ID!){
+          organization(login: $owner) {
+            repository(name: "llvm-project") {
+              ref(qualifiedName: "main") {
+                target {
+                  ... on Commit {
+                    history(author: {id: $user_id }, first: 5) {
+                      nodes {
+                        committedDate
+                      }
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+     """
+
+    data = run_graphql_query(query, variables, token)
+    repo = data["organization"]["repository"]
+    commits = repo["ref"]["target"]["history"]["nodes"]
+    if len(commits) == 0:
+        return True
+    committed_date = commits[-1]["committedDate"]
+    if datetime.datetime.strptime(committed_date, "%Y-%m-%dT%H:%M:%SZ") < start_date:
+        return False
+    return True
+
+
+def is_new_committer(user: str, start_date: datetime.datetime, token: str) -> bool:
+    """
+    Wrapper around is_new_commiter_query_repo to handle exceptions.
+    """
+    try:
+        return is_new_committer_query_repo(user, start_date, token)
+    except:
+        pass
+    return True
+
+
+def get_review_count(user: str, start_date: datetime.datetime, token: str) -> int:
+    """
+    Return the number of reviews that ``user`` has done since ``start_date``.
+    """
+    query = """
+        query ($query: String!) {
+          search(query: $query, type: ISSUE, first: 5) {
+            issueCount
+          }
+        }
+        """
+    formatted_start_date = start_date.strftime("%Y-%m-%dT%H:%M:%S")
+    variables = {
+        "owner": "llvm",
+        "repo": "llvm-project",
+        "user": user,
+        "query": f"type:pr commenter:{user} -author:{user} merged:>{formatted_start_date} org:llvm",
+    }
+
+    data = run_graphql_query(query, variables, token)
+    return int(data["search"]["issueCount"])
+
+
+def count_prs(triage_list: dict, start_date: datetime.datetime, token: str):
+    """
+    Fetch all the merged PRs for the project since ``start_date`` and update
+    ``triage_list`` with the number of PRs merged for each user.
+    """
+
+    query = """
+        query ($query: String!, $after: String) {
+          search(query: $query, type: ISSUE, first: 100, after: $after) {
+            issueCount,
+            nodes {
+              ... on PullRequest {
+                 author {
+                   login
+                 }
+                 mergedBy {
+                   login
+                 }
+              }
+            }
+            pageInfo {
+              hasNextPage
+              endCursor
+            }
+          }
+        }
+    """
+    date_begin = start_date
+    date_end = None
+    while date_begin < datetime.datetime.now():
+        date_end = date_begin + datetime.timedelta(days=7)
+        formatted_date_begin = date_begin.strftime("%Y-%m-%dT%H:%M:%S")
+        formatted_date_end = date_end.strftime("%Y-%m-%dT%H:%M:%S")
+        variables = {
+            "query": f"type:pr is:merged merged:{formatted_date_begin}..{formatted_date_end} org:llvm",
+        }
+        has_next_page = True
+        while has_next_page:
+            print(variables)
+            data = run_graphql_query(query, variables, token)
+            for pr in data["search"]["nodes"]:
+                # Users can be None if the user has been deleted.
+                if not pr["author"]:
+                    continue
+                author = pr["author"]["login"]
+                if author in triage_list:
+                    triage_list[author].add_authored()
+
+                if not pr["mergedBy"]:
+                    continue
+                merger = pr["mergedBy"]["login"]
+                if author == merger:
+                    continue
+                if merger not in triage_list:
+                    continue
+                triage_list[merger].add_merged()
+
+            has_next_page = data["search"]["pageInfo"]["hasNextPage"]
+            if has_next_page:
+                variables["after"] = data["search"]["pageInfo"]["endCursor"]
+        date_begin = date_end
+
+
+def main():
+    token = sys.argv[1]
+    gh = github.Github(login_or_token=token)
+    org = gh.get_organization("llvm")
+    repo = org.get_repo("llvm-project")
+    team = org.get_team_by_slug("llvm-committers")
+    one_year_ago = datetime.datetime.now() - datetime.timedelta(days=365)
+    triage_list = {}
+    for member in team.get_members():
+        triage_list[member.login] = User(member.login, triage_list)
+
+    print("Start:", len(triage_list), "triagers")
+    # Step 0 Check if users have requested commit access in the last year.
+    for user in check_manual_requests(one_year_ago, token):
+        if user in triage_list:
+            print(user, "requested commit access in the last year.")
+            del triage_list[user]
+    print("After Request Check:", len(triage_list), "triagers")
+
+    # Step 1 count all PRs authored or merged
+    count_prs(triage_list, one_year_ago, token)
+
+    print("After PRs:", len(triage_list), "triagers")
+
+    if len(triage_list) == 0:
+        sys.exit(0)
+
+    # Step 2 check for reviews
+    for user in list(triage_list.keys()):
+        review_count = get_review_count(user, one_year_ago, token)
+        triage_list[user].add_reviewed(review_count)
+
+    print("After Reviews:", len(triage_list), "triagers")
+
+    if len(triage_list) == 0:
+        sys.exit(0)
+
+    # Step 3 check for number of commits
+    for user in list(triage_list.keys()):
+        num_commits = get_num_commits(user, one_year_ago, token)
+        # Override the total number of commits to not double count commits and
+        # authored PRs.
+        triage_list[user].set_authored(num_commits)
+
+    print("After Commits:", len(triage_list), "triagers")
+
+    # Step 4 check for new committers
+    for user in list(triage_list.keys()):
+        print("Checking", user)
+        if is_new_committer(user, one_year_ago, token):
+            print("Removing new committer: ", user)
+            del triage_list[user]
+
+    print("Complete:", len(triage_list), "triagers")
+
+    with open("triagers.log", "w") as triagers_log:
+        for user in triage_list:
+            print(triage_list[user].__repr__())
+            triagers_log.write(user + "\n")
+
+
+if __name__ == "__main__":
+    main()
diff --git a/.github/workflows/commit-access-review.yml b/.github/workflows/commit-access-review.yml
new file mode 100644
index 000000000000000..f9195a1863deec9
--- /dev/null
+++ b/.github/workflows/commit-access-review.yml
@@ -0,0 +1,34 @@
+name: Commit Access Review
+
+on:
+  workflow_dispatch:
+  schedule:
+    # * is a special character in YAML so you have to quote this string
+    - cron:  '0 7 1 * *'
+
+permissions:
+  contents: read
+
+jobs:
+  commit-access-review:
+    if: github.repository_owner == 'llvm'
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Fetch LLVM sources
+        uses: actions/checkout@v4
+      
+      - name: Install dependencies
+        run: |
+          pip install --require-hashes -r ./llvm/utils/git/requirements.txt
+      
+      - name: Run Script
+        env:
+          GITHUB_TOKEN: ${{ secrets.RELEASE_TASKS_USER_TOKEN }}
+        run: |
+          python3 .github/workflows/commit-access-review.py $GITHUB_TOKEN
+
+      - name: Upload Triage List
+        uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 #v4.3.0
+        with:
+          name: triagers
+          path: triagers.log

From 68f6e7467651f38e0b97343bfbc49e0ce69eaedf Mon Sep 17 00:00:00 2001
From: Vitaly Buka <vitalybuka@google.com>
Date: Thu, 15 Aug 2024 17:14:14 -0700
Subject: [PATCH 104/441] [asan] Remove debug tracing from `report_globals`
 (#104404)

Printing globals registration is internal debug
tracing and should be controlled with verbosity.
---
 compiler-rt/lib/asan/asan_flags.inc           |  7 ++-----
 compiler-rt/lib/asan/asan_globals.cpp         | 19 ++++++++-----------
 .../Linux/initialization-nobug-lld.cpp        |  2 +-
 .../Linux/odr_indicator_unregister.cpp        |  2 +-
 .../asan/TestCases/Linux/odr_indicators.cpp   |  4 ++--
 .../asan/TestCases/initialization-nobug.cpp   |  8 ++++----
 6 files changed, 18 insertions(+), 24 deletions(-)

diff --git a/compiler-rt/lib/asan/asan_flags.inc b/compiler-rt/lib/asan/asan_flags.inc
index fad1577d912a5e1..5e0ced9706e6647 100644
--- a/compiler-rt/lib/asan/asan_flags.inc
+++ b/compiler-rt/lib/asan/asan_flags.inc
@@ -36,11 +36,8 @@ ASAN_FLAG(int, max_redzone, 2048,
 ASAN_FLAG(
     bool, debug, false,
     "If set, prints some debugging information and does additional checks.")
-ASAN_FLAG(
-    int, report_globals, 1,
-    "Controls the way to handle globals (0 - don't detect buffer overflow on "
-    "globals, 1 - detect buffer overflow, 2 - print data about registered "
-    "globals).")
+ASAN_FLAG(bool, report_globals, true,
+          "If set, detect and report errors on globals .")
 ASAN_FLAG(bool, check_initialization_order, false,
           "If set, attempts to catch initialization order issues.")
 ASAN_FLAG(
diff --git a/compiler-rt/lib/asan/asan_globals.cpp b/compiler-rt/lib/asan/asan_globals.cpp
index c83b782cb85f896..a1211430b1268a0 100644
--- a/compiler-rt/lib/asan/asan_globals.cpp
+++ b/compiler-rt/lib/asan/asan_globals.cpp
@@ -22,6 +22,7 @@
 #include "asan_thread.h"
 #include "sanitizer_common/sanitizer_common.h"
 #include "sanitizer_common/sanitizer_dense_map.h"
+#include "sanitizer_common/sanitizer_internal_defs.h"
 #include "sanitizer_common/sanitizer_list.h"
 #include "sanitizer_common/sanitizer_mutex.h"
 #include "sanitizer_common/sanitizer_placement_new.h"
@@ -179,7 +180,7 @@ int GetGlobalsForAddress(uptr addr, Global *globals, u32 *reg_sites,
   int res = 0;
   for (const auto &l : list_of_all_globals) {
     const Global &g = *l.g;
-    if (flags()->report_globals >= 2)
+    if (UNLIKELY(common_flags()->verbosity >= 3))
       ReportGlobal(g, "Search");
     if (IsAddressNearGlobal(addr, g)) {
       internal_memcpy(&globals[res], &g, sizeof(g));
@@ -270,7 +271,7 @@ static inline bool UseODRIndicator(const Global *g) {
 // so we store the globals in a map.
 static void RegisterGlobal(const Global *g) SANITIZER_REQUIRES(mu_for_globals) {
   CHECK(AsanInited());
-  if (flags()->report_globals >= 2)
+  if (UNLIKELY(common_flags()->verbosity >= 3))
     ReportGlobal(*g, "Added");
   CHECK(flags()->report_globals);
   CHECK(AddrIsInMem(g->beg));
@@ -307,7 +308,7 @@ static void RegisterGlobal(const Global *g) SANITIZER_REQUIRES(mu_for_globals) {
 static void UnregisterGlobal(const Global *g)
     SANITIZER_REQUIRES(mu_for_globals) {
   CHECK(AsanInited());
-  if (flags()->report_globals >= 2)
+  if (UNLIKELY(common_flags()->verbosity >= 3))
     ReportGlobal(*g, "Removed");
   CHECK(flags()->report_globals);
   CHECK(AddrIsInMem(g->beg));
@@ -438,7 +439,7 @@ void __asan_register_globals(__asan_global *globals, uptr n) {
   }
   GlobalRegistrationSite site = {stack_id, &globals[0], &globals[n - 1]};
   global_registration_site_vector->push_back(site);
-  if (flags()->report_globals >= 2) {
+  if (UNLIKELY(common_flags()->verbosity >= 3)) {
     PRINT_CURRENT_STACK();
     Printf("=== ID %d; %p %p\n", stack_id, (void *)&globals[0],
            (void *)&globals[n - 1]);
@@ -497,9 +498,7 @@ void __asan_before_dynamic_init(const char *module_name) {
   Lock lock(&mu_for_globals);
   if (current_dynamic_init_module_name == module_name)
     return;
-  if (flags()->report_globals >= 3)
-    Printf("DynInitPoison module: %s\n", module_name);
-
+  VPrintf(2, "DynInitPoison module: %s\n", module_name);
   if (current_dynamic_init_module_name == nullptr) {
     // First call, poison all globals from other modules.
     DynInitGlobals().forEach([&](auto &kv) {
@@ -545,8 +544,7 @@ static void UnpoisonBeforeMain(void) {
       return;
     allow_after_dynamic_init = true;
   }
-  if (flags()->report_globals >= 3)
-    Printf("UnpoisonBeforeMain\n");
+  VPrintf(2, "UnpoisonBeforeMain\n");
   __asan_after_dynamic_init();
 }
 
@@ -570,8 +568,7 @@ void __asan_after_dynamic_init() {
   if (!current_dynamic_init_module_name)
     return;
 
-  if (flags()->report_globals >= 3)
-    Printf("DynInitUnpoison\n");
+  VPrintf(2, "DynInitUnpoison\n");
 
   DynInitGlobals().forEach([&](auto &kv) {
     UnpoisonDynamicGlobals(kv.second, /*mark_initialized=*/false);
diff --git a/compiler-rt/test/asan/TestCases/Linux/initialization-nobug-lld.cpp b/compiler-rt/test/asan/TestCases/Linux/initialization-nobug-lld.cpp
index 5cec029811cbc84..ef82c7a29575ebd 100644
--- a/compiler-rt/test/asan/TestCases/Linux/initialization-nobug-lld.cpp
+++ b/compiler-rt/test/asan/TestCases/Linux/initialization-nobug-lld.cpp
@@ -1,4 +1,4 @@
-// RUN: %clangxx_asan -O3 %S/../initialization-nobug.cpp %S/../Helpers/initialization-nobug-extra.cpp -fuse-ld=lld -o %t && %env_asan_opts=check_initialization_order=true:report_globals=3 %run %t 2>&1 | FileCheck %s --implicit-check-not "DynInit"
+// RUN: %clangxx_asan -O3 %S/../initialization-nobug.cpp %S/../Helpers/initialization-nobug-extra.cpp -fuse-ld=lld -o %t && %env_asan_opts=check_initialization_order=true:report_globals=1:verbosity=2 %run %t 2>&1 | FileCheck %s --implicit-check-not "DynInit"
 
 // Same as initialization-nobug.cpp, but with lld we expect just one
 // `DynInitUnpoison` executed after `AfterDynamicInit` at the end.
diff --git a/compiler-rt/test/asan/TestCases/Linux/odr_indicator_unregister.cpp b/compiler-rt/test/asan/TestCases/Linux/odr_indicator_unregister.cpp
index 0f2ed6597154bba..b75f5be101ef8a4 100644
--- a/compiler-rt/test/asan/TestCases/Linux/odr_indicator_unregister.cpp
+++ b/compiler-rt/test/asan/TestCases/Linux/odr_indicator_unregister.cpp
@@ -4,7 +4,7 @@
 // RUN: %clangxx_asan -g -O0 -DSHARED_LIB -DSIZE=1 %s -fPIC -shared -o %t-so-1.so
 // RUN: %clangxx_asan -g -O0 -DSHARED_LIB -DSIZE=2 %s -fPIC -shared -o %t-so-2.so
 // RUN: %clangxx_asan -g -O0 %s %libdl -Wl,--export-dynamic -o %t
-// RUN: %env_asan_opts=report_globals=2:detect_odr_violation=1 %run %t 2>&1 | FileCheck %s
+// RUN: %env_asan_opts=report_globals=1:detect_odr_violation=1:verbosity=3 %run %t 2>&1 | FileCheck %s
 
 // FIXME: Checks do not match on Android.
 // UNSUPPORTED: android
diff --git a/compiler-rt/test/asan/TestCases/Linux/odr_indicators.cpp b/compiler-rt/test/asan/TestCases/Linux/odr_indicators.cpp
index 8af3ec09be78c4f..f28a9f6d07386d5 100644
--- a/compiler-rt/test/asan/TestCases/Linux/odr_indicators.cpp
+++ b/compiler-rt/test/asan/TestCases/Linux/odr_indicators.cpp
@@ -1,8 +1,8 @@
 // RUN: %clangxx_asan -fno-sanitize-address-use-odr-indicator -fPIC %s -o %t
-// RUN: %env_asan_opts=report_globals=2 %run %t 2>&1 | FileCheck %s --check-prefixes=CHECK,INDICATOR0
+// RUN: %env_asan_opts=report_globals=1:verbosity=3 %run %t 2>&1 | FileCheck %s --check-prefixes=CHECK,INDICATOR0
 
 // RUN: %clangxx_asan -fsanitize-address-use-odr-indicator -fPIC %s -o %t
-// RUN: %env_asan_opts=report_globals=2 %run %t 2>&1 | FileCheck %s --check-prefixes=CHECK,INDICATOR1
+// RUN: %env_asan_opts=report_globals=1:verbosity=3 %run %t 2>&1 | FileCheck %s --check-prefixes=CHECK,INDICATOR1
 
 #include <stdio.h>
 
diff --git a/compiler-rt/test/asan/TestCases/initialization-nobug.cpp b/compiler-rt/test/asan/TestCases/initialization-nobug.cpp
index f66d501124bc48b..61328b9de28ae6c 100644
--- a/compiler-rt/test/asan/TestCases/initialization-nobug.cpp
+++ b/compiler-rt/test/asan/TestCases/initialization-nobug.cpp
@@ -1,10 +1,10 @@
 // A collection of various initializers which shouldn't trip up initialization
 // order checking.  If successful, this will just return 0.
 
-// RUN: %clangxx_asan -O0 %s %p/Helpers/initialization-nobug-extra.cpp -o %t && %env_asan_opts=check_initialization_order=true:report_globals=3 %run %t 2>&1 | FileCheck %s --implicit-check-not "DynInitPoison"
-// RUN: %clangxx_asan -O1 %s %p/Helpers/initialization-nobug-extra.cpp -o %t && %env_asan_opts=check_initialization_order=true:report_globals=3 %run %t 2>&1 | FileCheck %s --implicit-check-not "DynInitPoison"
-// RUN: %clangxx_asan -O2 %s %p/Helpers/initialization-nobug-extra.cpp -o %t && %env_asan_opts=check_initialization_order=true:report_globals=3 %run %t 2>&1 | FileCheck %s --implicit-check-not "DynInitPoison"
-// RUN: %clangxx_asan -O3 %s %p/Helpers/initialization-nobug-extra.cpp -o %t && %env_asan_opts=check_initialization_order=true:report_globals=3 %run %t 2>&1 | FileCheck %s --implicit-check-not "DynInitPoison"
+// RUN: %clangxx_asan -O0 %s %p/Helpers/initialization-nobug-extra.cpp -o %t && %env_asan_opts=check_initialization_order=true:report_globals=1:verbosity=2 %run %t 2>&1 | FileCheck %s --implicit-check-not "DynInitPoison"
+// RUN: %clangxx_asan -O1 %s %p/Helpers/initialization-nobug-extra.cpp -o %t && %env_asan_opts=check_initialization_order=true:report_globals=1:verbosity=2 %run %t 2>&1 | FileCheck %s --implicit-check-not "DynInitPoison"
+// RUN: %clangxx_asan -O2 %s %p/Helpers/initialization-nobug-extra.cpp -o %t && %env_asan_opts=check_initialization_order=true:report_globals=1:verbosity=2 %run %t 2>&1 | FileCheck %s --implicit-check-not "DynInitPoison"
+// RUN: %clangxx_asan -O3 %s %p/Helpers/initialization-nobug-extra.cpp -o %t && %env_asan_opts=check_initialization_order=true:report_globals=1:verbosity=2 %run %t 2>&1 | FileCheck %s --implicit-check-not "DynInitPoison"
 
 // Simple access:
 // Make sure that accessing a global in the same TU is safe

From 4695c16a62261ce85ba28098f402ebcfb72798e9 Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Fri, 16 Aug 2024 00:17:51 +0000
Subject: [PATCH 105/441] [gn build] Port 3e1d4ec671c5

---
 llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/BUILD.gn | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/BUILD.gn
index 31f567d6df159d4..b5cd80034ef05e1 100644
--- a/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/BUILD.gn
@@ -39,7 +39,9 @@ static_library("Orc") {
     "LLJIT.cpp",
     "Layer.cpp",
     "LazyReexports.cpp",
+    "LoadRelocatableObject.cpp",
     "LookupAndRecordAddrs.cpp",
+    "MachO.cpp",
     "MachOPlatform.cpp",
     "Mangling.cpp",
     "MapperJITLinkMemoryManager.cpp",

From 26670e7fa4f032a019d23d56c6a02926e854e8af Mon Sep 17 00:00:00 2001
From: royitaqi <royitaqi@users.noreply.github.com>
Date: Thu, 15 Aug 2024 17:24:39 -0700
Subject: [PATCH 106/441] [lldb] Correctly fix a usage of `PATH_MAX`, and fix
 unit tests (#104502)

# Part 1: Correctly fix a usage of `PATH_MAX`

TL;DR: Adding a typedef `lldb_private::PathSmallString` which contains a
hardcoded initial size (128).

# Part 2: Fix unit tests

After https://github.com/llvm/llvm-project/pull/104493 fixed the build
break for Windows, unit test failure showed up for Windows. The
root-cause is that the `FileSpec`'s in the unit tests are not
style-specific. The fix is to apply either `WindowsSpec` or `PosixSpec`
specifically.
---
 lldb/include/lldb/lldb-private-types.h        |  3 +
 lldb/source/Utility/RealpathPrefixes.cpp      |  4 +-
 lldb/unittests/Utility/FileSpecListTest.cpp   | 73 ++++++++++---------
 .../Utility/RealpathPrefixesTest.cpp          | 71 ++++++++++--------
 4 files changed, 87 insertions(+), 64 deletions(-)

diff --git a/lldb/include/lldb/lldb-private-types.h b/lldb/include/lldb/lldb-private-types.h
index 055eea9f6456d6b..b82a2b8aa057443 100644
--- a/lldb/include/lldb/lldb-private-types.h
+++ b/lldb/include/lldb/lldb-private-types.h
@@ -12,6 +12,7 @@
 #include "lldb/lldb-private.h"
 
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/SmallString.h"
 
 #include <type_traits>
 
@@ -26,6 +27,8 @@ class Platform;
 class ExecutionContext;
 class RegisterFlags;
 
+typedef llvm::SmallString<256> PathSmallString;
+
 typedef llvm::sys::DynamicLibrary (*LoadPluginCallbackType)(
     const lldb::DebuggerSP &debugger_sp, const FileSpec &spec, Status &error);
 
diff --git a/lldb/source/Utility/RealpathPrefixes.cpp b/lldb/source/Utility/RealpathPrefixes.cpp
index ee1c404c8996764..f6e1d17e30c6960 100644
--- a/lldb/source/Utility/RealpathPrefixes.cpp
+++ b/lldb/source/Utility/RealpathPrefixes.cpp
@@ -8,11 +8,11 @@
 
 #include "lldb/Utility/RealpathPrefixes.h"
 
-#include "lldb/Host/PosixApi.h"
 #include "lldb/Utility/FileSpec.h"
 #include "lldb/Utility/FileSpecList.h"
 #include "lldb/Utility/LLDBLog.h"
 #include "lldb/Utility/Log.h"
+#include "lldb/lldb-private-types.h"
 
 using namespace lldb_private;
 
@@ -54,7 +54,7 @@ RealpathPrefixes::ResolveSymlinks(const FileSpec &file_spec) {
       LLDB_LOGF(log, "Realpath'ing support file %s", file_spec_path.c_str());
 
       // One prefix matched. Try to realpath.
-      llvm::SmallString<PATH_MAX> buff;
+      PathSmallString buff;
       std::error_code ec = m_fs->getRealPath(file_spec_path, buff);
       if (ec)
         return std::nullopt;
diff --git a/lldb/unittests/Utility/FileSpecListTest.cpp b/lldb/unittests/Utility/FileSpecListTest.cpp
index 4c04f434261301a..d3f89ad0dfcb3c6 100644
--- a/lldb/unittests/Utility/FileSpecListTest.cpp
+++ b/lldb/unittests/Utility/FileSpecListTest.cpp
@@ -133,20 +133,21 @@ TEST(SupportFileListTest, RelativePathMatchesWindows) {
 TEST(SupportFileListTest, SymlinkedAbsolutePaths) {
   // Prepare FS
   llvm::IntrusiveRefCntPtr<MockSymlinkFileSystem> fs(new MockSymlinkFileSystem(
-      FileSpec("/symlink_dir/foo.h"), FileSpec("/real_dir/foo.h")));
+      PosixSpec("/symlink_dir/foo.h"), PosixSpec("/real_dir/foo.h"),
+      FileSpec::Style::posix));
 
   // Prepare RealpathPrefixes
   FileSpecList file_spec_list;
-  file_spec_list.EmplaceBack("/symlink_dir");
+  file_spec_list.Append(PosixSpec("/symlink_dir"));
   RealpathPrefixes prefixes(file_spec_list, fs);
 
   // Prepare support file list
   SupportFileList support_file_list;
-  support_file_list.EmplaceBack(FileSpec("/symlink_dir/foo.h"));
+  support_file_list.Append(PosixSpec("/symlink_dir/foo.h"));
 
   // Test
   size_t ret = support_file_list.FindCompatibleIndex(
-      0, FileSpec("/real_dir/foo.h"), &prefixes);
+      0, PosixSpec("/real_dir/foo.h"), &prefixes);
   EXPECT_EQ(ret, (size_t)0);
 }
 
@@ -157,20 +158,21 @@ TEST(SupportFileListTest, SymlinkedAbsolutePaths) {
 TEST(SupportFileListTest, RootDirectory) {
   // Prepare FS
   llvm::IntrusiveRefCntPtr<MockSymlinkFileSystem> fs(new MockSymlinkFileSystem(
-      FileSpec("/symlink_dir/foo.h"), FileSpec("/real_dir/foo.h")));
+      PosixSpec("/symlink_dir/foo.h"), PosixSpec("/real_dir/foo.h"),
+      FileSpec::Style::posix));
 
   // Prepare RealpathPrefixes
   FileSpecList file_spec_list;
-  file_spec_list.EmplaceBack("/");
+  file_spec_list.Append(PosixSpec("/"));
   RealpathPrefixes prefixes(file_spec_list, fs);
 
   // Prepare support file list
   SupportFileList support_file_list;
-  support_file_list.EmplaceBack(FileSpec("/symlink_dir/foo.h"));
+  support_file_list.Append(PosixSpec("/symlink_dir/foo.h"));
 
   // Test
   size_t ret = support_file_list.FindCompatibleIndex(
-      0, FileSpec("/real_dir/foo.h"), &prefixes);
+      0, PosixSpec("/real_dir/foo.h"), &prefixes);
   EXPECT_EQ(ret, (size_t)0);
 }
 
@@ -181,20 +183,21 @@ TEST(SupportFileListTest, RootDirectory) {
 TEST(SupportFileListTest, SymlinkedRelativePaths) {
   // Prepare FS
   llvm::IntrusiveRefCntPtr<MockSymlinkFileSystem> fs(new MockSymlinkFileSystem(
-      FileSpec("symlink_dir/foo.h"), FileSpec("real_dir/foo.h")));
+      PosixSpec("symlink_dir/foo.h"), PosixSpec("real_dir/foo.h"),
+      FileSpec::Style::posix));
 
   // Prepare RealpathPrefixes
   FileSpecList file_spec_list;
-  file_spec_list.EmplaceBack("symlink_dir");
+  file_spec_list.Append(PosixSpec("symlink_dir"));
   RealpathPrefixes prefixes(file_spec_list, fs);
 
   // Prepare support file list
   SupportFileList support_file_list;
-  support_file_list.EmplaceBack(FileSpec("symlink_dir/foo.h"));
+  support_file_list.Append(PosixSpec("symlink_dir/foo.h"));
 
   // Test
   size_t ret = support_file_list.FindCompatibleIndex(
-      0, FileSpec("real_dir/foo.h"), &prefixes);
+      0, PosixSpec("real_dir/foo.h"), &prefixes);
   EXPECT_EQ(ret, (size_t)0);
 }
 
@@ -205,20 +208,21 @@ TEST(SupportFileListTest, SymlinkedRelativePaths) {
 TEST(SupportFileListTest, RealpathOnlyMatchFileName) {
   // Prepare FS
   llvm::IntrusiveRefCntPtr<MockSymlinkFileSystem> fs(new MockSymlinkFileSystem(
-      FileSpec("symlink_dir/foo.h"), FileSpec("real_dir/foo.h")));
+      PosixSpec("symlink_dir/foo.h"), PosixSpec("real_dir/foo.h"),
+      FileSpec::Style::posix));
 
   // Prepare RealpathPrefixes
   FileSpecList file_spec_list;
-  file_spec_list.EmplaceBack("symlink_dir");
+  file_spec_list.Append(PosixSpec("symlink_dir"));
   RealpathPrefixes prefixes(file_spec_list, fs);
 
   // Prepare support file list
   SupportFileList support_file_list;
-  support_file_list.EmplaceBack(FileSpec("symlink_dir/foo.h"));
+  support_file_list.Append(PosixSpec("symlink_dir/foo.h"));
 
   // Test
   size_t ret = support_file_list.FindCompatibleIndex(
-      0, FileSpec("some_other_dir/foo.h"), &prefixes);
+      0, PosixSpec("some_other_dir/foo.h"), &prefixes);
   EXPECT_EQ(ret, UINT32_MAX);
 }
 
@@ -228,21 +232,22 @@ TEST(SupportFileListTest, RealpathOnlyMatchFileName) {
 TEST(SupportFileListTest, DirectoryMatchStringPrefixButNotWholeDirectoryName) {
   // Prepare FS
   llvm::IntrusiveRefCntPtr<MockSymlinkFileSystem> fs(new MockSymlinkFileSystem(
-      FileSpec("symlink_dir/foo.h"), FileSpec("real_dir/foo.h")));
+      PosixSpec("symlink_dir/foo.h"), PosixSpec("real_dir/foo.h"),
+      FileSpec::Style::posix));
 
   // Prepare RealpathPrefixes
   FileSpecList file_spec_list;
-  file_spec_list.EmplaceBack("symlink"); // This is a string prefix of the
-                                         // symlink but not a path prefix.
+  file_spec_list.Append(PosixSpec("symlink")); // This is a string prefix of the
+                                               // symlink but not a path prefix.
   RealpathPrefixes prefixes(file_spec_list, fs);
 
   // Prepare support file list
   SupportFileList support_file_list;
-  support_file_list.EmplaceBack(FileSpec("symlink_dir/foo.h"));
+  support_file_list.Append(PosixSpec("symlink_dir/foo.h"));
 
   // Test
   size_t ret = support_file_list.FindCompatibleIndex(
-      0, FileSpec("real_dir/foo.h"), &prefixes);
+      0, PosixSpec("real_dir/foo.h"), &prefixes);
   EXPECT_EQ(ret, UINT32_MAX);
 }
 
@@ -253,20 +258,21 @@ TEST(SupportFileListTest, DirectoryMatchStringPrefixButNotWholeDirectoryName) {
 TEST(SupportFileListTest, PartialBreakpointPath) {
   // Prepare FS
   llvm::IntrusiveRefCntPtr<MockSymlinkFileSystem> fs(new MockSymlinkFileSystem(
-      FileSpec("symlink_dir/foo.h"), FileSpec("/real_dir/foo.h")));
+      PosixSpec("symlink_dir/foo.h"), PosixSpec("/real_dir/foo.h"),
+      FileSpec::Style::posix));
 
   // Prepare RealpathPrefixes
   FileSpecList file_spec_list;
-  file_spec_list.EmplaceBack("symlink_dir");
+  file_spec_list.Append(PosixSpec("symlink_dir"));
   RealpathPrefixes prefixes(file_spec_list, fs);
 
   // Prepare support file list
   SupportFileList support_file_list;
-  support_file_list.EmplaceBack(FileSpec("symlink_dir/foo.h"));
+  support_file_list.Append(PosixSpec("symlink_dir/foo.h"));
 
   // Test
   size_t ret = support_file_list.FindCompatibleIndex(
-      0, FileSpec("real_dir/foo.h"), &prefixes);
+      0, PosixSpec("real_dir/foo.h"), &prefixes);
   EXPECT_EQ(ret, (size_t)0);
 }
 
@@ -277,20 +283,21 @@ TEST(SupportFileListTest, PartialBreakpointPath) {
 TEST(SupportFileListTest, DifferentBasename) {
   // Prepare FS
   llvm::IntrusiveRefCntPtr<MockSymlinkFileSystem> fs(new MockSymlinkFileSystem(
-      FileSpec("/symlink_dir/foo.h"), FileSpec("/real_dir/bar.h")));
+      PosixSpec("/symlink_dir/foo.h"), PosixSpec("/real_dir/bar.h"),
+      FileSpec::Style::posix));
 
   // Prepare RealpathPrefixes
   FileSpecList file_spec_list;
-  file_spec_list.EmplaceBack("/symlink_dir");
+  file_spec_list.Append(PosixSpec("/symlink_dir"));
   RealpathPrefixes prefixes(file_spec_list, fs);
 
   // Prepare support file list
   SupportFileList support_file_list;
-  support_file_list.EmplaceBack(FileSpec("/symlink_dir/foo.h"));
+  support_file_list.Append(PosixSpec("/symlink_dir/foo.h"));
 
   // Test
   size_t ret = support_file_list.FindCompatibleIndex(
-      0, FileSpec("real_dir/bar.h"), &prefixes);
+      0, PosixSpec("real_dir/bar.h"), &prefixes);
   EXPECT_EQ(ret, UINT32_MAX);
 }
 
@@ -300,11 +307,11 @@ TEST(SupportFileListTest, DifferentBasename) {
 TEST(SupportFileListTest, NoPrefixes) {
   // Prepare support file list
   SupportFileList support_file_list;
-  support_file_list.EmplaceBack(FileSpec("/real_dir/bar.h"));
+  support_file_list.Append(PosixSpec("/real_dir/bar.h"));
 
   // Test
   size_t ret = support_file_list.FindCompatibleIndex(
-      0, FileSpec("/real_dir/foo.h"), nullptr);
+      0, PosixSpec("/real_dir/foo.h"), nullptr);
   EXPECT_EQ(ret, UINT32_MAX);
 }
 
@@ -314,10 +321,10 @@ TEST(SupportFileListTest, NoPrefixes) {
 TEST(SupportFileListTest, SameFile) {
   // Prepare support file list
   SupportFileList support_file_list;
-  support_file_list.EmplaceBack(FileSpec("/real_dir/foo.h"));
+  support_file_list.Append(PosixSpec("/real_dir/foo.h"));
 
   // Test
   size_t ret = support_file_list.FindCompatibleIndex(
-      0, FileSpec("/real_dir/foo.h"), nullptr);
+      0, PosixSpec("/real_dir/foo.h"), nullptr);
   EXPECT_EQ(ret, (size_t)0);
 }
diff --git a/lldb/unittests/Utility/RealpathPrefixesTest.cpp b/lldb/unittests/Utility/RealpathPrefixesTest.cpp
index 872ddf1fd223aa9..a4dc7d6dd9822df 100644
--- a/lldb/unittests/Utility/RealpathPrefixesTest.cpp
+++ b/lldb/unittests/Utility/RealpathPrefixesTest.cpp
@@ -15,57 +15,66 @@
 
 using namespace lldb_private;
 
+static FileSpec PosixSpec(llvm::StringRef path) {
+  return FileSpec(path, FileSpec::Style::posix);
+}
+
+static FileSpec WindowsSpec(llvm::StringRef path) {
+  return FileSpec(path, FileSpec::Style::windows);
+}
+
 // Should resolve a symlink which match an absolute prefix
 TEST(RealpathPrefixesTest, MatchingAbsolutePrefix) {
   // Prepare FS
   llvm::IntrusiveRefCntPtr<MockSymlinkFileSystem> fs(new MockSymlinkFileSystem(
-      FileSpec("/dir1/link.h"), FileSpec("/dir2/real.h")));
+      PosixSpec("/dir1/link.h"), PosixSpec("/dir2/real.h"),
+      FileSpec::Style::posix));
 
   // Prepare RealpathPrefixes
   FileSpecList file_spec_list;
-  file_spec_list.EmplaceBack("/dir1");
+  file_spec_list.Append(PosixSpec("/dir1"));
   RealpathPrefixes prefixes(file_spec_list, fs);
 
   // Test
   std::optional<FileSpec> ret =
-      prefixes.ResolveSymlinks(FileSpec("/dir1/link.h"));
-  EXPECT_EQ(ret, FileSpec("/dir2/real.h"));
+      prefixes.ResolveSymlinks(PosixSpec("/dir1/link.h"));
+  EXPECT_EQ(ret, PosixSpec("/dir2/real.h"));
 }
 
 // Should resolve a symlink which match a relative prefix
 TEST(RealpathPrefixesTest, MatchingRelativePrefix) {
   // Prepare FS
   llvm::IntrusiveRefCntPtr<MockSymlinkFileSystem> fs(new MockSymlinkFileSystem(
-      FileSpec("dir1/link.h"), FileSpec("dir2/real.h")));
+      PosixSpec("dir1/link.h"), PosixSpec("dir2/real.h"),
+      FileSpec::Style::posix));
 
   // Prepare RealpathPrefixes
   FileSpecList file_spec_list;
-  file_spec_list.EmplaceBack("dir1");
+  file_spec_list.Append(PosixSpec("dir1"));
   RealpathPrefixes prefixes(file_spec_list, fs);
 
   // Test
   std::optional<FileSpec> ret =
-      prefixes.ResolveSymlinks(FileSpec("dir1/link.h"));
-  EXPECT_EQ(ret, FileSpec("dir2/real.h"));
+      prefixes.ResolveSymlinks(PosixSpec("dir1/link.h"));
+  EXPECT_EQ(ret, PosixSpec("dir2/real.h"));
 }
 
 // Should resolve in Windows and/or with a case-insensitive support file
 TEST(RealpathPrefixesTest, WindowsAndCaseInsensitive) {
   // Prepare FS
   llvm::IntrusiveRefCntPtr<MockSymlinkFileSystem> fs(new MockSymlinkFileSystem(
-      FileSpec("f:\\dir1\\link.h", FileSpec::Style::windows),
-      FileSpec("f:\\dir2\\real.h", FileSpec::Style::windows),
+      WindowsSpec("f:\\dir1\\link.h"), WindowsSpec("f:\\dir2\\real.h"),
       FileSpec::Style::windows));
 
   // Prepare RealpathPrefixes
   FileSpecList file_spec_list;
-  file_spec_list.EmplaceBack(FileSpec("f:\\dir1", FileSpec::Style::windows));
+  file_spec_list.Append(WindowsSpec("f:\\dir1"));
   RealpathPrefixes prefixes(file_spec_list, fs);
 
   // Test
-  std::optional<FileSpec> ret = prefixes.ResolveSymlinks(
-      FileSpec("F:\\DIR1\\LINK.H", FileSpec::Style::windows));
-  EXPECT_EQ(ret, FileSpec("f:\\dir2\\real.h", FileSpec::Style::windows));
+  std::optional<FileSpec> ret =
+      prefixes.ResolveSymlinks(WindowsSpec("F:\\DIR1\\LINK.H"));
+  EXPECT_EQ(ret, WindowsSpec("f:\\dir2\\real.h"));
 }
 
 // Should resolve a symlink when there is mixture of matching and mismatching
@@ -73,52 +82,56 @@ TEST(RealpathPrefixesTest, WindowsAndCaseInsensitive) {
 TEST(RealpathPrefixesTest, MatchingAndMismatchingPrefix) {
   // Prepare FS
   llvm::IntrusiveRefCntPtr<MockSymlinkFileSystem> fs(new MockSymlinkFileSystem(
-      FileSpec("/dir1/link.h"), FileSpec("/dir2/real.h")));
+      PosixSpec("/dir1/link.h"), PosixSpec("/dir2/real.h"),
+      FileSpec::Style::posix));
 
   // Prepare RealpathPrefixes
   FileSpecList file_spec_list;
-  file_spec_list.EmplaceBack("/fake/path1");
-  file_spec_list.EmplaceBack("/dir1"); // Matching prefix
-  file_spec_list.EmplaceBack("/fake/path2");
+  file_spec_list.Append(PosixSpec("/fake/path1"));
+  file_spec_list.Append(PosixSpec("/dir1")); // Matching prefix
+  file_spec_list.Append(PosixSpec("/fake/path2"));
   RealpathPrefixes prefixes(file_spec_list, fs);
 
   // Test
   std::optional<FileSpec> ret =
-      prefixes.ResolveSymlinks(FileSpec("/dir1/link.h"));
-  EXPECT_EQ(ret, FileSpec("/dir2/real.h"));
+      prefixes.ResolveSymlinks(PosixSpec("/dir1/link.h"));
+  EXPECT_EQ(ret, PosixSpec("/dir2/real.h"));
 }
 
 // Should resolve a symlink when the prefixes matches after normalization
 TEST(RealpathPrefixesTest, ComplexPrefixes) {
   // Prepare FS
   llvm::IntrusiveRefCntPtr<MockSymlinkFileSystem> fs(new MockSymlinkFileSystem(
-      FileSpec("dir1/link.h"), FileSpec("dir2/real.h")));
+      PosixSpec("dir1/link.h"), PosixSpec("dir2/real.h"),
+      FileSpec::Style::posix));
 
   // Prepare RealpathPrefixes
   FileSpecList file_spec_list;
-  file_spec_list.EmplaceBack("./dir1/foo/../bar/.."); // Equivalent to "/dir1"
+  file_spec_list.Append(
+      PosixSpec("./dir1/foo/../bar/..")); // Equivalent to "/dir1"
   RealpathPrefixes prefixes(file_spec_list, fs);
 
   // Test
   std::optional<FileSpec> ret =
-      prefixes.ResolveSymlinks(FileSpec("dir1/link.h"));
-  EXPECT_EQ(ret, FileSpec("dir2/real.h"));
+      prefixes.ResolveSymlinks(PosixSpec("dir1/link.h"));
+  EXPECT_EQ(ret, PosixSpec("dir2/real.h"));
 }
 
 // Should not resolve a symlink which doesn't match any prefixes
 TEST(RealpathPrefixesTest, MismatchingPrefixes) {
   // Prepare FS
   llvm::IntrusiveRefCntPtr<MockSymlinkFileSystem> fs(new MockSymlinkFileSystem(
-      FileSpec("/dir1/link.h"), FileSpec("/dir2/real.h")));
+      PosixSpec("/dir1/link.h"), PosixSpec("/dir2/real.h"),
+      FileSpec::Style::posix));
 
   // Prepare RealpathPrefixes
   FileSpecList file_spec_list;
-  file_spec_list.EmplaceBack("/dir3");
+  file_spec_list.Append(PosixSpec("/dir3"));
   RealpathPrefixes prefixes(file_spec_list, fs);
 
   // Test
   std::optional<FileSpec> ret =
-      prefixes.ResolveSymlinks(FileSpec("/dir1/link.h"));
+      prefixes.ResolveSymlinks(PosixSpec("/dir1/link.h"));
   EXPECT_EQ(ret, std::nullopt);
 }
 
@@ -130,11 +143,11 @@ TEST(RealpathPrefixesTest, Realpath) {
 
   // Prepare RealpathPrefixes
   FileSpecList file_spec_list;
-  file_spec_list.EmplaceBack("/symlink_dir");
+  file_spec_list.Append(PosixSpec("/symlink_dir"));
   RealpathPrefixes prefixes(file_spec_list, fs);
 
   // Test
   std::optional<FileSpec> ret =
-      prefixes.ResolveSymlinks(FileSpec("/dir/real.h"));
+      prefixes.ResolveSymlinks(PosixSpec("/dir/real.h"));
   EXPECT_EQ(ret, std::nullopt);
 }

From 812e049ac1c2424f96746ac4c453d6255b0a6ca5 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Thu, 15 Aug 2024 17:32:16 -0700
Subject: [PATCH 107/441] [Driver] Correctly handle -Wa,--crel -Wa,--no-crel

Follow-up to #97378
---
 clang/lib/Driver/ToolChains/CommonArgs.cpp | 16 ++++++++--------
 clang/test/Driver/crel.c                   |  2 ++
 2 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp
index 1cba3e1220264ab..320d2901da06ed5 100644
--- a/clang/lib/Driver/ToolChains/CommonArgs.cpp
+++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp
@@ -1143,8 +1143,8 @@ void tools::addLTOOptions(const ToolChain &ToolChain, const ArgList &Args,
   addMachineOutlinerArgs(D, Args, CmdArgs, ToolChain.getEffectiveTriple(),
                          /*IsLTO=*/true, PluginOptPrefix);
 
+  bool Crel = false;
   for (const Arg *A : Args.filtered(options::OPT_Wa_COMMA)) {
-    bool Crel = false;
     for (StringRef V : A->getValues()) {
       if (V == "--crel")
         Crel = true;
@@ -1154,13 +1154,13 @@ void tools::addLTOOptions(const ToolChain &ToolChain, const ArgList &Args,
         continue;
       A->claim();
     }
-    if (Crel) {
-      if (Triple.isOSBinFormatELF() && !Triple.isMIPS()) {
-        CmdArgs.push_back(Args.MakeArgString(Twine(PluginOptPrefix) + "-crel"));
-      } else {
-        D.Diag(diag::err_drv_unsupported_opt_for_target)
-            << "-Wa,--crel" << D.getTargetTriple();
-      }
+  }
+  if (Crel) {
+    if (Triple.isOSBinFormatELF() && !Triple.isMIPS()) {
+      CmdArgs.push_back(Args.MakeArgString(Twine(PluginOptPrefix) + "-crel"));
+    } else {
+      D.Diag(diag::err_drv_unsupported_opt_for_target)
+          << "-Wa,--crel" << D.getTargetTriple();
     }
   }
 }
diff --git a/clang/test/Driver/crel.c b/clang/test/Driver/crel.c
index a47e7ebfeedd88f..e1b3f58c6ea5b73 100644
--- a/clang/test/Driver/crel.c
+++ b/clang/test/Driver/crel.c
@@ -24,8 +24,10 @@
 
 /// The --allow-experimental-crel error check doesn't apply to LTO.
 // RUN: %clang -### --target=x86_64-linux -Werror -flto -Wa,--crel %s 2>&1 | FileCheck %s --check-prefix=LTO
+// RUN: %clang -### --target=x86_64-linux -Werror -flto -Wa,--crel -Wa,--no-crel %s 2>&1 | FileCheck %s --check-prefix=LTO-NO
 
 // LTO: "-plugin-opt=-crel"
+// LTO-NO-NOT: "-plugin-opt=-crel"
 
 // RUN: touch %t.o
 // RUN: not %clang -### --target=mips64-linux-gnu -flto -Wa,--crel %t.o 2>&1 | FileCheck %s --check-prefix=ERR

From 5d48248755f41245be031791ff6ea7a79a5a8912 Mon Sep 17 00:00:00 2001
From: khaki3 <47756807+khaki3@users.noreply.github.com>
Date: Thu, 15 Aug 2024 17:32:56 -0700
Subject: [PATCH 108/441] [flang] Inline minval/maxval over elemental/designate
 (#103503)

This PR intends to optimize away `hlfir.elemental` operations, which
leave temporary buffers (`allocmem`) in FIR. We typically see elemental
operations in the arguments of reduction intrinsics, so extending
`OptimizedBufferization` shall be the first solution to get heap-free
code.

Here we newly handle `minval`/`maxval` along with other reduction
intrinsics. Those functions over elemental become do loops. Furthermore,
we take the same action with `hlfir.designate` in order to inline more
intrinsics, which otherwise call runtime routines.
---
 .../Transforms/OptimizedBufferization.cpp     | 264 +++++++++++-------
 flang/test/HLFIR/maxval-elemental.fir         |  95 +++++++
 flang/test/HLFIR/minval-elemental.fir         |  95 +++++++
 3 files changed, 360 insertions(+), 94 deletions(-)
 create mode 100644 flang/test/HLFIR/maxval-elemental.fir
 create mode 100644 flang/test/HLFIR/minval-elemental.fir

diff --git a/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp b/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
index c5b809514c54c6c..7553e05b4706341 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/OptimizedBufferization.cpp
@@ -702,8 +702,53 @@ static mlir::Value generateReductionLoop(fir::FirOpBuilder &builder,
   return reduction;
 }
 
-/// Given a reduction operation with an elemental mask, attempt to generate a
-/// do-loop to perform the operation inline.
+auto makeMinMaxInitValGenerator(bool isMax) {
+  return [isMax](fir::FirOpBuilder builder, mlir::Location loc,
+                 mlir::Type elementType) -> mlir::Value {
+    if (auto ty = mlir::dyn_cast<mlir::FloatType>(elementType)) {
+      const llvm::fltSemantics &sem = ty.getFloatSemantics();
+      llvm::APFloat limit = llvm::APFloat::getInf(sem, /*Negative=*/isMax);
+      return builder.createRealConstant(loc, elementType, limit);
+    }
+    unsigned bits = elementType.getIntOrFloatBitWidth();
+    int64_t limitInt =
+        isMax ? llvm::APInt::getSignedMinValue(bits).getSExtValue()
+              : llvm::APInt::getSignedMaxValue(bits).getSExtValue();
+    return builder.createIntegerConstant(loc, elementType, limitInt);
+  };
+}
+
+mlir::Value generateMinMaxComparison(fir::FirOpBuilder builder,
+                                     mlir::Location loc, mlir::Value elem,
+                                     mlir::Value reduction, bool isMax) {
+  if (mlir::isa<mlir::FloatType>(reduction.getType())) {
+    // For FP reductions we want the first smallest value to be used, that
+    // is not NaN. A OGL/OLT condition will usually work for this unless all
+    // the values are Nan or Inf. This follows the same logic as
+    // NumericCompare for Minloc/Maxlox in extrema.cpp.
+    mlir::Value cmp = builder.create<mlir::arith::CmpFOp>(
+        loc,
+        isMax ? mlir::arith::CmpFPredicate::OGT
+              : mlir::arith::CmpFPredicate::OLT,
+        elem, reduction);
+    mlir::Value cmpNan = builder.create<mlir::arith::CmpFOp>(
+        loc, mlir::arith::CmpFPredicate::UNE, reduction, reduction);
+    mlir::Value cmpNan2 = builder.create<mlir::arith::CmpFOp>(
+        loc, mlir::arith::CmpFPredicate::OEQ, elem, elem);
+    cmpNan = builder.create<mlir::arith::AndIOp>(loc, cmpNan, cmpNan2);
+    return builder.create<mlir::arith::OrIOp>(loc, cmp, cmpNan);
+  } else if (mlir::isa<mlir::IntegerType>(reduction.getType())) {
+    return builder.create<mlir::arith::CmpIOp>(
+        loc,
+        isMax ? mlir::arith::CmpIPredicate::sgt
+              : mlir::arith::CmpIPredicate::slt,
+        elem, reduction);
+  }
+  llvm_unreachable("unsupported type");
+}
+
+/// Given a reduction operation with an elemental/designate source, attempt to
+/// generate a do-loop to perform the operation inline.
 ///   %e = hlfir.elemental %shape unordered
 ///   %r = hlfir.count %e
 /// =>
@@ -712,17 +757,66 @@ static mlir::Value generateReductionLoop(fir::FirOpBuilder &builder,
 ///     %c = <reduce count> %i
 ///     fir.result %c
 template <typename Op>
-class ReductionElementalConversion : public mlir::OpRewritePattern<Op> {
+class ReductionConversion : public mlir::OpRewritePattern<Op> {
 public:
   using mlir::OpRewritePattern<Op>::OpRewritePattern;
 
   llvm::LogicalResult
   matchAndRewrite(Op op, mlir::PatternRewriter &rewriter) const override {
     mlir::Location loc = op.getLoc();
-    hlfir::ElementalOp elemental =
-        op.getMask().template getDefiningOp<hlfir::ElementalOp>();
-    if (!elemental || op.getDim())
-      return rewriter.notifyMatchFailure(op, "Did not find valid elemental");
+    // Select source and validate its arguments.
+    mlir::Value source;
+    bool valid = false;
+    if constexpr (std::is_same_v<Op, hlfir::AnyOp> ||
+                  std::is_same_v<Op, hlfir::AllOp> ||
+                  std::is_same_v<Op, hlfir::CountOp>) {
+      source = op.getMask();
+      valid = !op.getDim();
+    } else if constexpr (std::is_same_v<Op, hlfir::MaxvalOp> ||
+                         std::is_same_v<Op, hlfir::MinvalOp>) {
+      source = op.getArray();
+      valid = !op.getDim() && !op.getMask();
+    } else if constexpr (std::is_same_v<Op, hlfir::MaxlocOp> ||
+                         std::is_same_v<Op, hlfir::MinlocOp>) {
+      source = op.getArray();
+      valid = !op.getDim() && !op.getMask() && !op.getBack();
+    }
+    if (!valid)
+      return rewriter.notifyMatchFailure(
+          op, "Currently does not accept optional arguments");
+
+    hlfir::ElementalOp elemental;
+    hlfir::DesignateOp designate;
+    mlir::Value shape;
+    if ((elemental = source.template getDefiningOp<hlfir::ElementalOp>())) {
+      shape = elemental.getOperand(0);
+    } else if ((designate =
+                    source.template getDefiningOp<hlfir::DesignateOp>())) {
+      shape = designate.getShape();
+    } else {
+      return rewriter.notifyMatchFailure(op, "Did not find valid argument");
+    }
+
+    auto inlineSource =
+        [elemental, &designate](
+            fir::FirOpBuilder builder, mlir::Location loc,
+            const llvm::SmallVectorImpl<mlir::Value> &indices) -> mlir::Value {
+      if (elemental) {
+        // Inline the elemental and get the value from it.
+        auto yield = inlineElementalOp(loc, builder, elemental, indices);
+        auto tmp = yield.getElementValue();
+        yield->erase();
+        return tmp;
+      }
+      if (designate) {
+        // Create a designator over designator, then load the reference.
+        auto resEntity = hlfir::Entity{designate.getResult()};
+        auto tmp = builder.create<hlfir::DesignateOp>(
+            loc, getVariableElementType(resEntity), designate, indices);
+        return builder.create<fir::LoadOp>(loc, tmp);
+      }
+      llvm_unreachable("unsupported type");
+    };
 
     fir::KindMapping kindMap =
         fir::getKindMapping(op->template getParentOfType<mlir::ModuleOp>());
@@ -732,47 +826,38 @@ class ReductionElementalConversion : public mlir::OpRewritePattern<Op> {
     GenBodyFn genBodyFn;
     if constexpr (std::is_same_v<Op, hlfir::AnyOp>) {
       init = builder.createIntegerConstant(loc, builder.getI1Type(), 0);
-      genBodyFn = [elemental](fir::FirOpBuilder builder, mlir::Location loc,
-                              mlir::Value reduction,
-                              const llvm::SmallVectorImpl<mlir::Value> &indices)
+      genBodyFn =
+          [inlineSource](fir::FirOpBuilder builder, mlir::Location loc,
+                         mlir::Value reduction,
+                         const llvm::SmallVectorImpl<mlir::Value> &indices)
           -> mlir::Value {
-        // Inline the elemental and get the condition from it.
-        auto yield = inlineElementalOp(loc, builder, elemental, indices);
-        mlir::Value cond = builder.create<fir::ConvertOp>(
-            loc, builder.getI1Type(), yield.getElementValue());
-        yield->erase();
-
         // Conditionally set the reduction variable.
+        mlir::Value cond = builder.create<fir::ConvertOp>(
+            loc, builder.getI1Type(), inlineSource(builder, loc, indices));
         return builder.create<mlir::arith::OrIOp>(loc, reduction, cond);
       };
     } else if constexpr (std::is_same_v<Op, hlfir::AllOp>) {
       init = builder.createIntegerConstant(loc, builder.getI1Type(), 1);
-      genBodyFn = [elemental](fir::FirOpBuilder builder, mlir::Location loc,
-                              mlir::Value reduction,
-                              const llvm::SmallVectorImpl<mlir::Value> &indices)
+      genBodyFn =
+          [inlineSource](fir::FirOpBuilder builder, mlir::Location loc,
+                         mlir::Value reduction,
+                         const llvm::SmallVectorImpl<mlir::Value> &indices)
           -> mlir::Value {
-        // Inline the elemental and get the condition from it.
-        auto yield = inlineElementalOp(loc, builder, elemental, indices);
-        mlir::Value cond = builder.create<fir::ConvertOp>(
-            loc, builder.getI1Type(), yield.getElementValue());
-        yield->erase();
-
         // Conditionally set the reduction variable.
+        mlir::Value cond = builder.create<fir::ConvertOp>(
+            loc, builder.getI1Type(), inlineSource(builder, loc, indices));
         return builder.create<mlir::arith::AndIOp>(loc, reduction, cond);
       };
     } else if constexpr (std::is_same_v<Op, hlfir::CountOp>) {
       init = builder.createIntegerConstant(loc, op.getType(), 0);
-      genBodyFn = [elemental](fir::FirOpBuilder builder, mlir::Location loc,
-                              mlir::Value reduction,
-                              const llvm::SmallVectorImpl<mlir::Value> &indices)
+      genBodyFn =
+          [inlineSource](fir::FirOpBuilder builder, mlir::Location loc,
+                         mlir::Value reduction,
+                         const llvm::SmallVectorImpl<mlir::Value> &indices)
           -> mlir::Value {
-        // Inline the elemental and get the condition from it.
-        auto yield = inlineElementalOp(loc, builder, elemental, indices);
-        mlir::Value cond = builder.create<fir::ConvertOp>(
-            loc, builder.getI1Type(), yield.getElementValue());
-        yield->erase();
-
         // Conditionally add one to the current value
+        mlir::Value cond = builder.create<fir::ConvertOp>(
+            loc, builder.getI1Type(), inlineSource(builder, loc, indices));
         mlir::Value one =
             builder.createIntegerConstant(loc, reduction.getType(), 1);
         mlir::Value add1 =
@@ -780,29 +865,49 @@ class ReductionElementalConversion : public mlir::OpRewritePattern<Op> {
         return builder.create<mlir::arith::SelectOp>(loc, cond, add1,
                                                      reduction);
       };
+    } else if constexpr (std::is_same_v<Op, hlfir::MaxlocOp> ||
+                         std::is_same_v<Op, hlfir::MinlocOp>) {
+      // TODO: implement minloc/maxloc conversion.
+      return rewriter.notifyMatchFailure(
+          op, "Currently minloc/maxloc is not handled");
+    } else if constexpr (std::is_same_v<Op, hlfir::MaxvalOp> ||
+                         std::is_same_v<Op, hlfir::MinvalOp>) {
+      bool isMax = std::is_same_v<Op, hlfir::MaxvalOp>;
+      init = makeMinMaxInitValGenerator(isMax)(builder, loc, op.getType());
+      genBodyFn = [inlineSource,
+                   isMax](fir::FirOpBuilder builder, mlir::Location loc,
+                          mlir::Value reduction,
+                          const llvm::SmallVectorImpl<mlir::Value> &indices)
+          -> mlir::Value {
+        mlir::Value val = inlineSource(builder, loc, indices);
+        mlir::Value cmp =
+            generateMinMaxComparison(builder, loc, val, reduction, isMax);
+        return builder.create<mlir::arith::SelectOp>(loc, cmp, val, reduction);
+      };
     } else {
-      return mlir::failure();
+      llvm_unreachable("unsupported type");
     }
 
-    mlir::Value res = generateReductionLoop(builder, loc, init,
-                                            elemental.getOperand(0), genBodyFn);
+    mlir::Value res =
+        generateReductionLoop(builder, loc, init, shape, genBodyFn);
     if (res.getType() != op.getType())
       res = builder.create<fir::ConvertOp>(loc, op.getType(), res);
 
-    // Check if the op was the only user of the elemental (apart from a
-    // destroy), and remove it if so.
-    mlir::Operation::user_range elemUsers = elemental->getUsers();
-    hlfir::DestroyOp elemDestroy;
-    if (std::distance(elemUsers.begin(), elemUsers.end()) == 2) {
-      elemDestroy = mlir::dyn_cast<hlfir::DestroyOp>(*elemUsers.begin());
-      if (!elemDestroy)
-        elemDestroy = mlir::dyn_cast<hlfir::DestroyOp>(*++elemUsers.begin());
+    // Check if the op was the only user of the source (apart from a destroy),
+    // and remove it if so.
+    mlir::Operation *sourceOp = source.getDefiningOp();
+    mlir::Operation::user_range srcUsers = sourceOp->getUsers();
+    hlfir::DestroyOp srcDestroy;
+    if (std::distance(srcUsers.begin(), srcUsers.end()) == 2) {
+      srcDestroy = mlir::dyn_cast<hlfir::DestroyOp>(*srcUsers.begin());
+      if (!srcDestroy)
+        srcDestroy = mlir::dyn_cast<hlfir::DestroyOp>(*++srcUsers.begin());
     }
 
     rewriter.replaceOp(op, res);
-    if (elemDestroy) {
-      rewriter.eraseOp(elemDestroy);
-      rewriter.eraseOp(elemental);
+    if (srcDestroy) {
+      rewriter.eraseOp(srcDestroy);
+      rewriter.eraseOp(sourceOp);
     }
     return mlir::success();
   }
@@ -813,7 +918,7 @@ class ReductionElementalConversion : public mlir::OpRewritePattern<Op> {
 //  %e = hlfir.elemental %shape ({ ... })
 //  %m = hlfir.minloc %array mask %e
 template <typename Op>
-class MinMaxlocElementalConversion : public mlir::OpRewritePattern<Op> {
+class ReductionMaskConversion : public mlir::OpRewritePattern<Op> {
 public:
   using mlir::OpRewritePattern<Op>::OpRewritePattern;
 
@@ -848,19 +953,7 @@ class MinMaxlocElementalConversion : public mlir::OpRewritePattern<Op> {
         loc, fir::SequenceType::get(
                  rank, hlfir::getFortranElementType(mloc.getType())));
 
-    auto init = [isMax](fir::FirOpBuilder builder, mlir::Location loc,
-                        mlir::Type elementType) {
-      if (auto ty = mlir::dyn_cast<mlir::FloatType>(elementType)) {
-        const llvm::fltSemantics &sem = ty.getFloatSemantics();
-        llvm::APFloat limit = llvm::APFloat::getInf(sem, /*Negative=*/isMax);
-        return builder.createRealConstant(loc, elementType, limit);
-      }
-      unsigned bits = elementType.getIntOrFloatBitWidth();
-      int64_t limitInt =
-          isMax ? llvm::APInt::getSignedMinValue(bits).getSExtValue()
-                : llvm::APInt::getSignedMaxValue(bits).getSExtValue();
-      return builder.createIntegerConstant(loc, elementType, limitInt);
-    };
+    auto init = makeMinMaxInitValGenerator(isMax);
 
     auto genBodyOp =
         [&rank, &resultArr, &elemental, isMax](
@@ -900,33 +993,8 @@ class MinMaxlocElementalConversion : public mlir::OpRewritePattern<Op> {
       mlir::Value elem = builder.create<fir::LoadOp>(loc, addr);
 
       // Compare with the max reduction value
-      mlir::Value cmp;
-      if (mlir::isa<mlir::FloatType>(elementType)) {
-        // For FP reductions we want the first smallest value to be used, that
-        // is not NaN. A OGL/OLT condition will usually work for this unless all
-        // the values are Nan or Inf. This follows the same logic as
-        // NumericCompare for Minloc/Maxlox in extrema.cpp.
-        cmp = builder.create<mlir::arith::CmpFOp>(
-            loc,
-            isMax ? mlir::arith::CmpFPredicate::OGT
-                  : mlir::arith::CmpFPredicate::OLT,
-            elem, reduction);
-
-        mlir::Value cmpNan = builder.create<mlir::arith::CmpFOp>(
-            loc, mlir::arith::CmpFPredicate::UNE, reduction, reduction);
-        mlir::Value cmpNan2 = builder.create<mlir::arith::CmpFOp>(
-            loc, mlir::arith::CmpFPredicate::OEQ, elem, elem);
-        cmpNan = builder.create<mlir::arith::AndIOp>(loc, cmpNan, cmpNan2);
-        cmp = builder.create<mlir::arith::OrIOp>(loc, cmp, cmpNan);
-      } else if (mlir::isa<mlir::IntegerType>(elementType)) {
-        cmp = builder.create<mlir::arith::CmpIOp>(
-            loc,
-            isMax ? mlir::arith::CmpIPredicate::sgt
-                  : mlir::arith::CmpIPredicate::slt,
-            elem, reduction);
-      } else {
-        llvm_unreachable("unsupported type");
-      }
+      mlir::Value cmp =
+          generateMinMaxComparison(builder, loc, elem, reduction, isMax);
 
       // The condition used for the loop is isFirst || <the condition above>.
       isFirst = builder.create<fir::ConvertOp>(loc, cmp.getType(), isFirst);
@@ -1055,11 +1123,19 @@ class OptimizedBufferizationPass
     patterns.insert<ElementalAssignBufferization>(context);
     patterns.insert<BroadcastAssignBufferization>(context);
     patterns.insert<VariableAssignBufferization>(context);
-    patterns.insert<ReductionElementalConversion<hlfir::CountOp>>(context);
-    patterns.insert<ReductionElementalConversion<hlfir::AnyOp>>(context);
-    patterns.insert<ReductionElementalConversion<hlfir::AllOp>>(context);
-    patterns.insert<MinMaxlocElementalConversion<hlfir::MinlocOp>>(context);
-    patterns.insert<MinMaxlocElementalConversion<hlfir::MaxlocOp>>(context);
+    patterns.insert<ReductionConversion<hlfir::CountOp>>(context);
+    patterns.insert<ReductionConversion<hlfir::AnyOp>>(context);
+    patterns.insert<ReductionConversion<hlfir::AllOp>>(context);
+    // TODO: implement basic minloc/maxloc conversion.
+    // patterns.insert<ReductionConversion<hlfir::MaxlocOp>>(context);
+    // patterns.insert<ReductionConversion<hlfir::MinlocOp>>(context);
+    patterns.insert<ReductionConversion<hlfir::MaxvalOp>>(context);
+    patterns.insert<ReductionConversion<hlfir::MinvalOp>>(context);
+    patterns.insert<ReductionMaskConversion<hlfir::MinlocOp>>(context);
+    patterns.insert<ReductionMaskConversion<hlfir::MaxlocOp>>(context);
+    // TODO: implement masked minval/maxval conversion.
+    // patterns.insert<ReductionMaskConversion<hlfir::MaxvalOp>>(context);
+    // patterns.insert<ReductionMaskConversion<hlfir::MinvalOp>>(context);
 
     if (mlir::failed(mlir::applyPatternsAndFoldGreedily(
             getOperation(), std::move(patterns), config))) {
diff --git a/flang/test/HLFIR/maxval-elemental.fir b/flang/test/HLFIR/maxval-elemental.fir
new file mode 100644
index 000000000000000..aa642253b08323a
--- /dev/null
+++ b/flang/test/HLFIR/maxval-elemental.fir
@@ -0,0 +1,95 @@
+// Test maxval inlining for both elemental and designate
+// RUN: fir-opt %s -opt-bufferization | FileCheck %s
+
+// subroutine test(array)
+//   integer :: array(:), x
+//   x = maxval(abs(array))
+// end subroutine test
+
+func.func @_QPtest(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}) {
+  %c31_i32 = arith.constant 31 : i32
+  %c0 = arith.constant 0 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+  %2 = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFtestEx"}
+  %3:2 = hlfir.declare %2 {uniq_name = "_QFtestEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %4:3 = fir.box_dims %1#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
+  %5 = fir.shape %4#1 : (index) -> !fir.shape<1>
+  %6 = hlfir.elemental %5 unordered : (!fir.shape<1>) -> !hlfir.expr<?xi32> {
+  ^bb0(%arg1: index):
+    %8 = hlfir.designate %1#0 (%arg1)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+    %9 = fir.load %8 : !fir.ref<i32>
+    %10 = arith.shrsi %9, %c31_i32 : i32
+    %11 = arith.xori %9, %10 : i32
+    %12 = arith.subi %11, %10 : i32
+    hlfir.yield_element %12 : i32
+  }
+  %7 = hlfir.maxval %6 {fastmath = #arith.fastmath<contract>} : (!hlfir.expr<?xi32>) -> i32
+  hlfir.assign %7 to %3#0 : i32, !fir.ref<i32>
+  hlfir.destroy %6 : !hlfir.expr<?xi32>
+  return
+}
+
+// CHECK-LABEL: func.func @_QPtest(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}) {
+// CHECK-NEXT:    %c1 = arith.constant 1 : index
+// CHECK-NEXT:    %c-2147483648_i32 = arith.constant -2147483648 : i32
+// CHECK-NEXT:    %c31_i32 = arith.constant 31 : i32
+// CHECK-NEXT:    %c0 = arith.constant 0 : index
+// CHECK-NEXT:    %[[V0:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK-NEXT:    %[[V1:.*]]:2 = hlfir.declare %arg0 dummy_scope %[[V0]] {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+// CHECK-NEXT:    %[[V2:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFtestEx"}
+// CHECK-NEXT:    %[[V3:.*]]:2 = hlfir.declare %[[V2]] {uniq_name = "_QFtestEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK-NEXT:    %[[V4:.*]]:3 = fir.box_dims %[[V1]]#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
+// CHECK-NEXT:    %[[V5:.*]] = fir.do_loop %arg1 = %c1 to %[[V4]]#1 step %c1 iter_args(%arg2 = %c-2147483648_i32) -> (i32) {
+// CHECK-NEXT:      %[[V6:.*]] = hlfir.designate %[[V1]]#0 (%arg1)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// CHECK-NEXT:      %[[V7:.*]] = fir.load %[[V6]] : !fir.ref<i32>
+// CHECK-NEXT:      %[[V8:.*]] = arith.shrsi %[[V7]], %c31_i32 : i32
+// CHECK-NEXT:      %[[V9:.*]] = arith.xori %[[V7]], %[[V8]] : i32
+// CHECK-NEXT:      %[[V10:.*]] = arith.subi %[[V9]], %[[V8]] : i32
+// CHECK-NEXT:      %[[V11:.*]] = arith.cmpi sgt, %[[V10]], %arg2 : i32
+// CHECK-NEXT:      %[[V12:.*]] = arith.select %[[V11]], %[[V10]], %arg2 : i32
+// CHECK-NEXT:      fir.result %[[V12]] : i32
+// CHECK-NEXT:    }
+// CHECK-NEXT:    hlfir.assign %[[V5]] to %[[V3]]#0 : i32, !fir.ref<i32>
+// CHECK-NEXT:    return
+// CHECK-NEXT:  }
+
+// subroutine test(array)
+//   real :: array(:), x
+//   x = maxval(array(3:6))
+// end subroutine test
+
+func.func @_QPtest_float(%arg0: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "array"}) {
+  %c4 = arith.constant 4 : index
+  %c1 = arith.constant 1 : index
+  %c6 = arith.constant 6 : index
+  %c3 = arith.constant 3 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
+  %2 = fir.alloca f32 {bindc_name = "x", uniq_name = "_QFtestEx"}
+  %3:2 = hlfir.declare %2 {uniq_name = "_QFtestEx"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
+  %4 = fir.shape %c4 : (index) -> !fir.shape<1>
+  %5 = hlfir.designate %1#0 (%c3:%c6:%c1)  shape %4 : (!fir.box<!fir.array<?xf32>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<4xf32>>
+  %6 = hlfir.maxval %5 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<4xf32>>) -> f32
+  hlfir.assign %6 to %3#0 : f32, !fir.ref<f32>
+  return
+}
+
+// CHECK-LABEL: _QPtest_float
+// CHECK:       %cst = arith.constant 0xFF800000 : f32
+// CHECK:       %[[V4:.*]] = fir.shape %c4 : (index) -> !fir.shape<1>
+// CHECK-NEXT:  %[[V5:.*]] = hlfir.designate %{{.*}} (%c3:%c6:%c1)  shape %[[V4]] : (!fir.box<!fir.array<?xf32>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<4xf32>>
+// CHECK-NEXT:  %[[V6:.*]] = fir.do_loop %arg1 = %c1 to %c4 step %c1 iter_args(%arg2 = %cst) -> (f32) {
+// CHECK-NEXT:      %[[V7:.*]] = hlfir.designate %[[V5]] (%arg1)  : (!fir.box<!fir.array<4xf32>>, index) -> !fir.ref<f32>
+// CHECK-NEXT:      %[[V8:.*]] = fir.load %[[V7]] : !fir.ref<f32>
+// CHECK-NEXT:      %[[V9:.*]] = arith.cmpf ogt, %[[V8]], %arg2 : f32
+// CHECK-NEXT:      %[[V10:.*]] = arith.cmpf une, %arg2, %arg2 : f32
+// CHECK-NEXT:      %[[V11:.*]] = arith.cmpf oeq, %[[V8]], %[[V8]] : f32
+// CHECK-NEXT:      %[[V12:.*]] = arith.andi %[[V10]], %[[V11]] : i1
+// CHECK-NEXT:      %[[V13:.*]] = arith.ori %[[V9]], %[[V12]] : i1
+// CHECK-NEXT:      %[[V14:.*]] = arith.select %[[V13]], %[[V8]], %arg2 : f32
+// CHECK-NEXT:      fir.result %[[V14]] : f32
+// CHECK-NEXT:    }
+// CHECK-NEXT:    hlfir.assign %[[V6]] to %3#0 : f32, !fir.ref<f32>
+// CHECK-NEXT:    return
+// CHECK-NEXT:  }
diff --git a/flang/test/HLFIR/minval-elemental.fir b/flang/test/HLFIR/minval-elemental.fir
new file mode 100644
index 000000000000000..8da1b1bdf515b9f
--- /dev/null
+++ b/flang/test/HLFIR/minval-elemental.fir
@@ -0,0 +1,95 @@
+// Test maxval inlining for both elemental and designate
+// RUN: fir-opt %s -opt-bufferization | FileCheck %s
+
+// subroutine test(array)
+//   integer :: array(:), x
+//   x = minval(abs(array))
+// end subroutine test
+
+func.func @_QPtest(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}) {
+  %c31_i32 = arith.constant 31 : i32
+  %c0 = arith.constant 0 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+  %2 = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFtestEx"}
+  %3:2 = hlfir.declare %2 {uniq_name = "_QFtestEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %4:3 = fir.box_dims %1#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
+  %5 = fir.shape %4#1 : (index) -> !fir.shape<1>
+  %6 = hlfir.elemental %5 unordered : (!fir.shape<1>) -> !hlfir.expr<?xi32> {
+  ^bb0(%arg1: index):
+    %8 = hlfir.designate %1#0 (%arg1)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+    %9 = fir.load %8 : !fir.ref<i32>
+    %10 = arith.shrsi %9, %c31_i32 : i32
+    %11 = arith.xori %9, %10 : i32
+    %12 = arith.subi %11, %10 : i32
+    hlfir.yield_element %12 : i32
+  }
+  %7 = hlfir.minval %6 {fastmath = #arith.fastmath<contract>} : (!hlfir.expr<?xi32>) -> i32
+  hlfir.assign %7 to %3#0 : i32, !fir.ref<i32>
+  hlfir.destroy %6 : !hlfir.expr<?xi32>
+  return
+}
+
+// CHECK-LABEL: func.func @_QPtest(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "array"}) {
+// CHECK-NEXT:    %c1 = arith.constant 1 : index
+// CHECK-NEXT:    %c2147483647_i32 = arith.constant 2147483647 : i32
+// CHECK-NEXT:    %c31_i32 = arith.constant 31 : i32
+// CHECK-NEXT:    %c0 = arith.constant 0 : index
+// CHECK-NEXT:    %[[V0:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK-NEXT:    %[[V1:.*]]:2 = hlfir.declare %arg0 dummy_scope %[[V0]] {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+// CHECK-NEXT:    %[[V2:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFtestEx"}
+// CHECK-NEXT:    %[[V3:.*]]:2 = hlfir.declare %[[V2]] {uniq_name = "_QFtestEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK-NEXT:    %[[V4:.*]]:3 = fir.box_dims %[[V1]]#0, %c0 : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
+// CHECK-NEXT:    %[[V5:.*]] = fir.do_loop %arg1 = %c1 to %[[V4]]#1 step %c1 iter_args(%arg2 = %c2147483647_i32) -> (i32) {
+// CHECK-NEXT:      %[[V6:.*]] = hlfir.designate %[[V1]]#0 (%arg1)  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
+// CHECK-NEXT:      %[[V7:.*]] = fir.load %[[V6]] : !fir.ref<i32>
+// CHECK-NEXT:      %[[V8:.*]] = arith.shrsi %[[V7]], %c31_i32 : i32
+// CHECK-NEXT:      %[[V9:.*]] = arith.xori %[[V7]], %[[V8]] : i32
+// CHECK-NEXT:      %[[V10:.*]] = arith.subi %[[V9]], %[[V8]] : i32
+// CHECK-NEXT:      %[[V11:.*]] = arith.cmpi slt, %[[V10]], %arg2 : i32
+// CHECK-NEXT:      %[[V12:.*]] = arith.select %[[V11]], %[[V10]], %arg2 : i32
+// CHECK-NEXT:      fir.result %[[V12]] : i32
+// CHECK-NEXT:    }
+// CHECK-NEXT:    hlfir.assign %[[V5]] to %[[V3]]#0 : i32, !fir.ref<i32>
+// CHECK-NEXT:    return
+// CHECK-NEXT:  }
+
+// subroutine test(array)
+//   real :: array(:), x
+//   x = minval(array(3:6))
+// end subroutine test
+
+func.func @_QPtest_float(%arg0: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "array"}) {
+  %c4 = arith.constant 4 : index
+  %c1 = arith.constant 1 : index
+  %c6 = arith.constant 6 : index
+  %c3 = arith.constant 3 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFtestEarray"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
+  %2 = fir.alloca f32 {bindc_name = "x", uniq_name = "_QFtestEx"}
+  %3:2 = hlfir.declare %2 {uniq_name = "_QFtestEx"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
+  %4 = fir.shape %c4 : (index) -> !fir.shape<1>
+  %5 = hlfir.designate %1#0 (%c3:%c6:%c1)  shape %4 : (!fir.box<!fir.array<?xf32>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<4xf32>>
+  %6 = hlfir.minval %5 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<4xf32>>) -> f32
+  hlfir.assign %6 to %3#0 : f32, !fir.ref<f32>
+  return
+}
+
+// CHECK-LABEL: _QPtest_float
+// CHECK:       %cst = arith.constant 0x7F800000 : f32
+// CHECK:       %[[V4:.*]] = fir.shape %c4 : (index) -> !fir.shape<1>
+// CHECK-NEXT:  %[[V5:.*]] = hlfir.designate %{{.*}} (%c3:%c6:%c1)  shape %[[V4]] : (!fir.box<!fir.array<?xf32>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<4xf32>>
+// CHECK-NEXT:  %[[V6:.*]] = fir.do_loop %arg1 = %c1 to %c4 step %c1 iter_args(%arg2 = %cst) -> (f32) {
+// CHECK-NEXT:      %[[V7:.*]] = hlfir.designate %[[V5]] (%arg1)  : (!fir.box<!fir.array<4xf32>>, index) -> !fir.ref<f32>
+// CHECK-NEXT:      %[[V8:.*]] = fir.load %[[V7]] : !fir.ref<f32>
+// CHECK-NEXT:      %[[V9:.*]] = arith.cmpf olt, %[[V8]], %arg2 : f32
+// CHECK-NEXT:      %[[V10:.*]] = arith.cmpf une, %arg2, %arg2 : f32
+// CHECK-NEXT:      %[[V11:.*]] = arith.cmpf oeq, %[[V8]], %[[V8]] : f32
+// CHECK-NEXT:      %[[V12:.*]] = arith.andi %[[V10]], %[[V11]] : i1
+// CHECK-NEXT:      %[[V13:.*]] = arith.ori %[[V9]], %[[V12]] : i1
+// CHECK-NEXT:      %[[V14:.*]] = arith.select %[[V13]], %[[V8]], %arg2 : f32
+// CHECK-NEXT:      fir.result %[[V14]] : f32
+// CHECK-NEXT:    }
+// CHECK-NEXT:    hlfir.assign %[[V6]] to %3#0 : f32, !fir.ref<f32>
+// CHECK-NEXT:    return
+// CHECK-NEXT:  }

From 6992c09dc9a99215e2a426d2524c0e12f11a7b57 Mon Sep 17 00:00:00 2001
From: Lang Hames <lhames@gmail.com>
Date: Fri, 16 Aug 2024 10:45:12 +1000
Subject: [PATCH 109/441] [ORC] Add missing dependence on BinaryFormat library.

Commit 3e1d4ec671 introduced a dependence on BinaryFormat, but CMakeLists.txt
was not updated to include it.
---
 llvm/lib/ExecutionEngine/Orc/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt b/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt
index f8bc82ba8d6f9b8..7565b4762bb9f7c 100644
--- a/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt
+++ b/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt
@@ -62,6 +62,7 @@ add_llvm_component_library(LLVMOrcJIT
   ${atomic_lib}
 
   LINK_COMPONENTS
+  BinaryFormat
   Core
   ExecutionEngine
   JITLink

From da25e08a4c8d5869ca66b1ff6ca3dbc291eea6e8 Mon Sep 17 00:00:00 2001
From: Brad Smith <brad@comstyle.com>
Date: Thu, 15 Aug 2024 20:50:12 -0400
Subject: [PATCH 110/441] [builitins] Only try to use getauxval on Linux
 (#104047)

OpenBSD now has sys/auxv.h but does not use getauxval.
---
 compiler-rt/lib/builtins/cpu_model/aarch64.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/compiler-rt/lib/builtins/cpu_model/aarch64.c b/compiler-rt/lib/builtins/cpu_model/aarch64.c
index ec4b08bfa89ac86..0dd397783b67f5a 100644
--- a/compiler-rt/lib/builtins/cpu_model/aarch64.c
+++ b/compiler-rt/lib/builtins/cpu_model/aarch64.c
@@ -45,7 +45,7 @@ _Bool __aarch64_have_lse_atomics
 #elif defined(__ANDROID__)
 #include "aarch64/hwcap.inc"
 #include "aarch64/lse_atomics/android.inc"
-#elif __has_include(<sys/auxv.h>)
+#elif defined(__linux__) && __has_include(<sys/auxv.h>)
 #include "aarch64/hwcap.inc"
 #include "aarch64/lse_atomics/getauxval.inc"
 #else
@@ -73,7 +73,7 @@ struct {
 #elif defined(__ANDROID__)
 #include "aarch64/fmv/mrs.inc"
 #include "aarch64/fmv/android.inc"
-#elif __has_include(<sys/auxv.h>)
+#elif defined(__linux__) && __has_include(<sys/auxv.h>)
 #include "aarch64/fmv/mrs.inc"
 #include "aarch64/fmv/getauxval.inc"
 #else

From 224b2800439ec0813a3fa861e98404624f18a6d7 Mon Sep 17 00:00:00 2001
From: Lang Hames <lhames@gmail.com>
Date: Fri, 16 Aug 2024 10:52:26 +1000
Subject: [PATCH 111/441] [ORC] Gate testcase for 3e1d4ec671c on x86-64 and
 aarch64 target support.

The testcase requires both of these targets.
---
 .../JITLink/Generic/MachO_universal_binaries.test               | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/llvm/test/ExecutionEngine/JITLink/Generic/MachO_universal_binaries.test b/llvm/test/ExecutionEngine/JITLink/Generic/MachO_universal_binaries.test
index 684068807f01ee9..f537aaee2ef4835 100644
--- a/llvm/test/ExecutionEngine/JITLink/Generic/MachO_universal_binaries.test
+++ b/llvm/test/ExecutionEngine/JITLink/Generic/MachO_universal_binaries.test
@@ -6,4 +6,6 @@
 # RUN:   -arch arm64 %t.arm64.o
 # RUN: llvm-jitlink -noexec -triple arm64-apple-macosx %t.universal.o
 #
+# REQUIRES: x86-registered-target && aarch64-registered-target
+#
 # Check that universal binaries don't raise "not a valid object file" errors.

From dfa13c010ff10b7bbbbf71c648de1b06ce58e34e Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Thu, 15 Aug 2024 17:58:31 -0700
Subject: [PATCH 112/441] [CodeGen] Use a range-based for loop (NFC) (#104408)

---
 llvm/lib/CodeGen/CodeGenPrepare.cpp | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp
index 1fb37fb8406ef45..ec5b7579c66c262 100644
--- a/llvm/lib/CodeGen/CodeGenPrepare.cpp
+++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp
@@ -7382,12 +7382,9 @@ bool CodeGenPrepare::tryToSinkFreeOperands(Instruction *I) {
     if (IsHugeFunc) {
       // Now we clone an instruction, its operands' defs may sink to this BB
       // now. So we put the operands defs' BBs into FreshBBs to do optimization.
-      for (unsigned I = 0; I < NI->getNumOperands(); ++I) {
-        auto *OpDef = dyn_cast<Instruction>(NI->getOperand(I));
-        if (!OpDef)
-          continue;
-        FreshBBs.insert(OpDef->getParent());
-      }
+      for (Value *Op : NI->operands())
+        if (auto *OpDef = dyn_cast<Instruction>(Op))
+          FreshBBs.insert(OpDef->getParent());
     }
 
     NewInstructions[UI] = NI;

From 3080c806712b59d5d9be6f9dedd02e0e02c32297 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Thu, 15 Aug 2024 17:58:46 -0700
Subject: [PATCH 113/441] [PowerPC] Use range-based for loops (NFC) (#104410)

---
 llvm/lib/Target/PowerPC/PPCISelLowering.cpp | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index 8ff9f5a5a991e08..fff39a37265f20f 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -3388,17 +3388,15 @@ static void updateForAIXShLibTLSModelOpt(TLSModel::Model &Model,
     // global variables (global variables taken as the first parameter to
     // Intrinsic::threadlocal_address).
     const Function &Func = DAG.getMachineFunction().getFunction();
-    for (Function::const_iterator BI = Func.begin(), BE = Func.end(); BI != BE;
-         ++BI)
-      for (BasicBlock::const_iterator II = BI->begin(), IE = BI->end();
-           II != IE; ++II)
-        if (II->getOpcode() == Instruction::Call)
-          if (const CallInst *CI = dyn_cast<const CallInst>(&*II))
+    for (const BasicBlock &BB : Func)
+      for (const Instruction &I : BB)
+        if (I.getOpcode() == Instruction::Call)
+          if (const CallInst *CI = dyn_cast<const CallInst>(&I))
             if (Function *CF = CI->getCalledFunction())
               if (CF->isDeclaration() &&
                   CF->getIntrinsicID() == Intrinsic::threadlocal_address)
                 if (const GlobalValue *GV =
-                        dyn_cast<GlobalValue>(II->getOperand(0))) {
+                        dyn_cast<GlobalValue>(I.getOperand(0))) {
                   TLSModel::Model GVModel = TM.getTLSModel(GV);
                   if (GVModel == TLSModel::LocalDynamic)
                     TLSGV.insert(GV);

From dca820951c86b79b0b26d92f7c715f3d4305f8e8 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Thu, 15 Aug 2024 17:59:10 -0700
Subject: [PATCH 114/441] [llvm] Use llvm::any_of (NFC) (#104443)

---
 llvm/lib/ProfileData/Coverage/CoverageMapping.cpp | 6 +++---
 llvm/lib/Target/AArch64/AArch64InstrInfo.cpp      | 4 ++--
 llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp          | 4 ++--
 llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp    | 6 +++---
 4 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp b/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp
index 21ce0ac17d61860..18643c6b44485ea 100644
--- a/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp
+++ b/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp
@@ -1533,9 +1533,9 @@ LineCoverageStats::LineCoverageStats(
 
   // if there is any starting segment at this line with a counter, it must be
   // mapped
-  Mapped |= std::any_of(
-      LineSegments.begin(), LineSegments.end(),
-      [](const auto *Seq) { return Seq->IsRegionEntry && Seq->HasCount; });
+  Mapped |= any_of(LineSegments, [](const auto *Seq) {
+    return Seq->IsRegionEntry && Seq->HasCount;
+  });
 
   if (!Mapped) {
     return;
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index d818d3ba51c59d2..3ea2ba235ca39c9 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -9074,8 +9074,8 @@ AArch64InstrInfo::getOutliningCandidateInfo(
     // link register.
     outliner::Candidate &FirstCand = RepeatedSequenceLocs[0];
     bool ModStackToSaveLR = false;
-    if (std::any_of(FirstCand.begin(), std::prev(FirstCand.end()),
-                    [](const MachineInstr &MI) { return MI.isCall(); }))
+    if (any_of(drop_end(FirstCand),
+               [](const MachineInstr &MI) { return MI.isCall(); }))
       ModStackToSaveLR = true;
 
     // Handle the last instruction separately. If this is a tail call, then the
diff --git a/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp b/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
index 3b9195b4e3d85fe..1199052ca97e9c7 100644
--- a/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
@@ -6074,8 +6074,8 @@ ARMBaseInstrInfo::getOutliningCandidateInfo(
     // check if the range contains a call.  These require a save + restore of
     // the link register.
     outliner::Candidate &FirstCand = RepeatedSequenceLocs[0];
-    if (std::any_of(FirstCand.begin(), std::prev(FirstCand.end()),
-                    [](const MachineInstr &MI) { return MI.isCall(); }))
+    if (any_of(drop_end(FirstCand),
+               [](const MachineInstr &MI) { return MI.isCall(); }))
       NumBytesToCreateFrame += Costs.SaveRestoreLROnStack;
 
     // Handle the last instruction separately.  If it is tail call, then the
diff --git a/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp b/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
index e54314cc7d00afa..c2951bf6dbf78f8 100644
--- a/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
+++ b/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
@@ -13090,9 +13090,9 @@ bool ARMAsmParser::isMnemonicVPTPredicable(StringRef Mnemonic,
       "vshrn",      "vsli",     "vsri",      "vstrb",      "vstrd",
       "vstrw",      "vsub"};
 
-  return std::any_of(
-      std::begin(predicable_prefixes), std::end(predicable_prefixes),
-      [&Mnemonic](const char *prefix) { return Mnemonic.starts_with(prefix); });
+  return any_of(predicable_prefixes, [&Mnemonic](const char *prefix) {
+    return Mnemonic.starts_with(prefix);
+  });
 }
 
 std::unique_ptr<ARMOperand> ARMAsmParser::defaultCondCodeOp() {

From 217f5804ca7fe492910b4a001b06c420e5de1ba6 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Thu, 15 Aug 2024 17:59:23 -0700
Subject: [PATCH 115/441] [Analysis] Use a range-based for loop (NFC) (#104445)

---
 llvm/lib/Analysis/ValueTracking.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index 014a703a0005240..9bd61f2996434af 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -6035,8 +6035,8 @@ Value *llvm::isBytewiseValue(Value *V, const DataLayout &DL) {
 
   if (isa<ConstantAggregate>(C)) {
     Value *Val = UndefInt8;
-    for (unsigned I = 0, E = C->getNumOperands(); I != E; ++I)
-      if (!(Val = Merge(Val, isBytewiseValue(C->getOperand(I), DL))))
+    for (Value *Op : C->operands())
+      if (!(Val = Merge(Val, isBytewiseValue(Op, DL))))
         return nullptr;
     return Val;
   }

From f861e33912732d3de05369cdff852968fcdcb3a0 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Thu, 15 Aug 2024 18:01:10 -0700
Subject: [PATCH 116/441] [Driver] Reject -Wa,-mrelax-relocations= for non-ELF

---
 clang/lib/Driver/ToolChains/Clang.cpp | 15 +++++++++++----
 clang/test/Driver/relax.c             |  3 +++
 2 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index 2b7241964a345da..daffa7cf96978a3 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -2582,6 +2582,7 @@ static void CollectArgsForIntegratedAssembler(Compilation &C,
   bool TakeNextArg = false;
 
   const llvm::Triple &Triple = C.getDefaultToolChain().getTriple();
+  bool IsELF = Triple.isOSBinFormatELF();
   bool Crel = false, ExperimentalCrel = false;
   bool UseRelaxRelocations = C.getDefaultToolChain().useRelaxRelocations();
   bool UseNoExecStack = false;
@@ -2621,10 +2622,16 @@ static void CollectArgsForIntegratedAssembler(Compilation &C,
         continue; // LLVM handles bigobj automatically
 
       auto Equal = Value.split('=');
-      auto checkArg = [&](std::initializer_list<const char *> Set) {
-        if (!llvm::is_contained(Set, Equal.second))
+      auto checkArg = [&](bool ValidTarget,
+                          std::initializer_list<const char *> Set) {
+        if (!ValidTarget) {
+          D.Diag(diag::err_drv_unsupported_opt_for_target)
+              << (Twine("-Wa,") + Equal.first + "=").str()
+              << Triple.getTriple();
+        } else if (!llvm::is_contained(Set, Equal.second)) {
           D.Diag(diag::err_drv_unsupported_option_argument)
               << (Twine("-Wa,") + Equal.first + "=").str() << Equal.second;
+        }
       };
       switch (C.getDefaultToolChain().getArch()) {
       default:
@@ -2634,7 +2641,7 @@ static void CollectArgsForIntegratedAssembler(Compilation &C,
         if (Equal.first == "-mrelax-relocations" ||
             Equal.first == "--mrelax-relocations") {
           UseRelaxRelocations = Equal.second == "yes";
-          checkArg({"yes", "no"});
+          checkArg(IsELF, {"yes", "no"});
           continue;
         }
         if (Value == "-msse2avx") {
@@ -2656,7 +2663,7 @@ static void CollectArgsForIntegratedAssembler(Compilation &C,
         if (Equal.first == "-mimplicit-it") {
           // Only store the value; the last value set takes effect.
           ImplicitIt = Equal.second;
-          checkArg({"always", "never", "arm", "thumb"});
+          checkArg(true, {"always", "never", "arm", "thumb"});
           continue;
         }
         if (Value == "-mthumb")
diff --git a/clang/test/Driver/relax.c b/clang/test/Driver/relax.c
index 9315a0045f1f438..93249cac7f40f88 100644
--- a/clang/test/Driver/relax.c
+++ b/clang/test/Driver/relax.c
@@ -8,3 +8,6 @@
 
 // RUN: not %clang -### --target=aarch64 -c -Wa,-mrelax-relocations=no %s 2>&1 | FileCheck %s --check-prefix=ERR2
 // ERR2: error: unsupported argument '-mrelax-relocations=no' to option '-Wa,'
+
+// RUN: not %clang -### --target=x86_64-apple-darwin -c -Wa,-mrelax-relocations=no %s 2>&1 | FileCheck %s --check-prefix=ERR3
+// ERR3: error: unsupported option '-Wa,-mrelax-relocations=' for target 'x86_64-apple-darwin'

From 9088ac1e8335d32bed275fbd33f0cd1b9bd400d6 Mon Sep 17 00:00:00 2001
From: Max Winkler <max.enrico.winkler@gmail.com>
Date: Thu, 15 Aug 2024 18:02:08 -0700
Subject: [PATCH 117/441] [Clang] [Sema] Error on reference types inside a
 union with msvc 1900+ (#102851)

Godbolt for reference: https://godbolt.org/z/ovKjvWc46

I can confirm that this extension is no longer valid in VS2017, VS2019
and VS2022 under `/permissive` and `/permissive-`

MSDN, https://learn.microsoft.com/en-us/cpp/porting/visual-cpp-what-s-new-2003-through-2015?view=msvc-170, says this extension was officially removed in VS2015.
---
 clang/docs/ReleaseNotes.rst                |  6 +++++
 clang/lib/Sema/SemaDecl.cpp                | 14 +++++++----
 clang/test/SemaCXX/MicrosoftExtensions.cpp | 28 +++++++++++-----------
 3 files changed, 29 insertions(+), 19 deletions(-)

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 598fccfe59bff1b..ffdd063ec99037a 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -321,6 +321,12 @@ Android Support
 Windows Support
 ^^^^^^^^^^^^^^^
 
+- Clang no longer allows references inside a union when emulating MSVC 1900+ even if `fms-extensions` is enabled.
+  Starting with VS2015, MSVC 1900, this Microsoft extension is no longer allowed and always results in an error.
+  Clang now follows the MSVC behavior in this scenario.
+  When `-fms-compatibility-version=18.00` or prior is set on the command line this Microsoft extension is still
+  allowed as VS2013 and prior allow it.
+
 LoongArch Support
 ^^^^^^^^^^^^^^^^^
 
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index 83a652691e2e034..d19a16cf2ba150f 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -18507,11 +18507,15 @@ FieldDecl *Sema::CheckFieldDecl(DeclarationName Name, QualType T,
       // the program is ill-formed, except when compiling with MSVC extensions
       // enabled.
       if (EltTy->isReferenceType()) {
-        Diag(NewFD->getLocation(), getLangOpts().MicrosoftExt ?
-                                    diag::ext_union_member_of_reference_type :
-                                    diag::err_union_member_of_reference_type)
-          << NewFD->getDeclName() << EltTy;
-        if (!getLangOpts().MicrosoftExt)
+        const bool HaveMSExt =
+            getLangOpts().MicrosoftExt &&
+            !getLangOpts().isCompatibleWithMSVC(LangOptions::MSVC2015);
+
+        Diag(NewFD->getLocation(),
+             HaveMSExt ? diag::ext_union_member_of_reference_type
+                       : diag::err_union_member_of_reference_type)
+            << NewFD->getDeclName() << EltTy;
+        if (!HaveMSExt)
           NewFD->setInvalidDecl();
       }
     }
diff --git a/clang/test/SemaCXX/MicrosoftExtensions.cpp b/clang/test/SemaCXX/MicrosoftExtensions.cpp
index 98c19975095bbee..7454a01158f6b4a 100644
--- a/clang/test/SemaCXX/MicrosoftExtensions.cpp
+++ b/clang/test/SemaCXX/MicrosoftExtensions.cpp
@@ -1,8 +1,10 @@
-// RUN: %clang_cc1 -std=c++17 %s -triple i686-pc-win32 -fsyntax-only -Wmicrosoft -Wc++11-extensions -Wno-long-long -verify -fms-extensions -fexceptions -fcxx-exceptions -DTEST1
-// RUN: %clang_cc1 -std=c++98 %s -triple i686-pc-win32 -fsyntax-only -Wmicrosoft -Wc++11-extensions -Wno-long-long -verify=expected,precxx17 -fms-extensions -fexceptions -fcxx-exceptions -DTEST1
-// RUN: %clang_cc1 -std=c++11 %s -triple i686-pc-win32 -fsyntax-only -Wmicrosoft -Wc++11-extensions -Wno-long-long -verify=expected,precxx17 -fms-extensions -fexceptions -fcxx-exceptions -DTEST1
-// RUN: %clang_cc1 -std=c++14 %s -triple i686-pc-win32 -fsyntax-only -Wmicrosoft -Wc++11-extensions -Wno-long-long -verify=expected,precxx17 -fexceptions -fcxx-exceptions -DTEST2
-// RUN: %clang_cc1 %s -triple i686-pc-win32 -fsyntax-only -std=c++11 -fms-compatibility -verify -DTEST3
+// RUN: %clang_cc1 -std=c++17 %s -triple i686-pc-win32 -fsyntax-only -Wmicrosoft -Wc++11-extensions -Wno-long-long -verify=expected,ms-union-ext -fms-extensions -fexceptions -fcxx-exceptions -DTEST1
+// RUN: %clang_cc1 -std=c++98 %s -triple i686-pc-win32 -fsyntax-only -Wmicrosoft -Wc++11-extensions -Wno-long-long -verify=expected,precxx17,ms-union-ext -fms-extensions -fexceptions -fcxx-exceptions -DTEST1
+// RUN: %clang_cc1 -std=c++11 %s -triple i686-pc-win32 -fsyntax-only -Wmicrosoft -Wc++11-extensions -Wno-long-long -verify=expected,precxx17,ms-union-ext -fms-extensions -fexceptions -fcxx-exceptions -DTEST1
+// RUN: %clang_cc1 -std=c++14 %s -triple i686-pc-win32 -fsyntax-only -Wmicrosoft -Wc++11-extensions -Wno-long-long -verify=expected,precxx17,ms-union-ext-disabled -fexceptions -fcxx-exceptions -DTEST2
+// RUN: %clang_cc1 %s -triple i686-pc-win32 -fsyntax-only -std=c++11 -fms-compatibility -verify=expected,ms-union-ext -DTEST3
+// RUN: %clang_cc1 -std=c++17 %s -triple i686-pc-win32 -fsyntax-only -Wmicrosoft -verify=ms-union-ext -fms-extensions -fms-compatibility-version=18.00
+// RUN: %clang_cc1 -std=c++17 %s -triple i686-pc-win32 -fsyntax-only -Wmicrosoft -verify=ms-union-ext-disabled -fms-extensions -fms-compatibility-version=19.00
 
 #if TEST1
 
@@ -384,11 +386,6 @@ void TestSP9() {
   c3.h(); // Overloaded unary op operand
 }
 
-union u {
-  int *i1;
-  int &i2;  // expected-warning {{union member 'i2' has reference type 'int &', which is a Microsoft extension}}
-};
-
 // Property getter using reference.
 struct SP11 {
   __declspec(property(get=GetV)) int V;
@@ -619,9 +616,12 @@ template<typename T> struct A {};
 template<typename T> struct B : A<A<T>> { A<T>::C::D d; }; // expected-warning {{implicit 'typename' is a C++20 extension}}
 }
 
-#else
-
-#error Unknown test mode
-
 #endif
 
+union u {
+    int *i1;
+
+    // ms-union-ext-warning@+2 {{union member 'i2' has reference type 'int &', which is a Microsoft extension}}
+    // ms-union-ext-disabled-error@+1 {{union member 'i2' has reference type 'int &'}}
+    int &i2;
+};

From 293aa56ebce2a5f5eb97308364645f5685944a42 Mon Sep 17 00:00:00 2001
From: Sergei Barannikov <barannikov88@gmail.com>
Date: Fri, 16 Aug 2024 04:28:11 +0300
Subject: [PATCH 118/441] [Sparc] Remove dead code (NFC) (#104264)

---
 llvm/lib/Target/Sparc/SparcAsmPrinter.cpp | 12 ++----------
 1 file changed, 2 insertions(+), 10 deletions(-)

diff --git a/llvm/lib/Target/Sparc/SparcAsmPrinter.cpp b/llvm/lib/Target/Sparc/SparcAsmPrinter.cpp
index 6855471840e9dbf..182779fa93e5033 100644
--- a/llvm/lib/Target/Sparc/SparcAsmPrinter.cpp
+++ b/llvm/lib/Target/Sparc/SparcAsmPrinter.cpp
@@ -50,8 +50,7 @@ namespace {
     StringRef getPassName() const override { return "Sparc Assembly Printer"; }
 
     void printOperand(const MachineInstr *MI, int opNum, raw_ostream &OS);
-    void printMemOperand(const MachineInstr *MI, int opNum, raw_ostream &OS,
-                         const char *Modifier = nullptr);
+    void printMemOperand(const MachineInstr *MI, int opNum, raw_ostream &OS);
 
     void emitFunctionBodyStart() override;
     void emitInstruction(const MachineInstr *MI) override;
@@ -401,16 +400,9 @@ void SparcAsmPrinter::printOperand(const MachineInstr *MI, int opNum,
 }
 
 void SparcAsmPrinter::printMemOperand(const MachineInstr *MI, int opNum,
-                                      raw_ostream &O, const char *Modifier) {
+                                      raw_ostream &O) {
   printOperand(MI, opNum, O);
 
-  // If this is an ADD operand, emit it like normal operands.
-  if (Modifier && !strcmp(Modifier, "arith")) {
-    O << ", ";
-    printOperand(MI, opNum+1, O);
-    return;
-  }
-
   if (MI->getOperand(opNum+1).isReg() &&
       MI->getOperand(opNum+1).getReg() == SP::G0)
     return;   // don't print "+%g0"

From 648f4d0658ab00cf1e95330c8811aaea9481a274 Mon Sep 17 00:00:00 2001
From: Krzysztof Drewniak <Krzysztof.Drewniak@amd.com>
Date: Thu, 15 Aug 2024 18:46:59 -0700
Subject: [PATCH 119/441] [ScalarizeMaskedMemIntr] Pre-commit tests for splat
 optimizations (#104527)

Commit tests that track the current behavior when the mask argument to a
llvm.masked.load or llvm.masked.store is a splat of a con-constant value
(that is, it does nothing special).
---
 .../X86/expand-masked-load.ll                 | 35 +++++++++++++++++++
 .../X86/expand-masked-store.ll                | 33 +++++++++++++++++
 2 files changed, 68 insertions(+)

diff --git a/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-load.ll b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-load.ll
index 8c95a630ebce767..9b1c59829b9ffb7 100644
--- a/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-load.ll
+++ b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-load.ll
@@ -58,6 +58,41 @@ define <2 x i64> @scalarize_v2i64_const_mask(ptr %p, <2 x i64> %passthru) {
   ret <2 x i64> %ret
 }
 
+; To be fixed: If the mask is the splat/broadcast of a non-constant value, use a
+; vector load
+define <2 x i64> @scalarize_v2i64_splat_mask(ptr %p, i1 %mask, <2 x i64> %passthrough) {
+; CHECK-LABEL: @scalarize_v2i64_splat_mask(
+; CHECK-NEXT:    [[MASK_VEC:%.*]] = insertelement <2 x i1> poison, i1 [[MASK:%.*]], i32 0
+; CHECK-NEXT:    [[MASK_SPLAT:%.*]] = shufflevector <2 x i1> [[MASK_VEC]], <2 x i1> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK_SPLAT]] to i2
+; CHECK-NEXT:    [[TMP1:%.*]] = and i2 [[SCALAR_MASK]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i2 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[TMP2]], label [[COND_LOAD:%.*]], label [[ELSE:%.*]]
+; CHECK:       cond.load:
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[P:%.*]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr [[TMP3]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x i64> [[PASSTHROUGH:%.*]], i64 [[TMP4]], i64 0
+; CHECK-NEXT:    br label [[ELSE]]
+; CHECK:       else:
+; CHECK-NEXT:    [[RES_PHI_ELSE:%.*]] = phi <2 x i64> [ [[TMP5]], [[COND_LOAD]] ], [ [[PASSTHROUGH]], [[TMP0:%.*]] ]
+; CHECK-NEXT:    [[TMP6:%.*]] = and i2 [[SCALAR_MASK]], -2
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ne i2 [[TMP6]], 0
+; CHECK-NEXT:    br i1 [[TMP7]], label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]
+; CHECK:       cond.load1:
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[P]], i32 1
+; CHECK-NEXT:    [[TMP9:%.*]] = load i64, ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <2 x i64> [[RES_PHI_ELSE]], i64 [[TMP9]], i64 1
+; CHECK-NEXT:    br label [[ELSE2]]
+; CHECK:       else2:
+; CHECK-NEXT:    [[RES_PHI_ELSE3:%.*]] = phi <2 x i64> [ [[TMP10]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ]
+; CHECK-NEXT:    ret <2 x i64> [[RES_PHI_ELSE3]]
+;
+  %mask.vec = insertelement <2 x i1> poison, i1 %mask, i32 0
+  %mask.splat = shufflevector <2 x i1> %mask.vec, <2 x i1> poison, <2 x i32> zeroinitializer
+  %ret = call <2 x i64> @llvm.masked.load.v2i64.p0(ptr %p, i32 8, <2 x i1> %mask.splat, <2 x i64> %passthrough)
+  ret <2 x i64> %ret
+}
+
 ; This use a byte sized but non power of 2 element size. This used to crash due to bad alignment calculation.
 define <2 x i24> @scalarize_v2i24(ptr %p, <2 x i1> %mask, <2 x i24> %passthru) {
 ; CHECK-LABEL: @scalarize_v2i24(
diff --git a/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-store.ll b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-store.ll
index f6e54bc6fe94d38..cd2815e67e67205 100644
--- a/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-store.ll
+++ b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-store.ll
@@ -56,4 +56,37 @@ define void @scalarize_v2i64_const_mask(ptr %p, <2 x i64> %data) {
   ret void
 }
 
+; To be fixed: If the mask is the splat/broadcast of a non-constant value, use a
+; vector store
+define void @scalarize_v2i64_splat_mask(ptr %p, <2 x i64> %data, i1 %mask) {
+; CHECK-LABEL: @scalarize_v2i64_splat_mask(
+; CHECK-NEXT:    [[MASK_VEC:%.*]] = insertelement <2 x i1> poison, i1 [[MASK:%.*]], i32 0
+; CHECK-NEXT:    [[MASK_SPLAT:%.*]] = shufflevector <2 x i1> [[MASK_VEC]], <2 x i1> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK_SPLAT]] to i2
+; CHECK-NEXT:    [[TMP1:%.*]] = and i2 [[SCALAR_MASK]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i2 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[TMP2]], label [[COND_STORE:%.*]], label [[ELSE:%.*]]
+; CHECK:       cond.store:
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x i64> [[DATA:%.*]], i64 0
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[P:%.*]], i32 0
+; CHECK-NEXT:    store i64 [[TMP3]], ptr [[TMP4]], align 8
+; CHECK-NEXT:    br label [[ELSE]]
+; CHECK:       else:
+; CHECK-NEXT:    [[TMP5:%.*]] = and i2 [[SCALAR_MASK]], -2
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne i2 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[TMP6]], label [[COND_STORE1:%.*]], label [[ELSE2:%.*]]
+; CHECK:       cond.store1:
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i64> [[DATA]], i64 1
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[P]], i32 1
+; CHECK-NEXT:    store i64 [[TMP7]], ptr [[TMP8]], align 8
+; CHECK-NEXT:    br label [[ELSE2]]
+; CHECK:       else2:
+; CHECK-NEXT:    ret void
+;
+  %mask.vec = insertelement <2 x i1> poison, i1 %mask, i32 0
+  %mask.splat = shufflevector <2 x i1> %mask.vec, <2 x i1> poison, <2 x i32> zeroinitializer
+  call void @llvm.masked.store.v2i64.p0(<2 x i64> %data, ptr %p, i32 8, <2 x i1> %mask.splat)
+  ret void
+}
+
 declare void @llvm.masked.store.v2i64.p0(<2 x i64>, ptr, i32, <2 x i1>)

From c20e7b6fe105e4dffc8ecff77b03a049d327c567 Mon Sep 17 00:00:00 2001
From: Xiang Li <python3kgae@outlook.com>
Date: Thu, 15 Aug 2024 19:44:07 -0700
Subject: [PATCH 120/441] [HLSL] update default validator version to 1.8.
 (#104040)

Change default validator version to 1.8 in Options.td.
Remove the default value set in HLSL.cpp.

Fixes #102963

This also helps #99910.
---
 clang/include/clang/Driver/Options.td                 |  2 +-
 clang/lib/Driver/ToolChains/HLSL.cpp                  |  8 --------
 clang/test/CodeGenHLSL/default_validator_version.hlsl | 10 ++++++++++
 3 files changed, 11 insertions(+), 9 deletions(-)
 create mode 100644 clang/test/CodeGenHLSL/default_validator_version.hlsl

diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index c9ed08c20fc04f9..cfd9e595c551784 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -8840,7 +8840,7 @@ def dxil_validator_version : Option<["/", "-"], "validator-version", KIND_SEPARA
   Visibility<[DXCOption, ClangOption, CC1Option]>,
   HelpText<"Override validator version for module. Format: <major.minor>;"
            "Default: DXIL.dll version or current internal version">,
-  MarshallingInfoString<TargetOpts<"DxilValidatorVersion">>;
+  MarshallingInfoString<TargetOpts<"DxilValidatorVersion">, "\"1.8\"">;
 def target_profile : DXCJoinedOrSeparate<"T">, MetaVarName<"<profile>">,
   HelpText<"Set target profile">,
   Values<"ps_6_0, ps_6_1, ps_6_2, ps_6_3, ps_6_4, ps_6_5, ps_6_6, ps_6_7,"
diff --git a/clang/lib/Driver/ToolChains/HLSL.cpp b/clang/lib/Driver/ToolChains/HLSL.cpp
index 8286e3be21803f2..ad44c2cfcd8115f 100644
--- a/clang/lib/Driver/ToolChains/HLSL.cpp
+++ b/clang/lib/Driver/ToolChains/HLSL.cpp
@@ -283,14 +283,6 @@ HLSLToolChain::TranslateArgs(const DerivedArgList &Args, StringRef BoundArch,
     DAL->append(A);
   }
 
-  // Add default validator version if not set.
-  // TODO: remove this once read validator version from validator.
-  if (!DAL->hasArg(options::OPT_dxil_validator_version)) {
-    const StringRef DefaultValidatorVer = "1.7";
-    DAL->AddSeparateArg(nullptr,
-                        Opts.getOption(options::OPT_dxil_validator_version),
-                        DefaultValidatorVer);
-  }
   if (!DAL->hasArg(options::OPT_O_Group)) {
     DAL->AddJoinedArg(nullptr, Opts.getOption(options::OPT_O), "3");
   }
diff --git a/clang/test/CodeGenHLSL/default_validator_version.hlsl b/clang/test/CodeGenHLSL/default_validator_version.hlsl
new file mode 100644
index 000000000000000..907526dd685a966
--- /dev/null
+++ b/clang/test/CodeGenHLSL/default_validator_version.hlsl
@@ -0,0 +1,10 @@
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -emit-llvm -xhlsl -o - %s | FileCheck %s
+
+// CHECK:!dx.valver = !{![[valver:[0-9]+]]}
+// CHECK:![[valver]] = !{i32 1, i32 8}
+
+float bar(float a, float b);
+
+float foo(float a, float b) {
+  return bar(a, b);
+}

From 5f01fda4acae211fcf67b603a7c90bc557bc2aff Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Thu, 15 Aug 2024 20:17:40 -0700
Subject: [PATCH 121/441] [Bitcode] Use range-based for loops (NFC) (#104534)

---
 llvm/lib/Bitcode/Writer/BitcodeWriter.cpp | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
index 460db6fc01de4a0..03d0537291dada1 100644
--- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -2820,9 +2820,9 @@ void ModuleBitcodeWriter::writeConstants(unsigned FirstVal, unsigned LastVal,
           Code = bitc::CST_CODE_CE_GEP_WITH_INRANGE;
           emitConstantRange(Record, *Range, /*EmitBitWidth=*/true);
         }
-        for (unsigned i = 0, e = CE->getNumOperands(); i != e; ++i) {
-          Record.push_back(VE.getTypeID(C->getOperand(i)->getType()));
-          Record.push_back(VE.getValueID(C->getOperand(i)));
+        for (const Value *Op : CE->operands()) {
+          Record.push_back(VE.getTypeID(Op->getType()));
+          Record.push_back(VE.getValueID(Op));
         }
         break;
       }
@@ -3078,8 +3078,8 @@ void ModuleBitcodeWriter::writeInstruction(const Instruction &I,
         if (!pushValueAndType(I.getOperand(0), InstID, Vals))
           AbbrevToUse = FUNCTION_INST_RET_VAL_ABBREV;
       } else {
-        for (unsigned i = 0, e = NumOperands; i != e; ++i)
-          pushValueAndType(I.getOperand(i), InstID, Vals);
+        for (const Value *Op : I.operands())
+          pushValueAndType(Op, InstID, Vals);
       }
     }
     break;

From 019fbcc4d728895f0decab13738b5d89d01b1436 Mon Sep 17 00:00:00 2001
From: Longsheng Mou <moulongsheng@huawei.com>
Date: Fri, 16 Aug 2024 11:19:59 +0800
Subject: [PATCH 122/441] [mlir][tosa] Add missing check for new_shape of
 `tosa.reshape` (#104394)

This patch adds check for new_shape of `tosa.reshape`. Tensor dimension
with size less than -1 is invalid. Fix #103062.
---
 mlir/lib/Dialect/Tosa/IR/TosaOps.cpp | 7 ++++++-
 mlir/test/Dialect/Tosa/invalid.mlir  | 8 ++++++++
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp b/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
index d4e49b6e3c044cb..c8e2b04eea0e22a 100644
--- a/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
+++ b/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp
@@ -990,11 +990,16 @@ llvm::LogicalResult tosa::ReshapeOp::verify() {
     return emitOpError() << "new shape does not match result rank";
 
   for (auto [newShapeDim, outputShapeDim] :
-       zip(getNewShape(), outputType.getShape()))
+       zip(getNewShape(), outputType.getShape())) {
     if (newShapeDim != -1 && outputShapeDim != ShapedType::kDynamic &&
         newShapeDim != outputShapeDim)
       return emitOpError() << "new shape is inconsistent with result shape";
 
+    if (newShapeDim != ShapedType::kDynamic && newShapeDim < -1)
+      return emitOpError() << "new shape has invalid tensor dimension size "
+                           << newShapeDim;
+  }
+
   if (inputType.hasStaticShape() && outputType.hasStaticShape()) {
     int64_t inputElementsNum = inputType.getNumElements();
     int64_t outputElementsNum = outputType.getNumElements();
diff --git a/mlir/test/Dialect/Tosa/invalid.mlir b/mlir/test/Dialect/Tosa/invalid.mlir
index e723aef3815ce63..806ba22e1bbe8c2 100644
--- a/mlir/test/Dialect/Tosa/invalid.mlir
+++ b/mlir/test/Dialect/Tosa/invalid.mlir
@@ -291,6 +291,14 @@ func.func @test_reshape_invalid_placeholders(%arg0 : tensor<?xf32>) -> () {
 
 // -----
 
+func.func @test_reshape_invalid_tensor_dim(%arg0 : tensor<4x?xf32>) -> () {
+  // expected-error@+1 {{'tosa.reshape' op new shape has invalid tensor dimension size -2}}
+  %0 = "tosa.reshape" (%arg0) {new_shape = array<i64: -2, -1>} : (tensor<4x?xf32>) -> tensor<?x4xf32>
+  return
+}
+
+// -----
+
 func.func @test_reverse_axis_out_of_range(%arg0 : tensor<13x21x3xf32>) -> () {
   // expected-error@+1 {{'tosa.reverse' op expect input tensor rank (3) to be larger than reverse axis (5)}}
   %0 = tosa.reverse %arg0 {axis = 5 : i32} : (tensor<13x21x3xf32>) -> tensor<?x?x?xi32>

From 5c3a3dc9eb0a70979ab456a69f2f40dab6515008 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Valentin=20Clement=20=28=E3=83=90=E3=83=AC=E3=83=B3?=
 =?UTF-8?q?=E3=82=BF=E3=82=A4=E3=83=B3=20=E3=82=AF=E3=83=AC=E3=83=A1?=
 =?UTF-8?q?=E3=83=B3=29?= <clementval@gmail.com>
Date: Thu, 15 Aug 2024 20:45:33 -0700
Subject: [PATCH 123/441] [flang][cuda] Add version in libCufRuntime name
 (#104506)

---
 flang/runtime/CUDA/CMakeLists.txt           | 9 +++++++--
 flang/unittests/Runtime/CUDA/CMakeLists.txt | 2 +-
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/flang/runtime/CUDA/CMakeLists.txt b/flang/runtime/CUDA/CMakeLists.txt
index c9a20ebcc82e072..81055b2737c0f15 100644
--- a/flang/runtime/CUDA/CMakeLists.txt
+++ b/flang/runtime/CUDA/CMakeLists.txt
@@ -8,7 +8,12 @@
 
 include_directories(${CUDAToolkit_INCLUDE_DIRS})
 
-add_flang_library(CufRuntime
+# libCufRuntime depends on a certain version of CUDA. To be able to have
+# multiple build of this library with different CUDA version, the version is
+# added to the library name.
+set(CUFRT_LIBNAME CufRuntime_cuda_${CUDAToolkit_VERSION_MAJOR})
+
+add_flang_library(${CUFRT_LIBNAME}
   allocator.cpp
   descriptor.cpp
 )
@@ -19,7 +24,7 @@ else()
   set(CUDA_RT_TARGET CUDA::cudart_static)
 endif()
 
-target_link_libraries(CufRuntime
+target_link_libraries(${CUFRT_LIBNAME}
   PRIVATE
   FortranRuntime
   ${CUDA_RT_TARGET}
diff --git a/flang/unittests/Runtime/CUDA/CMakeLists.txt b/flang/unittests/Runtime/CUDA/CMakeLists.txt
index 14b5c788719b870..ed0caece3d15db4 100644
--- a/flang/unittests/Runtime/CUDA/CMakeLists.txt
+++ b/flang/unittests/Runtime/CUDA/CMakeLists.txt
@@ -6,7 +6,7 @@ add_flang_unittest(FlangCufRuntimeTests
 
 target_link_libraries(FlangCufRuntimeTests
   PRIVATE
-  CufRuntime
+  CufRuntime_cuda_${CUDAToolkit_VERSION_MAJOR}
   FortranRuntime
 )
 

From e027e04f0139126b0a4af3aba0dbb31fdf22ea62 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Thu, 15 Aug 2024 21:07:22 -0700
Subject: [PATCH 124/441] [DAGCombiner] Don't let scalarizeBinOpOfSplats create
 illegal scalar MULHS/MULHU (#104518)

Type legalization lacks generic support for these operations. They are
normally only created when the type is legal. This scalarization case is
new.

We could update type legalization, but there some corner cases that make
it not straightforward. For example, if the promoted type isn't 2x the
narrow type we need to over promote.

Fixes #104480
---
 llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp |  4 +
 llvm/test/CodeGen/RISCV/rvv/pr104480.ll       | 93 +++++++++++++++++++
 2 files changed, 97 insertions(+)
 create mode 100644 llvm/test/CodeGen/RISCV/rvv/pr104480.ll

diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 25644c24855a62f..b21bea649e956b6 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -27140,6 +27140,10 @@ static SDValue scalarizeBinOpOfSplats(SDNode *N, SelectionDAG &DAG,
                       : TLI.getTypeToTransformTo(*DAG.getContext(), EltVT)))
     return SDValue();
 
+  // FIXME: Type legalization can't handle illegal MULHS/MULHU.
+  if ((Opcode == ISD::MULHS || Opcode == ISD::MULHU) && !TLI.isTypeLegal(EltVT))
+    return SDValue();
+
   SDValue IndexC = DAG.getVectorIdxConstant(Index0, DL);
   SDValue X = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Src0, IndexC);
   SDValue Y = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Src1, IndexC);
diff --git a/llvm/test/CodeGen/RISCV/rvv/pr104480.ll b/llvm/test/CodeGen/RISCV/rvv/pr104480.ll
new file mode 100644
index 000000000000000..93cf4d3766089a1
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/pr104480.ll
@@ -0,0 +1,93 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=riscv32 -mattr=+m,+v | FileCheck %s
+
+define <vscale x 4 x i16> @test_mulhs_promote(<vscale x 4 x i16> %broadcast.splatinsert, <vscale x 4 x i1> %0, <vscale x 4 x i1> %1) {
+; CHECK-LABEL: test_mulhs_promote:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vrgather.vi v9, v8, 0
+; CHECK-NEXT:    lui a0, 5
+; CHECK-NEXT:    addi a0, a0, 1366
+; CHECK-NEXT:    vmulh.vx v8, v9, a0
+; CHECK-NEXT:    vsrl.vi v10, v8, 15
+; CHECK-NEXT:    vadd.vv v8, v8, v10
+; CHECK-NEXT:    li a0, 3
+; CHECK-NEXT:    vnmsub.vx v8, a0, v9
+; CHECK-NEXT:    ret
+entry:
+  %broadcast.splat = shufflevector <vscale x 4 x i16> %broadcast.splatinsert, <vscale x 4 x i16> zeroinitializer, <vscale x 4 x i32> zeroinitializer
+  %2 = srem <vscale x 4 x i16> %broadcast.splat, shufflevector (<vscale x 4 x i16> insertelement (<vscale x 4 x i16> poison, i16 3, i64 0), <vscale x 4 x i16> poison, <vscale x 4 x i32> zeroinitializer)
+ ret <vscale x 4 x i16> %2
+}
+
+define <vscale x 4 x i16> @test_mulhu_promote(<vscale x 4 x i16> %broadcast.splatinsert, <vscale x 4 x i1> %0, <vscale x 4 x i1> %1) {
+; CHECK-LABEL: test_mulhu_promote:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vrgather.vi v9, v8, 0
+; CHECK-NEXT:    lui a0, 1048571
+; CHECK-NEXT:    addi a0, a0, -1365
+; CHECK-NEXT:    vmulhu.vx v8, v9, a0
+; CHECK-NEXT:    vsrl.vi v8, v8, 1
+; CHECK-NEXT:    li a0, 3
+; CHECK-NEXT:    vnmsub.vx v8, a0, v9
+; CHECK-NEXT:    ret
+entry:
+  %broadcast.splat = shufflevector <vscale x 4 x i16> %broadcast.splatinsert, <vscale x 4 x i16> zeroinitializer, <vscale x 4 x i32> zeroinitializer
+  %2 = urem <vscale x 4 x i16> %broadcast.splat, shufflevector (<vscale x 4 x i16> insertelement (<vscale x 4 x i16> poison, i16 3, i64 0), <vscale x 4 x i16> poison, <vscale x 4 x i32> zeroinitializer)
+ ret <vscale x 4 x i16> %2
+}
+
+define <vscale x 4 x i64> @test_mulhs_expand(<vscale x 4 x i64> %broadcast.splatinsert, <vscale x 4 x i1> %0, <vscale x 4 x i1> %1) {
+; CHECK-LABEL: test_mulhs_expand:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    lui a0, 349525
+; CHECK-NEXT:    addi a1, a0, 1365
+; CHECK-NEXT:    sw a1, 12(sp)
+; CHECK-NEXT:    addi a0, a0, 1366
+; CHECK-NEXT:    sw a0, 8(sp)
+; CHECK-NEXT:    addi a0, sp, 8
+; CHECK-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
+; CHECK-NEXT:    vlse64.v v12, (a0), zero
+; CHECK-NEXT:    vrgather.vi v16, v8, 0
+; CHECK-NEXT:    vmulh.vv v8, v16, v12
+; CHECK-NEXT:    li a0, 63
+; CHECK-NEXT:    vsrl.vx v12, v8, a0
+; CHECK-NEXT:    vadd.vv v8, v8, v12
+; CHECK-NEXT:    li a0, 3
+; CHECK-NEXT:    vnmsub.vx v8, a0, v16
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+entry:
+  %broadcast.splat = shufflevector <vscale x 4 x i64> %broadcast.splatinsert, <vscale x 4 x i64> zeroinitializer, <vscale x 4 x i32> zeroinitializer
+  %2 = srem <vscale x 4 x i64> %broadcast.splat, shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 3, i64 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
+ ret <vscale x 4 x i64> %2
+}
+
+define <vscale x 4 x i64> @test_mulhu_expand(<vscale x 4 x i64> %broadcast.splatinsert, <vscale x 4 x i1> %0, <vscale x 4 x i1> %1) {
+; CHECK-LABEL: test_mulhu_expand:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    lui a0, 699051
+; CHECK-NEXT:    addi a1, a0, -1366
+; CHECK-NEXT:    sw a1, 12(sp)
+; CHECK-NEXT:    addi a0, a0, -1365
+; CHECK-NEXT:    sw a0, 8(sp)
+; CHECK-NEXT:    addi a0, sp, 8
+; CHECK-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
+; CHECK-NEXT:    vlse64.v v12, (a0), zero
+; CHECK-NEXT:    vrgather.vi v16, v8, 0
+; CHECK-NEXT:    vmulhu.vv v8, v16, v12
+; CHECK-NEXT:    vsrl.vi v8, v8, 1
+; CHECK-NEXT:    li a0, 3
+; CHECK-NEXT:    vnmsub.vx v8, a0, v16
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
+entry:
+  %broadcast.splat = shufflevector <vscale x 4 x i64> %broadcast.splatinsert, <vscale x 4 x i64> zeroinitializer, <vscale x 4 x i32> zeroinitializer
+  %2 = urem <vscale x 4 x i64> %broadcast.splat, shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 3, i64 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
+ ret <vscale x 4 x i64> %2
+}

From fe55c34d19628304e0ca6a0e14a0b786b93d0e02 Mon Sep 17 00:00:00 2001
From: Zhaoshi Zheng <zhaoshiz@quicinc.com>
Date: Thu, 15 Aug 2024 21:37:51 -0700
Subject: [PATCH 125/441] [MLIR][test] Run SVE and SME Integration tests using
 qemu-aarch64 (#101568)

To run integration tests using qemu-aarch64 on x64 host, below flags are
added to the cmake command when building mlir/llvm:

      -DMLIR_INCLUDE_INTEGRATION_TESTS=ON \
      -DMLIR_RUN_ARM_SVE_TESTS=ON \
      -DMLIR_RUN_ARM_SME_TESTS=ON \
      -DARM_EMULATOR_EXECUTABLE="<...>/qemu-aarch64" \
      -DARM_EMULATOR_OPTIONS="-L /usr/aarch64-linux-gnu" \

-DARM_EMULATOR_MLIR_CPU_RUNNER_EXECUTABLE="<llvm_arm64_build_top>/bin/mlir-cpu-runner-arm64"
\
      -DARM_EMULATOR_LLI_EXECUTABLE="<llvm_arm64_build_top>/bin/lli" \
      -DARM_EMULATOR_UTILS_LIB_DIR="<llvm_arm64_build_top>/lib"

The last three above are prebuilt on, or cross-built for, an aarch64
host.

This patch introduced substittutions of "%native_mlir_runner_utils" etc. and use
them in SVE/SME integration tests. When configured to run using qemu-aarch64,
mlir runtime util libs will be loaded from ARM_EMULATOR_UTILS_LIB_DIR, if set.

Some tests marked with 'UNSUPPORTED: target=aarch64{{.*}}' are still run
when configured with ARM_EMULATOR_EXECUTABLE and the default target is
not aarch64.
A lit config feature 'mlir_arm_emulator' is added in
mlir/test/lit.site.cfg.py.in and to UNSUPPORTED list of such tests.
---
 .../Dialect/Linalg/CPU/ArmSME/fill-2d.mlir    |  2 +-
 .../Linalg/CPU/ArmSME/matmul-transpose-a.mlir |  2 +-
 .../Dialect/Linalg/CPU/ArmSME/matmul.mlir     |  2 +-
 .../ArmSME/multi-tile-matmul-mixed-types.mlir |  2 +-
 .../Linalg/CPU/ArmSME/multi-tile-matmul.mlir  |  2 +-
 .../Linalg/CPU/ArmSME/use-too-many-tiles.mlir |  2 +-
 .../Linalg/CPU/ArmSVE/1d-depthwise-conv.mlir  |  2 +-
 .../Dialect/Linalg/CPU/ArmSVE/fill-1d.mlir    |  2 +-
 .../Dialect/Linalg/CPU/ArmSVE/matmul.mlir     |  2 +-
 .../Dialect/Linalg/CPU/ArmSVE/reduce_1d.mlir  |  2 +-
 .../Dialect/Linalg/CPU/ArmSVE/reduce_2d.mlir  |  2 +-
 .../Dialect/SparseTensor/CPU/block.mlir       |  3 +-
 .../SparseTensor/CPU/block_majors.mlir        |  3 +-
 .../SparseTensor/CPU/concatenate_dim_0.mlir   |  3 +-
 .../CPU/concatenate_dim_0_permute.mlir        |  3 +-
 .../SparseTensor/CPU/concatenate_dim_1.mlir   |  3 +-
 .../CPU/concatenate_dim_1_permute.mlir        |  3 +-
 .../SparseTensor/CPU/dense_output.mlir        |  3 +-
 .../SparseTensor/CPU/dense_output_bf16.mlir   |  3 +-
 .../SparseTensor/CPU/dense_output_f16.mlir    |  3 +-
 .../SparseTensor/CPU/dual_sparse_conv_2d.mlir |  3 +-
 .../CPU/iterator-based-sqsum.mlir             |  3 +-
 .../Dialect/SparseTensor/CPU/lit.local.cfg    |  3 +-
 .../CPU/padded_sparse_conv_2d.mlir            |  3 +-
 .../Dialect/SparseTensor/CPU/reshape_dot.mlir |  3 +-
 .../Dialect/SparseTensor/CPU/sparse_abs.mlir  |  3 +-
 .../SparseTensor/CPU/sparse_binary.mlir       |  3 +-
 .../SparseTensor/CPU/sparse_block3d.mlir      |  3 +-
 .../SparseTensor/CPU/sparse_block_matmul.mlir |  3 +-
 .../Dialect/SparseTensor/CPU/sparse_cast.mlir |  3 +-
 .../Dialect/SparseTensor/CPU/sparse_cmp.mlir  |  3 +-
 .../SparseTensor/CPU/sparse_codegen_dim.mlir  |  3 +-
 .../CPU/sparse_codegen_foreach.mlir           |  3 +-
 .../CPU/sparse_collapse_shape.mlir            |  3 +-
 .../SparseTensor/CPU/sparse_complex32.mlir    |  3 +-
 .../SparseTensor/CPU/sparse_complex64.mlir    |  3 +-
 .../SparseTensor/CPU/sparse_complex_ops.mlir  |  3 +-
 .../CPU/sparse_constant_to_sparse_tensor.mlir |  3 +-
 .../CPU/sparse_conv_1d_nwc_wcf.mlir           |  3 +-
 .../SparseTensor/CPU/sparse_conv_2d.mlir      |  3 +-
 .../SparseTensor/CPU/sparse_conv_2d_55.mlir   |  3 +-
 .../CPU/sparse_conv_2d_nchw_fchw.mlir         |  3 +-
 .../CPU/sparse_conv_2d_nhwc_hwcf.mlir         |  3 +-
 .../SparseTensor/CPU/sparse_conv_3d.mlir      |  3 +-
 .../CPU/sparse_conv_3d_ndhwc_dhwcf.mlir       |  3 +-
 .../SparseTensor/CPU/sparse_conversion.mlir   |  3 +-
 .../CPU/sparse_conversion_block.mlir          |  3 +-
 .../CPU/sparse_conversion_dyn.mlir            |  3 +-
 .../CPU/sparse_conversion_element.mlir        |  3 +-
 .../CPU/sparse_conversion_ptr.mlir            |  3 +-
 .../CPU/sparse_conversion_sparse2dense.mlir   |  3 +-
 .../CPU/sparse_conversion_sparse2sparse.mlir  |  3 +-
 .../SparseTensor/CPU/sparse_coo_test.mlir     |  3 +-
 .../CPU/sparse_dilated_conv_2d_nhwc_hwcf.mlir |  3 +-
 .../Dialect/SparseTensor/CPU/sparse_dot.mlir  |  3 +-
 .../Dialect/SparseTensor/CPU/sparse_ds.mlir   |  3 +-
 .../SparseTensor/CPU/sparse_empty.mlir        |  3 +-
 .../SparseTensor/CPU/sparse_expand.mlir       |  3 +-
 .../SparseTensor/CPU/sparse_expand_shape.mlir |  3 +-
 .../CPU/sparse_filter_conv2d.mlir             |  3 +-
 .../SparseTensor/CPU/sparse_flatten.mlir      |  3 +-
 .../CPU/sparse_foreach_slices.mlir            |  3 +-
 .../SparseTensor/CPU/sparse_generate.mlir     |  3 +-
 .../SparseTensor/CPU/sparse_index.mlir        |  3 +-
 .../SparseTensor/CPU/sparse_index_dense.mlir  |  3 +-
 .../SparseTensor/CPU/sparse_insert_1d.mlir    |  3 +-
 .../SparseTensor/CPU/sparse_insert_2d.mlir    |  3 +-
 .../SparseTensor/CPU/sparse_insert_3d.mlir    |  3 +-
 .../SparseTensor/CPU/sparse_loose.mlir        |  3 +-
 .../SparseTensor/CPU/sparse_matmul.mlir       |  3 +-
 .../SparseTensor/CPU/sparse_matmul_slice.mlir |  3 +-
 .../SparseTensor/CPU/sparse_matrix_ops.mlir   |  3 +-
 .../SparseTensor/CPU/sparse_matvec.mlir       |  3 +-
 .../SparseTensor/CPU/sparse_mttkrp.mlir       |  3 +-
 .../SparseTensor/CPU/sparse_out_mult_elt.mlir |  3 +-
 .../CPU/sparse_out_reduction.mlir             |  3 +-
 .../SparseTensor/CPU/sparse_out_simple.mlir   |  3 +-
 .../Dialect/SparseTensor/CPU/sparse_pack.mlir |  3 +-
 .../SparseTensor/CPU/sparse_pack_d.mlir       |  3 +-
 .../SparseTensor/CPU/sparse_permute.mlir      |  3 +-
 .../SparseTensor/CPU/sparse_pooling_nhwc.mlir |  3 +-
 .../SparseTensor/CPU/sparse_print.mlir        |  3 +-
 .../SparseTensor/CPU/sparse_print_3d.mlir     |  3 +-
 .../CPU/sparse_quantized_matmul.mlir          |  3 +-
 .../SparseTensor/CPU/sparse_re_im.mlir        |  3 +-
 .../CPU/sparse_reduce_custom.mlir             |  3 +-
 .../CPU/sparse_reduce_custom_prod.mlir        |  3 +-
 .../CPU/sparse_reduce_custom_sum.mlir         |  3 +-
 .../SparseTensor/CPU/sparse_reductions.mlir   |  3 +-
 .../CPU/sparse_reductions_min.mlir            |  3 +-
 .../CPU/sparse_reductions_prod.mlir           |  3 +-
 .../SparseTensor/CPU/sparse_reshape.mlir      |  3 +-
 .../CPU/sparse_rewrite_push_back.mlir         |  3 +-
 .../CPU/sparse_rewrite_sort_coo.mlir          |  3 +-
 .../CPU/sparse_sampled_matmul.mlir            |  3 +-
 .../CPU/sparse_sampled_mm_fusion.mlir         |  3 +-
 .../SparseTensor/CPU/sparse_scale.mlir        |  3 +-
 .../SparseTensor/CPU/sparse_scf_nested.mlir   |  3 +-
 .../SparseTensor/CPU/sparse_select.mlir       |  3 +-
 .../CPU/sparse_semiring_select.mlir           |  3 +-
 .../Dialect/SparseTensor/CPU/sparse_sign.mlir |  3 +-
 .../SparseTensor/CPU/sparse_sorted_coo.mlir   |  3 +-
 .../Dialect/SparseTensor/CPU/sparse_spmm.mlir |  3 +-
 .../SparseTensor/CPU/sparse_storage.mlir      |  3 +-
 .../CPU/sparse_strided_conv_2d_nhwc_hwcf.mlir |  3 +-
 .../Dialect/SparseTensor/CPU/sparse_sum.mlir  |  3 +-
 .../SparseTensor/CPU/sparse_sum_bf16.mlir     |  5 +--
 .../SparseTensor/CPU/sparse_sum_c32.mlir      |  3 +-
 .../SparseTensor/CPU/sparse_sum_f16.mlir      |  3 +-
 .../Dialect/SparseTensor/CPU/sparse_tanh.mlir |  5 +--
 .../SparseTensor/CPU/sparse_tensor_mul.mlir   |  3 +-
 .../SparseTensor/CPU/sparse_tensor_ops.mlir   |  3 +-
 .../SparseTensor/CPU/sparse_transpose.mlir    |  3 +-
 .../CPU/sparse_transpose_coo.mlir             |  3 +-
 .../SparseTensor/CPU/sparse_unary.mlir        |  3 +-
 .../SparseTensor/CPU/sparse_vector_ops.mlir   |  3 +-
 .../CPU/ArmSME/load-store-128-bit-tile.mlir   |  2 +-
 .../Vector/CPU/ArmSME/load-vertical.mlir      |  2 +-
 .../CPU/ArmSME/multi-tile-transpose.mlir      |  2 +-
 .../CPU/ArmSME/outerproduct-f16f16f32.mlir    |  2 +-
 .../Vector/CPU/ArmSME/outerproduct-f32.mlir   |  2 +-
 .../Vector/CPU/ArmSME/outerproduct-f64.mlir   |  2 +-
 .../CPU/ArmSME/outerproduct-i8i8i32.mlir      |  2 +-
 .../Dialect/Vector/CPU/ArmSME/ssve.mlir       |  2 +-
 .../Vector/CPU/ArmSME/test-setArmSVLBits.mlir |  2 +-
 .../Dialect/Vector/CPU/ArmSME/tile-fill.mlir  |  2 +-
 .../Vector/CPU/ArmSME/transfer-read-2d.mlir   |  2 +-
 .../Vector/CPU/ArmSME/transfer-write-2d.mlir  |  2 +-
 .../Dialect/Vector/CPU/ArmSME/transpose.mlir  |  2 +-
 .../Vector/CPU/ArmSME/vector-load-store.mlir  |  2 +-
 .../Dialect/Vector/CPU/ArmSME/vector-ops.mlir |  2 +-
 .../ArmSVE/arrays-of-scalable-vectors.mlir    |  2 +-
 .../Vector/CPU/ArmSVE/contraction.mlir        |  2 +-
 .../CPU/ArmSVE/scalable-interleave.mlir       |  2 +-
 .../Dialect/Vector/CPU/ArmSVE/sve.mlir        |  2 +-
 .../ArmSVE/test-scalable-deinterleave.mlir    |  2 +-
 .../Vector/CPU/ArmSVE/test-setArmVLBits.mlir  |  2 +-
 mlir/test/Integration/lit.local.cfg           | 33 ++++++++++++-------
 mlir/test/lit.site.cfg.py.in                  |  5 +++
 139 files changed, 270 insertions(+), 151 deletions(-)

diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/fill-2d.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/fill-2d.mlir
index 881e2799b5b06b0..ed4342b7d50afa2 100644
--- a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/fill-2d.mlir
+++ b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/fill-2d.mlir
@@ -6,7 +6,7 @@
 // RUN: %mcr_aarch64_cmd \
 // RUN:   -e=entry -entry-point-result=void \
 // RUN:   -march=aarch64 -mattr="+sve,+sme" \
-// RUN:   -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils,%arm_sme_abi_shlib | \
+// RUN:   -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils,%native_arm_sme_abi_shlib | \
 // RUN: FileCheck %s
 
 func.func @entry() {
diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/matmul-transpose-a.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/matmul-transpose-a.mlir
index a8b6457d64becce..a57348a543c3cfa 100644
--- a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/matmul-transpose-a.mlir
+++ b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/matmul-transpose-a.mlir
@@ -5,7 +5,7 @@
 // RUN: %mcr_aarch64_cmd \
 // RUN:   -e=main -entry-point-result=void \
 // RUN:   -march=aarch64 -mattr="+sve,+sme" \
-// RUN:   -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils,%arm_sme_abi_shlib | \
+// RUN:   -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils,%native_arm_sme_abi_shlib | \
 // RUN: FileCheck %s
 
 func.func @matmul_transpose_a(%A : tensor<?x?xf32>, %B : tensor<?x?xf32>, %C : tensor<?x?xf32>) {
diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/matmul.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/matmul.mlir
index 091665223188f6b..79c9fcac70604bf 100644
--- a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/matmul.mlir
+++ b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/matmul.mlir
@@ -4,7 +4,7 @@
 // RUN: %mcr_aarch64_cmd \
 // RUN:   -e=main -entry-point-result=void \
 // RUN:   -march=aarch64 -mattr="+sve,+sme" \
-// RUN:   -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils,%arm_sme_abi_shlib | \
+// RUN:   -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils,%native_arm_sme_abi_shlib | \
 // RUN: FileCheck %s
 
 func.func @matmul(%A : tensor<?x?xf32>, %B : tensor<?x?xf32>, %C : tensor<?x?xf32>) {
diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/multi-tile-matmul-mixed-types.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/multi-tile-matmul-mixed-types.mlir
index 5784ecbbe40141d..8a4c8688d647482 100644
--- a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/multi-tile-matmul-mixed-types.mlir
+++ b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/multi-tile-matmul-mixed-types.mlir
@@ -6,7 +6,7 @@
 // RUN: %mcr_aarch64_cmd \
 // RUN:   -e=main -entry-point-result=void \
 // RUN:   -march=aarch64 -mattr="+sve,+sme" \
-// RUN:   -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils,%arm_sme_abi_shlib,%mlir_arm_runner_utils | \
+// RUN:   -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils,%native_arm_sme_abi_shlib,%native_mlir_arm_runner_utils | \
 // RUN: FileCheck %s
 
 /// This is very similar to the SME multi-tile-matmul.mlir test, except that it
diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/multi-tile-matmul.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/multi-tile-matmul.mlir
index 03a7d25cffa7677..243f9e5cde9f53d 100644
--- a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/multi-tile-matmul.mlir
+++ b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/multi-tile-matmul.mlir
@@ -6,7 +6,7 @@
 // RUN: %mcr_aarch64_cmd \
 // RUN:   -e=main -entry-point-result=void \
 // RUN:   -march=aarch64 -mattr="+sve,+sme" \
-// RUN:   -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils,%arm_sme_abi_shlib,%mlir_arm_runner_utils | \
+// RUN:   -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils,%native_arm_sme_abi_shlib,%native_mlir_arm_runner_utils | \
 // RUN: FileCheck %s
 
 /// This is very similar to the SME matmul.mlir test, except that it uses a tile
diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/use-too-many-tiles.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/use-too-many-tiles.mlir
index 0b84b5a71b39bba..05eee31ccc6fafa 100644
--- a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/use-too-many-tiles.mlir
+++ b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/use-too-many-tiles.mlir
@@ -3,7 +3,7 @@
 // RUN: %mcr_aarch64_cmd \
 // RUN:   -e=main -entry-point-result=void \
 // RUN:   -march=aarch64 -mattr="+sve,+sme" \
-// RUN:   -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils,%arm_sme_abi_shlib | \
+// RUN:   -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils,%native_arm_sme_abi_shlib | \
 // RUN: FileCheck %s
 
 /// This function uses too many tiles! There's only two i16 tiles (ZA0.H and
diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/1d-depthwise-conv.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/1d-depthwise-conv.mlir
index 56dce1c65401702..31931475525259b 100644
--- a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/1d-depthwise-conv.mlir
+++ b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/1d-depthwise-conv.mlir
@@ -4,7 +4,7 @@
 // DEFINE:    -convert-vector-to-llvm="enable-arm-sve" -test-lower-to-llvm -o %t
 // DEFINE: %{entry_point} = conv
 // DEFINE: %{run} = %mcr_aarch64_cmd %t -e %{entry_point} -entry-point-result=void --march=aarch64 --mattr="+sve"\
-// DEFINE:    -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils
+// DEFINE:    -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 
 // RUN: rm -f %t && %{compile} && %{run} | FileCheck %s
 
diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/fill-1d.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/fill-1d.mlir
index 44b555c8c3ad5e5..6099eea1910d959 100644
--- a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/fill-1d.mlir
+++ b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/fill-1d.mlir
@@ -1,5 +1,5 @@
 // RUN: mlir-opt %s -transform-interpreter -test-transform-dialect-erase-schedule -lower-vector-mask -one-shot-bufferize -buffer-deallocation-pipeline -test-lower-to-llvm | \
-// RUN: %mcr_aarch64_cmd -e=entry -entry-point-result=void --march=aarch64 --mattr="+sve" -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils | \
+// RUN: %mcr_aarch64_cmd -e=entry -entry-point-result=void --march=aarch64 --mattr="+sve" -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils | \
 // RUN: FileCheck %s
 
 func.func @entry() {
diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/matmul.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/matmul.mlir
index edb9de922808c08..bcfeef772743b78 100644
--- a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/matmul.mlir
+++ b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/matmul.mlir
@@ -4,7 +4,7 @@
 // DEFINE:    -convert-vector-to-llvm="enable-arm-sve" -test-lower-to-llvm -o %t
 // DEFINE: %{entry_point} = matmul_f32
 // DEFINE: %{run} = %mcr_aarch64_cmd %t -e %{entry_point} -entry-point-result=void --march=aarch64 --mattr="+sve"\
-// DEFINE:    -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils
+// DEFINE:    -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 
 // RUN: %{compile}
 
diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/reduce_1d.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/reduce_1d.mlir
index 7cdb35918c4c0fc..3090e921553c613 100644
--- a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/reduce_1d.mlir
+++ b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/reduce_1d.mlir
@@ -4,7 +4,7 @@
 // DEFINE:    -convert-vector-to-llvm="enable-arm-sve" -test-lower-to-llvm -o %t
 // DEFINE: %{entry_point} = reduce_1d_f32
 // DEFINE: %{run} = %mcr_aarch64_cmd %t -e %{entry_point} -entry-point-result=void --march=aarch64 --mattr="+sve"\
-// DEFINE:    -shared-libs=%mlir_native_utils_lib_dir/libmlir_runner_utils%shlibext,%mlir_native_utils_lib_dir/libmlir_c_runner_utils%shlibext
+// DEFINE:    -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 
 // RUN: %{compile}
 
diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/reduce_2d.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/reduce_2d.mlir
index bcfe12e374b4e2a..fc0fd40b4d2652c 100644
--- a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/reduce_2d.mlir
+++ b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSVE/reduce_2d.mlir
@@ -4,7 +4,7 @@
 // DEFINE:    -convert-vector-to-llvm="enable-arm-sve" -test-lower-to-llvm -o %t
 // DEFINE: %{entry_point} = reduce_2d_f32
 // DEFINE: %{run} = %mcr_aarch64_cmd %t -e %{entry_point} -entry-point-result=void --march=aarch64 --mattr="+sve"\
-// DEFINE:    -shared-libs=%mlir_native_utils_lib_dir/libmlir_runner_utils%shlibext,%mlir_native_utils_lib_dir/libmlir_c_runner_utils%shlibext
+// DEFINE:    -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 
 // RUN: %{compile}
 
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/block.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/block.mlir
index ab4fd0e30d6547b..8cea9184d0f9c56 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/block.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/block.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/block_majors.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/block_majors.mlir
index caa0d6a71ed37a9..c2162ba56367d4b 100755
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/block_majors.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/block_majors.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_0.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_0.mlir
index 7edb76cc8045dd1..9a9c545445ef08f 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_0.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_0.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_0_permute.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_0_permute.mlir
index d17e110e2c2d9da..e004fda1f3a259c 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_0_permute.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_0_permute.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_1.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_1.mlir
index c2a4e95e7922e33..7f25338d81fea35 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_1.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_1.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_1_permute.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_1_permute.mlir
index 8fe7e08a66d3813..15e0d8615fe56c9 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_1_permute.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/concatenate_dim_1_permute.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output.mlir
index d00d4c87f9bd4d7..2653fd393c9b0b6 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output_bf16.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output_bf16.mlir
index 49f182ddb1d441a..c59993abb16735c 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output_bf16.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output_bf16.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output_f16.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output_f16.mlir
index cc2a3733c86333b..1eb334750383404 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output_f16.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output_f16.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/dual_sparse_conv_2d.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/dual_sparse_conv_2d.mlir
index f33a3abc7a5f7e3..9602c198082f86a 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/dual_sparse_conv_2d.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/dual_sparse_conv_2d.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/iterator-based-sqsum.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/iterator-based-sqsum.mlir
index 6424c35b9c5d5f6..6d03565f8f7b2a0 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/iterator-based-sqsum.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/iterator-based-sqsum.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/lit.local.cfg b/mlir/test/Integration/Dialect/SparseTensor/CPU/lit.local.cfg
index 6e07eb87987a4a5..9caae19726cb114 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/lit.local.cfg
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/lit.local.cfg
@@ -1,7 +1,7 @@
 import sys
 from lit.llvm import llvm_config
 
-# FIXME: %mlir_native_utils_lib_dir is set incorrectly on Windows
+# No JIT on win32.
 if sys.platform == "win32":
     config.unsupported = True
 
@@ -14,4 +14,3 @@ if config.mlir_run_arm_sve_tests:
 else:
     config.substitutions.append(("%ENABLE_VLA", "false"))
     config.substitutions.append(("%VLA_ARCH_ATTR_OPTIONS", ""))
-    config.substitutions.append(("%mlir_native_utils_lib_dir", config.mlir_lib_dir))
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/padded_sparse_conv_2d.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/padded_sparse_conv_2d.mlir
index 50dd989416e2a0e..f25a71a6cae148c 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/padded_sparse_conv_2d.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/padded_sparse_conv_2d.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/reshape_dot.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/reshape_dot.mlir
index 73dddefb0e4aa5e..b29c4336b40d0fa 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/reshape_dot.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/reshape_dot.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_abs.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_abs.mlir
index 707c6c34d8dce3f..02950fc81228b2e 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_abs.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_abs.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_binary.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_binary.mlir
index 69be2ee752216a7..795a41797d0cf39 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_binary.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_binary.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_block3d.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_block3d.mlir
index ac5f773d67182cd..a6fb24e266a1876 100755
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_block3d.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_block3d.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_block_matmul.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_block_matmul.mlir
index efef01155cc784e..24e1e5ba1c770f9 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_block_matmul.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_block_matmul.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_cast.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_cast.mlir
index 3b5168db23c584f..f4bfb577efd4882 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_cast.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_cast.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_cmp.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_cmp.mlir
index edeffea211717b5..6710ed95e13439f 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_cmp.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_cmp.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_codegen_dim.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_codegen_dim.mlir
index c5d002aa1639175..e5281049b44c9fb 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_codegen_dim.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_codegen_dim.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_codegen_foreach.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_codegen_foreach.mlir
index 9deb5cd05fa3b3a..ecd2819ff2fea03 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_codegen_foreach.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_codegen_foreach.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_collapse_shape.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_collapse_shape.mlir
index 12132155e7cb35e..ac6f6672fa842b3 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_collapse_shape.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_collapse_shape.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_complex32.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_complex32.mlir
index 087360f7a1ceb70..db32ea9168bbfb9 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_complex32.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_complex32.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_complex64.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_complex64.mlir
index 3f748015c9580ac..183c16fd429e8f2 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_complex64.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_complex64.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_complex_ops.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_complex_ops.mlir
index 2326234bc06ccf0..fe87882ce9ba41e 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_complex_ops.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_complex_ops.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_constant_to_sparse_tensor.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_constant_to_sparse_tensor.mlir
index 51c13085cf3e9d3..df348ab5c211ac9 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_constant_to_sparse_tensor.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_constant_to_sparse_tensor.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_1d_nwc_wcf.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_1d_nwc_wcf.mlir
index 3e46b6d65112f52..2c35ab051a46612 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_1d_nwc_wcf.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_1d_nwc_wcf.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d.mlir
index 97e9d1783f67fad..d782a8bfa3b9812 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d_55.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d_55.mlir
index 00805d198013d50..c218f95c2e53205 100755
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d_55.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d_55.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d_nchw_fchw.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d_nchw_fchw.mlir
index 9150e97e72481bd..20508f980350fc1 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d_nchw_fchw.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d_nchw_fchw.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d_nhwc_hwcf.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d_nhwc_hwcf.mlir
index 429175c1a16457f..892abea7372932b 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d_nhwc_hwcf.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d_nhwc_hwcf.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_3d.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_3d.mlir
index b23b2dcc173d92f..1d9ca4d4e8fba03 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_3d.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_3d.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_3d_ndhwc_dhwcf.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_3d_ndhwc_dhwcf.mlir
index 8fb6704c7f509e9..9624e347bd5ca3f 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_3d_ndhwc_dhwcf.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_3d_ndhwc_dhwcf.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion.mlir
index 5de3aa0a2e97130..096a1618df2d87d 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_block.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_block.mlir
index 66215a340a0b630..d8c3025947bd4d1 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_block.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_block.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_dyn.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_dyn.mlir
index 0d9722cd37e65d0..8de0f9a5da9a56a 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_dyn.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_dyn.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_element.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_element.mlir
index a2ec6df392aaa39..a4bcb09cfb8fc5f 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_element.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_element.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_ptr.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_ptr.mlir
index 531efb4f7f37f46..a93002dfeb4529f 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_ptr.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_ptr.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_sparse2dense.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_sparse2dense.mlir
index e145c4542a7bfc2..51e81225991bb21 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_sparse2dense.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_sparse2dense.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_sparse2sparse.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_sparse2sparse.mlir
index 12f8e34500d818a..8844e4f9acf36ef 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_sparse2sparse.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conversion_sparse2sparse.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_coo_test.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_coo_test.mlir
index c16ae0de182039b..191bca746c0684a 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_coo_test.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_coo_test.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_dilated_conv_2d_nhwc_hwcf.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_dilated_conv_2d_nhwc_hwcf.mlir
index 40738a9f7d7f193..046405aa48cb2a4 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_dilated_conv_2d_nhwc_hwcf.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_dilated_conv_2d_nhwc_hwcf.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_dot.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_dot.mlir
index b41fda19459e659..adede14cfc5e856 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_dot.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_dot.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_ds.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_ds.mlir
index 17fab93b9b21787..2d202132b3daa14 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_ds.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_ds.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_empty.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_empty.mlir
index 7255649ccd424af..6a1a5821ad9dafc 100755
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_empty.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_empty.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_expand.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_expand.mlir
index 6e875de7481ecd2..ed771862e7140bb 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_expand.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_expand.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_expand_shape.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_expand_shape.mlir
index 5e021596efea6f7..0792bda162a7618 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_expand_shape.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_expand_shape.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_filter_conv2d.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_filter_conv2d.mlir
index 93b8eda2c2aec83..a4ed2a6d05fc074 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_filter_conv2d.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_filter_conv2d.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_flatten.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_flatten.mlir
index 8a5712d3fa1b109..db3e2381b5c9bff 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_flatten.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_flatten.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_foreach_slices.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_foreach_slices.mlir
index aef3a947e4f0bab..2b3cbf28ed014a5 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_foreach_slices.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_foreach_slices.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_generate.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_generate.mlir
index 63a6d3acd737e2f..75d1941c50ada78 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_generate.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_generate.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_index.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_index.mlir
index 005398445828169..e81c8f52d6969d4 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_index.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_index.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_index_dense.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_index_dense.mlir
index fc7b82fdecea343..20bf265a70b5176 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_index_dense.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_index_dense.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_1d.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_1d.mlir
index a81ec172f599d43..d7f9b45bff8e09b 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_1d.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_1d.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_2d.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_2d.mlir
index baab6e7598868ff..af6dcdcfeb26350 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_2d.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_2d.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_3d.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_3d.mlir
index 12ef94fc2baa215..8c21f14b5abebbe 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_3d.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_insert_3d.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_loose.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_loose.mlir
index 416c137a1dc3e74..1682e808b35eeaa 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_loose.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_loose.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matmul.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matmul.mlir
index 14fa22a7013478e..e1daaaf9bb1e5b3 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matmul.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matmul.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matmul_slice.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matmul_slice.mlir
index c76bf2ccfe35e45..f6000ccf34d7121 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matmul_slice.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matmul_slice.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matrix_ops.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matrix_ops.mlir
index 770c4f55a280615..ecc351619d12610 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matrix_ops.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matrix_ops.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matvec.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matvec.mlir
index b9d1148301dd169..30e7a5f18b7f792 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matvec.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_matvec.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_mttkrp.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_mttkrp.mlir
index d615cb66c3d08d9..04392ca7a6209d3 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_mttkrp.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_mttkrp.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_out_mult_elt.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_out_mult_elt.mlir
index 683be61be2a201e..506d357fd4a3ced 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_out_mult_elt.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_out_mult_elt.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_out_reduction.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_out_reduction.mlir
index 8eadc348020f9aa..3539246e1e5a854 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_out_reduction.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_out_reduction.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_out_simple.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_out_simple.mlir
index 334ffe492952e43..b56d9d4d5a602b6 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_out_simple.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_out_simple.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pack.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pack.mlir
index 5415625ff05d6d4..71416ffa6a9065f 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pack.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pack.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pack_d.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pack_d.mlir
index b48ff9c9df7483c..a53c248eeed494a 100755
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pack_d.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pack_d.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_permute.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_permute.mlir
index 664a86c7ad58fd8..1df1caf11386dc3 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_permute.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_permute.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pooling_nhwc.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pooling_nhwc.mlir
index 7c78bfc362007d1..187ea81fcdb9804 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pooling_nhwc.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_pooling_nhwc.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_print.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_print.mlir
index f3c721535e75c5c..2c505ecf29568c2 100755
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_print.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_print.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_print_3d.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_print_3d.mlir
index 4f1e4312d7bc361..ea369338ce09668 100755
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_print_3d.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_print_3d.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_quantized_matmul.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_quantized_matmul.mlir
index 873322929232a75..28f65b10db47f68 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_quantized_matmul.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_quantized_matmul.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_re_im.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_re_im.mlir
index fc23fe501fcf1b6..aba1c52a5af3c33 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_re_im.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_re_im.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reduce_custom.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reduce_custom.mlir
index 5da028c3685c4e5..cb6af2daf1c007a 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reduce_custom.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reduce_custom.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reduce_custom_prod.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reduce_custom_prod.mlir
index 8cbe8fd1ed695ef..624af1fa9f08ee3 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reduce_custom_prod.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reduce_custom_prod.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reduce_custom_sum.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reduce_custom_sum.mlir
index 8588ebd98cc918c..c6b42af47834721 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reduce_custom_sum.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reduce_custom_sum.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reductions.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reductions.mlir
index 96ec8bad8b55152..bb1ea60bf8d34dd 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reductions.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reductions.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reductions_min.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reductions_min.mlir
index 16c1d7df7e89dd8..58a5747116ee220 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reductions_min.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reductions_min.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reductions_prod.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reductions_prod.mlir
index 4797fbb8f5319c0..28702976893d80c 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reductions_prod.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reductions_prod.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reshape.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reshape.mlir
index 317fe0f225ee699..403cbf4cc6fb59f 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reshape.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_reshape.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_rewrite_push_back.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_rewrite_push_back.mlir
index 1536249e60f286e..a90324c3e6d6061 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_rewrite_push_back.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_rewrite_push_back.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_rewrite_sort_coo.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_rewrite_sort_coo.mlir
index 0682bc6f314fd17..9e0b44ef15dae62 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_rewrite_sort_coo.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_rewrite_sort_coo.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sampled_matmul.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sampled_matmul.mlir
index 085b36a368704dd..ebd3c643c65ab96 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sampled_matmul.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sampled_matmul.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sampled_mm_fusion.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sampled_mm_fusion.mlir
index eecd970e01ac9c7..de6a1e1aa3c1111 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sampled_mm_fusion.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sampled_mm_fusion.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_scale.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_scale.mlir
index c62cdc900b83d90..cb391b7d59793d0 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_scale.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_scale.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_scf_nested.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_scf_nested.mlir
index 3f0cf70675ba04c..d80255918433872 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_scf_nested.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_scf_nested.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_select.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_select.mlir
index bd61563b4b2dd3a..85a4e2fea410ada 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_select.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_select.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_semiring_select.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_semiring_select.mlir
index d96b07a0db331c2..22555dcee8611fc 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_semiring_select.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_semiring_select.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sign.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sign.mlir
index 11d23d681c8289b..6c0e1f3703ee474 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sign.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sign.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sorted_coo.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sorted_coo.mlir
index 3117d2539f17f50..47aded8825d99f6 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sorted_coo.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sorted_coo.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_spmm.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_spmm.mlir
index ca8bcd7744c8f40..d74722d8ddce891 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_spmm.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_spmm.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_storage.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_storage.mlir
index da87b5cc3c6da7c..3ccbdb36128f956 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_storage.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_storage.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_strided_conv_2d_nhwc_hwcf.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_strided_conv_2d_nhwc_hwcf.mlir
index 2b2b8536fe39ed5..713e0f5d0abfd2e 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_strided_conv_2d_nhwc_hwcf.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_strided_conv_2d_nhwc_hwcf.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum.mlir
index d1c58bfb6d59efd..2d279995dada1dd 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum_bf16.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum_bf16.mlir
index 16a8b50ab08e5c8..3b2629a2cd2ae5d 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum_bf16.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum_bf16.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
@@ -29,7 +30,7 @@
 // Do the same run, but now with  VLA vectorization.
 // RUN: %if mlir_arm_sve_tests %{ %{compile_sve} | %{run_sve} | FileCheck %s %}
 
-// UNSUPPORTED: target=aarch64{{.*}}
+// UNSUPPORTED: target=aarch64{{.*}}, mlir_arm_emulator
 
 !Filename = !llvm.ptr
 
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum_c32.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum_c32.mlir
index f95c163a57c1649..3d6c944cd2223a0 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum_c32.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum_c32.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum_f16.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum_f16.mlir
index 30be587c8f6119c..d9321b926141c4c 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum_f16.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum_f16.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_tanh.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_tanh.mlir
index 748fffc1e637156..6fdbf161b002978 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_tanh.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_tanh.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
@@ -31,7 +32,7 @@
 // RUN: %if mlir_arm_sve_tests %{ %{compile_sve} | %{run_sve} | FileCheck %s %}
 
 // Current fails for SVE, see https://github.com/llvm/llvm-project/issues/60626
-// UNSUPPORTED: target=aarch64{{.*}}
+// UNSUPPORTED: target=aarch64{{.*}}, mlir_arm_emulator
 
 #SparseVector = #sparse_tensor.encoding<{ map = (d0) -> (d0 : compressed) }>
 
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_tensor_mul.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_tensor_mul.mlir
index fe2f2690e860d6e..06f35562941457d 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_tensor_mul.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_tensor_mul.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_tensor_ops.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_tensor_ops.mlir
index a46c3a8d5ef6c6d..de437509c2e9481 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_tensor_ops.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_tensor_ops.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_transpose.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_transpose.mlir
index 434cc95094645fc..f6d3e5c7ca99463 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_transpose.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_transpose.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_transpose_coo.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_transpose_coo.mlir
index 3b7760e5052c393..83e965a26b33612 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_transpose_coo.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_transpose_coo.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_unary.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_unary.mlir
index acb7a99a3418016..0d29afb27c5c2a9 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_unary.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_unary.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_vector_ops.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_vector_ops.mlir
index 10ccf47c3408313..b771f92525e267a 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_vector_ops.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_vector_ops.mlir
@@ -10,9 +10,10 @@
 // DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}"
 // DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}"
 // DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils
+// DEFINE: %{run_libs_sve} = -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 // DEFINE: %{run_opts} = -e main -entry-point-result=void
 // DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs}
-// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs}
+// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs_sve}
 //
 // DEFINE: %{env} =
 //--------------------------------------------------------------------------------------------------
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/load-store-128-bit-tile.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/load-store-128-bit-tile.mlir
index 41e724844fe4037..f4275fdde3c78c4 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/load-store-128-bit-tile.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/load-store-128-bit-tile.mlir
@@ -3,7 +3,7 @@
 // DEFINE: %{run} = %mcr_aarch64_cmd \
 // DEFINE:  -march=aarch64 -mattr=+sve,+sme \
 // DEFINE:  -e %{entry_point} -entry-point-result=void \
-// DEFINE:  -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils,%arm_sme_abi_shlib
+// DEFINE:  -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils,%native_arm_sme_abi_shlib
 
 // RUN: %{compile} | %{run} | FileCheck %s
 
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/load-vertical.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/load-vertical.mlir
index 68c31ac1dd8e9cf..b7144be08a853f7 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/load-vertical.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/load-vertical.mlir
@@ -3,7 +3,7 @@
 // DEFINE: %{run} = %mcr_aarch64_cmd \
 // DEFINE:   -march=aarch64 -mattr=+sve,+sme \
 // DEFINE:   -e %{entry_point} -entry-point-result=void \
-// DEFINE:   -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils,%arm_sme_abi_shlib
+// DEFINE:   -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils,%native_arm_sme_abi_shlib
 
 // RUN: %{compile} | %{run} | FileCheck %s
 
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/multi-tile-transpose.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/multi-tile-transpose.mlir
index cd48f2a9ebfd895..7f1566d635cbb57 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/multi-tile-transpose.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/multi-tile-transpose.mlir
@@ -2,7 +2,7 @@
 // RUN: %mcr_aarch64_cmd \
 // RUN:   -e=main -entry-point-result=void \
 // RUN:   -march=aarch64 -mattr="+sve,+sme" \
-// RUN:   -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils,%arm_sme_abi_shlib,%mlir_arm_runner_utils | \
+// RUN:   -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils,%native_arm_sme_abi_shlib,%native_mlir_arm_runner_utils | \
 // RUN: FileCheck %s
 
 #transpose = affine_map<(d0, d1) -> (d1, d0)>
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/outerproduct-f16f16f32.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/outerproduct-f16f16f32.mlir
index a06ad37b054e45a..886a990125b4c95 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/outerproduct-f16f16f32.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/outerproduct-f16f16f32.mlir
@@ -5,7 +5,7 @@
 // DEFINE: %{run} = %mcr_aarch64_cmd %t \
 // DEFINE:   -march=aarch64 -mattr=+sve,+sme \
 // DEFINE:   -e %{entry} -entry-point-result=void \
-// DEFINE:   -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils,%mlir_arm_runner_utils,%arm_sme_abi_shlib
+// DEFINE:   -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils,%native_mlir_arm_runner_utils,%native_arm_sme_abi_shlib
 
 // RUN: %{compile}
 
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/outerproduct-f32.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/outerproduct-f32.mlir
index fb6c06cfd69999d..cc05940fb6d02c3 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/outerproduct-f32.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/outerproduct-f32.mlir
@@ -4,7 +4,7 @@
 // DEFINE: %{run} = %mcr_aarch64_cmd %t \
 // DEFINE:   -march=aarch64 -mattr=+sve,+sme \
 // DEFINE:   -e %{entry_point} -entry-point-result=void \
-// DEFINE:   -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils,%arm_sme_abi_shlib
+// DEFINE:   -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils,%native_arm_sme_abi_shlib
 
 // RUN: %{compile}
 
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/outerproduct-f64.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/outerproduct-f64.mlir
index b8458606d3f324e..aa5483572dcf941 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/outerproduct-f64.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/outerproduct-f64.mlir
@@ -4,7 +4,7 @@
 // DEFINE: %{run} = %mcr_aarch64_cmd %t \
 // DEFINE:   -march=aarch64 -mattr=+sve,+sme-f64f64 \
 // DEFINE:   -e %{entry_point} -entry-point-result=void \
-// DEFINE:   -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils,%arm_sme_abi_shlib
+// DEFINE:   -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils,%native_arm_sme_abi_shlib
 
 // RUN: %{compile}
 
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/outerproduct-i8i8i32.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/outerproduct-i8i8i32.mlir
index 9a353ec2d2f6665..653b0d7beb2c78e 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/outerproduct-i8i8i32.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/outerproduct-i8i8i32.mlir
@@ -3,7 +3,7 @@
 // DEFINE: %{run} = %mcr_aarch64_cmd \
 // DEFINE:   -march=aarch64 -mattr=+sve,+sme \
 // DEFINE:   -e %{entry} -entry-point-result=void \
-// DEFINE:   -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils,%mlir_arm_runner_utils,%arm_sme_abi_shlib
+// DEFINE:   -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils,%native_mlir_arm_runner_utils,%native_arm_sme_abi_shlib
 
 // RUN: %{compile} | %{run} | FileCheck %s
 
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/ssve.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/ssve.mlir
index fcf070186359b22..aa8a2aa45d65b32 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/ssve.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/ssve.mlir
@@ -3,7 +3,7 @@
 // DEFINE: %{run} = %mcr_aarch64_cmd \
 // DEFINE:  -march=aarch64 -mattr=+sve,+sme \
 // DEFINE:  -e %{entry_point} -entry-point-result=i32 \
-// DEFINE:  -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils
+// DEFINE:  -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils
 
 // NOTE: To run this test, your CPU must support SME.
 
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/test-setArmSVLBits.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/test-setArmSVLBits.mlir
index 4c02bcb9d192fb9..4a6c6f2ee528bb5 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/test-setArmSVLBits.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/test-setArmSVLBits.mlir
@@ -6,7 +6,7 @@
 // DEFINE: %{run} = %mcr_aarch64_cmd \
 // DEFINE:  -march=aarch64 -mattr=+sve,+sme \
 // DEFINE:  -e %{entry_point} -entry-point-result=void \
-// DEFINE:  -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils,%mlir_arm_runner_utils
+// DEFINE:  -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils,%native_mlir_arm_runner_utils
 
 // RUN: %{compile} | %{run} | FileCheck %s
 
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/tile-fill.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/tile-fill.mlir
index 3d74508cd23b573..b94345c70c7136f 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/tile-fill.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/tile-fill.mlir
@@ -2,7 +2,7 @@
 // RUN: %mcr_aarch64_cmd \
 // RUN:  -march=aarch64 -mattr=+sve,+sme \
 // RUN:  -e entry -entry-point-result=i32 \
-// RUN:  -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils,%arm_sme_abi_shlib | \
+// RUN:  -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils,%native_arm_sme_abi_shlib | \
 // RUN: FileCheck %s
 
 // Integration test demonstrating filling a 32-bit element ZA tile with a
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/transfer-read-2d.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/transfer-read-2d.mlir
index 7421521b96bf92c..77f5a325728b326 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/transfer-read-2d.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/transfer-read-2d.mlir
@@ -3,7 +3,7 @@
 // DEFINE: %{run} = %mcr_aarch64_cmd \
 // DEFINE:  -march=aarch64 -mattr=+sve,+sme \
 // DEFINE:  -e %{entry_point} -entry-point-result=void \
-// DEFINE:  -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils,%arm_sme_abi_shlib
+// DEFINE:  -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils,%native_arm_sme_abi_shlib
 
 // RUN: %{compile} | %{run} | FileCheck %s
 
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/transfer-write-2d.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/transfer-write-2d.mlir
index 2fef705861f28ed..c3bf379cde61721 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/transfer-write-2d.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/transfer-write-2d.mlir
@@ -3,7 +3,7 @@
 // DEFINE: %{run} = %mcr_aarch64_cmd \
 // DEFINE:  -march=aarch64 -mattr=+sve,+sme \
 // DEFINE:  -e %{entry_point} -entry-point-result=void \
-// DEFINE:  -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils,%arm_sme_abi_shlib
+// DEFINE:  -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils,%native_arm_sme_abi_shlib
 
 // RUN: %{compile} | %{run} | FileCheck %s
 
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/transpose.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/transpose.mlir
index 177c96f1d8aae6a..ff20f99b63cd1da 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/transpose.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/transpose.mlir
@@ -3,7 +3,7 @@
 // DEFINE: %{run} = %mcr_aarch64_cmd \
 // DEFINE:   -march=aarch64 -mattr=+sve,+sme \
 // DEFINE:   -e %{entry_point} -entry-point-result=void \
-// DEFINE:   -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils,%arm_sme_abi_shlib
+// DEFINE:   -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils,%native_arm_sme_abi_shlib
 
 // RUN: %{compile} | %{run} | FileCheck %s
 
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/vector-load-store.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/vector-load-store.mlir
index 48080fd0a26a2ba..6e25bee65f09529 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/vector-load-store.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/vector-load-store.mlir
@@ -3,7 +3,7 @@
 // DEFINE: %{run} = %mcr_aarch64_cmd \
 // DEFINE:  -march=aarch64 -mattr=+sve,+sme \
 // DEFINE:  -e %{entry_point} -entry-point-result=i32 \
-// DEFINE:  -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils,%arm_sme_abi_shlib
+// DEFINE:  -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils,%native_arm_sme_abi_shlib
 
 // RUN: %{compile} | %{run} | FileCheck %s --check-prefix=CHECK-ZA0_D
 
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/vector-ops.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/vector-ops.mlir
index c8c401bed144695..697fb90f6315498 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/vector-ops.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/vector-ops.mlir
@@ -3,7 +3,7 @@
 // DEFINE: %{run} = %mcr_aarch64_cmd \
 // DEFINE:  -march=aarch64 -mattr=+sve,+sme \
 // DEFINE:  -e %{entry_point} -entry-point-result=i32 \
-// DEFINE:  -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils,%arm_sme_abi_shlib
+// DEFINE:  -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils,%native_arm_sme_abi_shlib
 
 // RUN: %{compile} | %{run} | FileCheck %s
 
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/arrays-of-scalable-vectors.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/arrays-of-scalable-vectors.mlir
index afb23e8e5206601..4ce6281d9fd45c6 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/arrays-of-scalable-vectors.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/arrays-of-scalable-vectors.mlir
@@ -1,5 +1,5 @@
 // RUN: mlir-opt %s -convert-vector-to-scf -arm-sve-legalize-vector-storage -convert-vector-to-llvm="enable-arm-sve" -test-lower-to-llvm | \
-// RUN: %mcr_aarch64_cmd -e=entry -entry-point-result=void --march=aarch64 --mattr="+sve" -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \
+// RUN: %mcr_aarch64_cmd -e=entry -entry-point-result=void --march=aarch64 --mattr="+sve" -shared-libs=%native_mlir_c_runner_utils | \
 // RUN: FileCheck %s
 
 /// This tests basic functionality of arrays of scalable vectors, which in MLIR
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/contraction.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/contraction.mlir
index 79121bf31c26ed5..c9904328763dbce 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/contraction.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/contraction.mlir
@@ -2,7 +2,7 @@
 // DEFINE:    -cse -canonicalize -convert-vector-to-scf -arm-sve-legalize-vector-storage\
 // DEFINE:    -convert-vector-to-llvm="enable-arm-sve" -test-lower-to-llvm -o %t
 // DEFINE: %{entry} =
-// DEFINE: %{run} = %mcr_aarch64_cmd %t -e=%{entry} -entry-point-result=void --march=aarch64 --mattr="+sve" -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext
+// DEFINE: %{run} = %mcr_aarch64_cmd %t -e=%{entry} -entry-point-result=void --march=aarch64 --mattr="+sve" -shared-libs=%native_mlir_c_runner_utils
 
 // This check whether the files compiles and generates a temporary that will be executed further down.
 // RUN: %{compile}
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/scalable-interleave.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/scalable-interleave.mlir
index e9f1bbeafacddfc..d3b1fa4bbbc377f 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/scalable-interleave.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/scalable-interleave.mlir
@@ -1,6 +1,6 @@
 // RUN: mlir-opt %s -test-lower-to-llvm | \
 // RUN: %mcr_aarch64_cmd -e entry -entry-point-result=void  \
-// RUN:   -shared-libs=%mlir_c_runner_utils,%mlir_arm_runner_utils \
+// RUN:   -shared-libs=%native_mlir_c_runner_utils,%native_mlir_arm_runner_utils \
 // RUN:   -march=aarch64 -mattr=+sve | \
 // RUN: FileCheck %s
 
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/sve.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/sve.mlir
index 824963cdea3b336..40307c5f9da071d 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/sve.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/sve.mlir
@@ -1,5 +1,5 @@
 // RUN: mlir-opt %s -lower-affine -convert-vector-to-scf -convert-scf-to-cf -convert-vector-to-llvm="enable-arm-sve" -finalize-memref-to-llvm -convert-func-to-llvm -convert-arith-to-llvm -canonicalize | \
-// RUN: %mcr_aarch64_cmd -e=entry -entry-point-result=void --march=aarch64 --mattr="+sve" -shared-libs=%mlir_lib_dir/libmlir_c_runner_utils%shlibext | \
+// RUN: %mcr_aarch64_cmd -e=entry -entry-point-result=void --march=aarch64 --mattr="+sve" -shared-libs=%native_mlir_c_runner_utils | \
 // RUN: FileCheck %s
 
 // Note: To run this test, your CPU must support SVE
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/test-scalable-deinterleave.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/test-scalable-deinterleave.mlir
index 3ef41b0bf5cf453..c071bb0b17f4476 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/test-scalable-deinterleave.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/test-scalable-deinterleave.mlir
@@ -4,7 +4,7 @@
 // DEFINE: %{compile} = mlir-opt %s -test-lower-to-llvm
 // DEFINE: %{run} = %mcr_aarch64_cmd -march=aarch64 -mattr=+sve \
 // DEFINE:  -e %{entry_point} -entry-point-result=void \
-// DEFINE:  -shared-libs=%mlir_c_runner_utils,%mlir_arm_runner_utils
+// DEFINE:  -shared-libs=%native_mlir_c_runner_utils,%native_mlir_arm_runner_utils
 
 // RUN: %{compile} | %{run} | FileCheck %s
 
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/test-setArmVLBits.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/test-setArmVLBits.mlir
index 6e25d9d16b8bde5..4f0869d92da6e3c 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/test-setArmVLBits.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/test-setArmVLBits.mlir
@@ -4,7 +4,7 @@
 // DEFINE: %{compile} = mlir-opt %s -test-lower-to-llvm
 // DEFINE: %{run} = %mcr_aarch64_cmd -march=aarch64 -mattr=+sve \
 // DEFINE:  -e %{entry_point} -entry-point-result=void \
-// DEFINE:  -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils,%mlir_arm_runner_utils
+// DEFINE:  -shared-libs=%native_mlir_runner_utils,%native_mlir_c_runner_utils,%native_mlir_arm_runner_utils
 
 // RUN: %{compile} | %{run} | FileCheck %s
 
diff --git a/mlir/test/Integration/lit.local.cfg b/mlir/test/Integration/lit.local.cfg
index 1dc40f986ead147..a73300780f39a82 100644
--- a/mlir/test/Integration/lit.local.cfg
+++ b/mlir/test/Integration/lit.local.cfg
@@ -11,14 +11,8 @@ def configure_aarch64_mcr_cmd():
     # under emulation, the SVE specific RUN lines in the SparseTensor tests
     # will run under emulation.
     if not (config.mlir_run_arm_sve_tests or config.mlir_run_arm_sme_tests):
-        return mcr_cmd
-
-    config.substitutions.append(
-        (
-            "%mlir_native_utils_lib_dir",
-            config.arm_emulator_utils_lib_dir or config.mlir_lib_dir,
-        )
-    )
+        config.substitutions.append(("%mcr_aarch64_cmd", mcr_cmd))
+        return
 
     if config.arm_emulator_executable:
         if config.arm_emulator_mlir_cpu_runner_executable:
@@ -43,10 +37,24 @@ def configure_aarch64_mcr_cmd():
 
         mcr_cmd = f"{emulation_cmd} {mcr_cmd}"
 
-    return mcr_cmd
+    config.substitutions.append(("%mcr_aarch64_cmd", mcr_cmd))
 
+def configure_aarch64_mlir_utils():
+    # Use mlir runtime libs from the location passed by -DARM_EMULATOR_UTILS_LIB_DIR.
+    if config.arm_emulator_utils_lib_dir:
+      base_dir = config.arm_emulator_utils_lib_dir
+    else:
+      base_dir = config.mlir_lib_dir
 
-aarch64_mcr_cmd = configure_aarch64_mcr_cmd()
+    config.substitutions.append(("%native_mlir_runner_utils",
+                                 base_dir + "/libmlir_runner_utils" + config.llvm_shlib_ext))
+    config.substitutions.append(("%native_mlir_c_runner_utils",
+                                 base_dir + "/libmlir_c_runner_utils" + config.llvm_shlib_ext))
+    config.substitutions.append(("%native_mlir_arm_runner_utils",
+                                 base_dir + "/libmlir_arm_runner_utils" + config.llvm_shlib_ext))
+    # Use passed Arm SME ABI routines, if not present default to stubs.
+    config.substitutions.append(("%native_arm_sme_abi_shlib", config.arm_sme_abi_routines_shlib or
+                                 (base_dir + "/libmlir_arm_sme_abi_stubs" + config.llvm_shlib_ext)))
 
 # AArch64 tests will run under emulation if configured at build time by the
 # following CMake options:
@@ -57,4 +65,7 @@ aarch64_mcr_cmd = configure_aarch64_mcr_cmd()
 #                                             support cross-compilation
 # * ARM_EMULATOR_UTILS_LIB_DIR - AArch64 native utilites library to support
 #                                cross-compilation.
-config.substitutions.append(("%mcr_aarch64_cmd", aarch64_mcr_cmd))
+
+# Check for above options and set correct executables and runtime libs
+configure_aarch64_mcr_cmd()
+configure_aarch64_mlir_utils()
diff --git a/mlir/test/lit.site.cfg.py.in b/mlir/test/lit.site.cfg.py.in
index 411a02849f9dd88..d68d9b1a1f43a27 100644
--- a/mlir/test/lit.site.cfg.py.in
+++ b/mlir/test/lit.site.cfg.py.in
@@ -54,6 +54,11 @@ config.mlir_run_cuda_sm80_lt_tests = @MLIR_RUN_CUDA_SM80_LT_TESTS@
 config.mlir_run_cuda_sm90_tests = @MLIR_RUN_CUDA_SM90_TESTS@
 config.mlir_include_integration_tests = @MLIR_INCLUDE_INTEGRATION_TESTS@
 config.arm_emulator_executable = "@ARM_EMULATOR_EXECUTABLE@"
+# Some tests marked with 'UNSUPPORTED: target=aarch64{{.*}}' are still run when
+# configured with ARM_EMULATOR_EXECUTABLE and the default target is not aarch64.
+# Add feature 'mlir_arm_emulator' here and to UNSUPPORTED list of such tests.
+if config.arm_emulator_executable:
+    config.available_features.add("mlir_arm_emulator")
 config.arm_emulator_options = "@ARM_EMULATOR_OPTIONS@"
 config.arm_emulator_mlir_cpu_runner_executable = "@ARM_EMULATOR_MLIR_CPU_RUNNER_EXECUTABLE@"
 config.arm_emulator_lli_executable = "@ARM_EMULATOR_LLI_EXECUTABLE@"

From ef6760116fa2fa21f78e7a3b499f77e1a3eb7b92 Mon Sep 17 00:00:00 2001
From: Thurston Dang <thurston@google.com>
Date: Thu, 15 Aug 2024 22:15:48 -0700
Subject: [PATCH 126/441] [asan] Fix build breakage from report_globals change

https://github.com/llvm/llvm-project/pull/104404 broke the Windows
buildbot (https://lab.llvm.org/buildbot/#/builders/107/builds/1926).
This fixes forward by updating the flags in the tests accordingly.
---
 .../test/asan/TestCases/Windows/dll_global_dead_strip.c       | 4 ++--
 .../Windows/dll_report_globals_symbolization_at_startup.cpp   | 2 +-
 compiler-rt/test/asan/TestCases/Windows/global_dead_strip.c   | 4 ++--
 .../asan/TestCases/Windows/report_globals_vs_freelibrary.cpp  | 2 +-
 4 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/compiler-rt/test/asan/TestCases/Windows/dll_global_dead_strip.c b/compiler-rt/test/asan/TestCases/Windows/dll_global_dead_strip.c
index a0c96622efeea41..e5bd27bdf65fdf7 100644
--- a/compiler-rt/test/asan/TestCases/Windows/dll_global_dead_strip.c
+++ b/compiler-rt/test/asan/TestCases/Windows/dll_global_dead_strip.c
@@ -1,11 +1,11 @@
 // RUN: %clang_cl_asan %Od %p/dll_host.cpp %Fe%t
 //
 // RUN: %clang_cl_nocxx_asan %Gw %LD %Od %s %Fe%t.dll
-// RUN: %env_asan_opts=report_globals=2 %run %t %t.dll 2>&1 | FileCheck %s --check-prefix=NOSTRIP
+// RUN: %env_asan_opts=report_globals=1:verbosity=3 %run %t %t.dll 2>&1 | FileCheck %s --check-prefix=NOSTRIP
 // RUN: %clang_cl_nocxx_asan %Gw %LD -O2 %s %Fe%t.dll \
 // RUN:   %if target={{.*-windows-gnu}} %{ -Wl,--gc-sections %} \
 // RUN:   %else %{ -link -opt:ref %}
-// RUN: %env_asan_opts=report_globals=2 %run %t %t.dll 2>&1 | FileCheck %s --check-prefix=STRIP
+// RUN: %env_asan_opts=report_globals=1:verbosity=3 %run %t %t.dll 2>&1 | FileCheck %s --check-prefix=STRIP
 
 #include <stdio.h>
 
diff --git a/compiler-rt/test/asan/TestCases/Windows/dll_report_globals_symbolization_at_startup.cpp b/compiler-rt/test/asan/TestCases/Windows/dll_report_globals_symbolization_at_startup.cpp
index 06a632e6708b1e6..c74b66f2b43b3e6 100644
--- a/compiler-rt/test/asan/TestCases/Windows/dll_report_globals_symbolization_at_startup.cpp
+++ b/compiler-rt/test/asan/TestCases/Windows/dll_report_globals_symbolization_at_startup.cpp
@@ -1,7 +1,7 @@
 // RUN: %clang_cl_asan %LD %Od -DDLL %s %Fe%t.dll \
 // RUN:   %if target={{.*-windows-gnu}} %{ -Wl,--out-implib,%t.lib %}
 // RUN: %clang_cl_asan %Od -DEXE %s %t.lib %Fe%te.exe
-// RUN: %env_asan_opts=report_globals=2 %run %te.exe 2>&1 | FileCheck %s
+// RUN: %env_asan_opts=report_globals=1:verbosity=3 %run %te.exe 2>&1 | FileCheck %s
 
 // FIXME: Currently, the MT runtime build crashes on startup due to dbghelp.dll
 // initialization failure.
diff --git a/compiler-rt/test/asan/TestCases/Windows/global_dead_strip.c b/compiler-rt/test/asan/TestCases/Windows/global_dead_strip.c
index 0e15120a46f776e..7f2405fdfc83644 100644
--- a/compiler-rt/test/asan/TestCases/Windows/global_dead_strip.c
+++ b/compiler-rt/test/asan/TestCases/Windows/global_dead_strip.c
@@ -1,9 +1,9 @@
 // RUN: %clang_cl_nocxx_asan %Gw %Od %s %Fe%t.exe
-// RUN: %env_asan_opts=report_globals=2 %t.exe 2>&1 | FileCheck %s --check-prefix=NOSTRIP
+// RUN: %env_asan_opts=report_globals=1:verbosity=3 %t.exe 2>&1 | FileCheck %s --check-prefix=NOSTRIP
 // RUN: %clang_cl_nocxx_asan %Gw -O2 %s %Fe%t.exe \
 // RUN:   %if target={{.*-windows-gnu}} %{ -Wl,--gc-sections %} \
 // RUN:   %else %{ -link -opt:ref %}
-// RUN: %env_asan_opts=report_globals=2 %t.exe 2>&1 | FileCheck %s --check-prefix=STRIP
+// RUN: %env_asan_opts=report_globals=1:verbosity=3 %t.exe 2>&1 | FileCheck %s --check-prefix=STRIP
 
 #include <stdio.h>
 int dead_global = 42;
diff --git a/compiler-rt/test/asan/TestCases/Windows/report_globals_vs_freelibrary.cpp b/compiler-rt/test/asan/TestCases/Windows/report_globals_vs_freelibrary.cpp
index 7cad3f39be1ec29..34ce18e146d6771 100644
--- a/compiler-rt/test/asan/TestCases/Windows/report_globals_vs_freelibrary.cpp
+++ b/compiler-rt/test/asan/TestCases/Windows/report_globals_vs_freelibrary.cpp
@@ -1,6 +1,6 @@
 // RUN: %clang_cl_asan %LD %Od -DDLL %s %Fe%t.dll
 // RUN: %clang_cl_asan %Od -DEXE %s %Fe%te.exe
-// RUN: %env_asan_opts=report_globals=2 %run %te.exe %t.dll 2>&1 | FileCheck %s
+// RUN: %env_asan_opts=report_globals=1:verbosity=3 %run %te.exe %t.dll 2>&1 | FileCheck %s
 
 #include <windows.h>
 #include <stdio.h>

From ac92dcc7562ca1c0b1ee84bcc678738aefbe5c18 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke@igalia.com>
Date: Fri, 16 Aug 2024 13:26:30 +0800
Subject: [PATCH 127/441] [RISCV] Lower fixed reverse vector_shuffles through
 vector_reverse (#104461)

This teaches lowerVECTOR_REVERSE to handle fixed length vectors, and
then lowers reverse vector_shuffles through it.

The motiviation for this is to allow fixed length vectors to share a
potential optimization on vector_reverse in an upcoming patch (splitting
up LMUL > 1 vrgathers.vv)
---
 llvm/lib/Target/RISCV/RISCVISelLowering.cpp   | 48 ++++++++++-----
 .../rvv/fixed-vectors-shuffle-reverse.ll      | 44 +++++++-------
 .../test/CodeGen/RISCV/rvv/shuffle-reverse.ll | 60 +++++++++----------
 3 files changed, 83 insertions(+), 69 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 515d7dbcd47bd93..434f22bb9c8df3f 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -1087,8 +1087,9 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
         setOperationAction({ISD::INSERT_SUBVECTOR, ISD::EXTRACT_SUBVECTOR}, VT,
                            Custom);
 
-        setOperationAction({ISD::BUILD_VECTOR, ISD::CONCAT_VECTORS}, VT,
-                           Custom);
+        setOperationAction(
+            {ISD::BUILD_VECTOR, ISD::CONCAT_VECTORS, ISD::VECTOR_REVERSE}, VT,
+            Custom);
 
         setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT},
                            VT, Custom);
@@ -1235,8 +1236,8 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
         // expansion to a build_vector of 0s.
         setOperationAction(ISD::UNDEF, VT, Custom);
 
-        setOperationAction({ISD::CONCAT_VECTORS, ISD::INSERT_SUBVECTOR,
-                            ISD::EXTRACT_SUBVECTOR},
+        setOperationAction({ISD::CONCAT_VECTORS, ISD::VECTOR_REVERSE,
+                            ISD::INSERT_SUBVECTOR, ISD::EXTRACT_SUBVECTOR},
                            VT, Custom);
 
         // FIXME: mload, mstore, mgather, mscatter, vp_load/store,
@@ -5160,6 +5161,9 @@ static SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG,
     return convertFromScalableVector(VT, Res, DAG, Subtarget);
   }
 
+  if (ShuffleVectorInst::isReverseMask(Mask, NumElts) && V2.isUndef())
+    return DAG.getNode(ISD::VECTOR_REVERSE, DL, VT, V1);
+
   // If this is a deinterleave and we can widen the vector, then we can use
   // vnsrl to deinterleave.
   if (isDeinterleaveShuffle(VT, ContainerVT, V1, V2, Mask, Subtarget)) {
@@ -10308,14 +10312,24 @@ SDValue RISCVTargetLowering::lowerVECTOR_REVERSE(SDValue Op,
     SDValue Op2 = DAG.getNode(ISD::VECTOR_REVERSE, DL, WidenVT, Op1);
     return DAG.getNode(ISD::TRUNCATE, DL, VecVT, Op2);
   }
-  unsigned EltSize = VecVT.getScalarSizeInBits();
-  unsigned MinSize = VecVT.getSizeInBits().getKnownMinValue();
+
+  MVT ContainerVT = VecVT;
+  SDValue Vec = Op.getOperand(0);
+  if (VecVT.isFixedLengthVector()) {
+    ContainerVT = getContainerForFixedLengthVector(VecVT);
+    Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
+  }
+
+  unsigned EltSize = ContainerVT.getScalarSizeInBits();
+  unsigned MinSize = ContainerVT.getSizeInBits().getKnownMinValue();
   unsigned VectorBitsMax = Subtarget.getRealMaxVLen();
   unsigned MaxVLMAX =
-    RISCVTargetLowering::computeVLMAX(VectorBitsMax, EltSize, MinSize);
+      VecVT.isFixedLengthVector()
+          ? VecVT.getVectorNumElements()
+          : RISCVTargetLowering::computeVLMAX(VectorBitsMax, EltSize, MinSize);
 
   unsigned GatherOpc = RISCVISD::VRGATHER_VV_VL;
-  MVT IntVT = VecVT.changeVectorElementTypeToInteger();
+  MVT IntVT = ContainerVT.changeVectorElementTypeToInteger();
 
   // If this is SEW=8 and VLMAX is potentially more than 256, we need
   // to use vrgatherei16.vv.
@@ -10340,7 +10354,7 @@ SDValue RISCVTargetLowering::lowerVECTOR_REVERSE(SDValue Op,
     }
 
     // Just promote the int type to i16 which will double the LMUL.
-    IntVT = MVT::getVectorVT(MVT::i16, VecVT.getVectorElementCount());
+    IntVT = MVT::getVectorVT(MVT::i16, ContainerVT.getVectorElementCount());
     GatherOpc = RISCVISD::VRGATHEREI16_VV_VL;
   }
 
@@ -10354,12 +10368,13 @@ SDValue RISCVTargetLowering::lowerVECTOR_REVERSE(SDValue Op,
   }
 
   MVT XLenVT = Subtarget.getXLenVT();
-  auto [Mask, VL] = getDefaultScalableVLOps(VecVT, DL, DAG, Subtarget);
+  auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
 
   // Calculate VLMAX-1 for the desired SEW.
-  SDValue VLMinus1 = DAG.getNode(ISD::SUB, DL, XLenVT,
-                                 computeVLMax(VecVT, DL, DAG),
-                                 DAG.getConstant(1, DL, XLenVT));
+  SDValue VLMinus1 = DAG.getNode(
+      ISD::SUB, DL, XLenVT,
+      DAG.getElementCount(DL, XLenVT, VecVT.getVectorElementCount()),
+      DAG.getConstant(1, DL, XLenVT));
 
   // Splat VLMAX-1 taking care to handle SEW==64 on RV32.
   bool IsRV32E64 =
@@ -10375,8 +10390,11 @@ SDValue RISCVTargetLowering::lowerVECTOR_REVERSE(SDValue Op,
   SDValue Indices = DAG.getNode(RISCVISD::SUB_VL, DL, IntVT, SplatVL, VID,
                                 DAG.getUNDEF(IntVT), Mask, VL);
 
-  return DAG.getNode(GatherOpc, DL, VecVT, Op.getOperand(0), Indices,
-                     DAG.getUNDEF(VecVT), Mask, VL);
+  SDValue Gather = DAG.getNode(GatherOpc, DL, ContainerVT, Vec, Indices,
+                               DAG.getUNDEF(ContainerVT), Mask, VL);
+  if (VecVT.isFixedLengthVector())
+    Gather = convertFromScalableVector(VecVT, Gather, DAG, Subtarget);
+  return Gather;
 }
 
 SDValue RISCVTargetLowering::lowerVECTOR_SPLICE(SDValue Op,
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-reverse.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-reverse.ll
index 4b1f0beb4870084..d6e66577fa97a78 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-reverse.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-reverse.ll
@@ -101,10 +101,10 @@ define <32 x i1> @reverse_v32i1(<32 x i1> %a) {
 ; NO-ZVBB-LABEL: reverse_v32i1:
 ; NO-ZVBB:       # %bb.0:
 ; NO-ZVBB-NEXT:    li a0, 32
-; NO-ZVBB-NEXT:    lui a1, %hi(.LCPI4_0)
-; NO-ZVBB-NEXT:    addi a1, a1, %lo(.LCPI4_0)
 ; NO-ZVBB-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
-; NO-ZVBB-NEXT:    vle8.v v8, (a1)
+; NO-ZVBB-NEXT:    vid.v v8
+; NO-ZVBB-NEXT:    li a0, 31
+; NO-ZVBB-NEXT:    vrsub.vx v8, v8, a0
 ; NO-ZVBB-NEXT:    vmv.v.i v10, 0
 ; NO-ZVBB-NEXT:    vmerge.vim v10, v10, 1, v0
 ; NO-ZVBB-NEXT:    vrgather.vv v12, v10, v8
@@ -124,10 +124,10 @@ define <64 x i1> @reverse_v64i1(<64 x i1> %a) {
 ; NO-ZVBB-LABEL: reverse_v64i1:
 ; NO-ZVBB:       # %bb.0:
 ; NO-ZVBB-NEXT:    li a0, 64
-; NO-ZVBB-NEXT:    lui a1, %hi(.LCPI5_0)
-; NO-ZVBB-NEXT:    addi a1, a1, %lo(.LCPI5_0)
 ; NO-ZVBB-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
-; NO-ZVBB-NEXT:    vle8.v v8, (a1)
+; NO-ZVBB-NEXT:    vid.v v8
+; NO-ZVBB-NEXT:    li a0, 63
+; NO-ZVBB-NEXT:    vrsub.vx v8, v8, a0
 ; NO-ZVBB-NEXT:    vmv.v.i v12, 0
 ; NO-ZVBB-NEXT:    vmerge.vim v12, v12, 1, v0
 ; NO-ZVBB-NEXT:    vrgather.vv v16, v12, v8
@@ -147,10 +147,10 @@ define <128 x i1> @reverse_v128i1(<128 x i1> %a) {
 ; CHECK-LABEL: reverse_v128i1:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    li a0, 128
-; CHECK-NEXT:    lui a1, %hi(.LCPI6_0)
-; CHECK-NEXT:    addi a1, a1, %lo(.LCPI6_0)
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
-; CHECK-NEXT:    vle8.v v8, (a1)
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    li a0, 127
+; CHECK-NEXT:    vrsub.vx v8, v8, a0
 ; CHECK-NEXT:    vmv.v.i v16, 0
 ; CHECK-NEXT:    vmerge.vim v16, v16, 1, v0
 ; CHECK-NEXT:    vrgather.vv v24, v16, v8
@@ -229,10 +229,10 @@ define <32 x i8> @reverse_v32i8(<32 x i8> %a) {
 ; CHECK-LABEL: reverse_v32i8:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    li a0, 32
-; CHECK-NEXT:    lui a1, %hi(.LCPI12_0)
-; CHECK-NEXT:    addi a1, a1, %lo(.LCPI12_0)
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
-; CHECK-NEXT:    vle8.v v12, (a1)
+; CHECK-NEXT:    vid.v v10
+; CHECK-NEXT:    li a0, 31
+; CHECK-NEXT:    vrsub.vx v12, v10, a0
 ; CHECK-NEXT:    vrgather.vv v10, v8, v12
 ; CHECK-NEXT:    vmv.v.v v8, v10
 ; CHECK-NEXT:    ret
@@ -244,10 +244,10 @@ define <64 x i8> @reverse_v64i8(<64 x i8> %a) {
 ; CHECK-LABEL: reverse_v64i8:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    li a0, 64
-; CHECK-NEXT:    lui a1, %hi(.LCPI13_0)
-; CHECK-NEXT:    addi a1, a1, %lo(.LCPI13_0)
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m4, ta, ma
-; CHECK-NEXT:    vle8.v v16, (a1)
+; CHECK-NEXT:    vid.v v12
+; CHECK-NEXT:    li a0, 63
+; CHECK-NEXT:    vrsub.vx v16, v12, a0
 ; CHECK-NEXT:    vrgather.vv v12, v8, v16
 ; CHECK-NEXT:    vmv.v.v v8, v12
 ; CHECK-NEXT:    ret
@@ -324,11 +324,10 @@ define <32 x i16> @reverse_v32i16(<32 x i16> %a) {
 ; CHECK-LABEL: reverse_v32i16:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    li a0, 32
-; CHECK-NEXT:    lui a1, %hi(.LCPI19_0)
-; CHECK-NEXT:    addi a1, a1, %lo(.LCPI19_0)
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    vle8.v v12, (a1)
-; CHECK-NEXT:    vsext.vf2 v16, v12
+; CHECK-NEXT:    vid.v v12
+; CHECK-NEXT:    li a0, 31
+; CHECK-NEXT:    vrsub.vx v16, v12, a0
 ; CHECK-NEXT:    vrgather.vv v12, v8, v16
 ; CHECK-NEXT:    vmv.v.v v8, v12
 ; CHECK-NEXT:    ret
@@ -521,11 +520,10 @@ define <32 x half> @reverse_v32f16(<32 x half> %a) {
 ; CHECK-LABEL: reverse_v32f16:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    li a0, 32
-; CHECK-NEXT:    lui a1, %hi(.LCPI34_0)
-; CHECK-NEXT:    addi a1, a1, %lo(.LCPI34_0)
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    vle8.v v12, (a1)
-; CHECK-NEXT:    vsext.vf2 v16, v12
+; CHECK-NEXT:    vid.v v12
+; CHECK-NEXT:    li a0, 31
+; CHECK-NEXT:    vrsub.vx v16, v12, a0
 ; CHECK-NEXT:    vrgather.vv v12, v8, v16
 ; CHECK-NEXT:    vmv.v.v v8, v12
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/rvv/shuffle-reverse.ll b/llvm/test/CodeGen/RISCV/rvv/shuffle-reverse.ll
index 368f454fa5fda1f..9e62bb623a03bd0 100644
--- a/llvm/test/CodeGen/RISCV/rvv/shuffle-reverse.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/shuffle-reverse.ll
@@ -106,21 +106,20 @@ define <16 x i8> @v16i8(<16 x i8> %a) {
 define <32 x i8> @v16i8_2(<16 x i8> %a, <16 x i8> %b) {
 ; CHECK-LABEL: v16i8_2:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmv1r.v v12, v9
 ; CHECK-NEXT:    li a0, 32
-; CHECK-NEXT:    lui a1, %hi(.LCPI7_0)
-; CHECK-NEXT:    addi a1, a1, %lo(.LCPI7_0)
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
-; CHECK-NEXT:    vle8.v v12, (a1)
-; CHECK-NEXT:    vmv1r.v v14, v9
-; CHECK-NEXT:    vrgather.vv v10, v8, v12
-; CHECK-NEXT:    vid.v v8
-; CHECK-NEXT:    vrsub.vi v8, v8, 15
+; CHECK-NEXT:    vid.v v14
+; CHECK-NEXT:    li a0, 31
+; CHECK-NEXT:    vrsub.vx v16, v14, a0
+; CHECK-NEXT:    vrgather.vv v10, v8, v16
+; CHECK-NEXT:    vrsub.vi v8, v14, 15
 ; CHECK-NEXT:    lui a0, 16
 ; CHECK-NEXT:    addi a0, a0, -1
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
 ; CHECK-NEXT:    vmv.s.x v0, a0
 ; CHECK-NEXT:    vsetvli zero, zero, e8, m2, ta, mu
-; CHECK-NEXT:    vrgather.vv v10, v14, v8, v0.t
+; CHECK-NEXT:    vrgather.vv v10, v12, v8, v0.t
 ; CHECK-NEXT:    vmv.v.v v8, v10
 ; CHECK-NEXT:    ret
   %v32i8 = shufflevector <16 x i8> %a, <16 x i8> %b,  <32 x i32> <i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
@@ -230,22 +229,21 @@ define <16 x i16> @v16i16(<16 x i16> %a) {
 define <32 x i16> @v16i16_2(<16 x i16> %a, <16 x i16> %b) {
 ; CHECK-LABEL: v16i16_2:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmv2r.v v16, v10
 ; CHECK-NEXT:    li a0, 32
-; CHECK-NEXT:    lui a1, %hi(.LCPI15_0)
-; CHECK-NEXT:    addi a1, a1, %lo(.LCPI15_0)
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    vle16.v v16, (a1)
-; CHECK-NEXT:    vmv2r.v v20, v10
-; CHECK-NEXT:    vmv2r.v v12, v8
-; CHECK-NEXT:    vrgather.vv v8, v12, v16
-; CHECK-NEXT:    vid.v v12
-; CHECK-NEXT:    vrsub.vi v12, v12, 15
+; CHECK-NEXT:    vid.v v20
+; CHECK-NEXT:    li a0, 31
+; CHECK-NEXT:    vrsub.vx v24, v20, a0
+; CHECK-NEXT:    vrgather.vv v12, v8, v24
+; CHECK-NEXT:    vrsub.vi v8, v20, 15
 ; CHECK-NEXT:    lui a0, 16
 ; CHECK-NEXT:    addi a0, a0, -1
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
 ; CHECK-NEXT:    vmv.s.x v0, a0
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m4, ta, mu
-; CHECK-NEXT:    vrgather.vv v8, v20, v12, v0.t
+; CHECK-NEXT:    vrgather.vv v12, v16, v8, v0.t
+; CHECK-NEXT:    vmv.v.v v8, v12
 ; CHECK-NEXT:    ret
   %v32i16 = shufflevector <16 x i16> %a, <16 x i16> %b,  <32 x i32> <i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <32 x i16> %v32i16
@@ -363,17 +361,17 @@ define <16 x i32> @v16i32(<16 x i32> %a) {
 define <32 x i32> @v16i32_2(<16 x i32> %a, <16 x i32> %b) {
 ; CHECK-LABEL: v16i32_2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a0, 32
-; CHECK-NEXT:    lui a1, %hi(.LCPI23_0)
-; CHECK-NEXT:    addi a1, a1, %lo(.LCPI23_0)
-; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
-; CHECK-NEXT:    vle16.v v20, (a1)
 ; CHECK-NEXT:    vmv4r.v v24, v12
 ; CHECK-NEXT:    vmv4r.v v16, v8
-; CHECK-NEXT:    vrgatherei16.vv v8, v16, v20
+; CHECK-NEXT:    li a0, 32
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
+; CHECK-NEXT:    vid.v v20
+; CHECK-NEXT:    li a0, 31
+; CHECK-NEXT:    vrsub.vx v28, v20, a0
+; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
+; CHECK-NEXT:    vrgatherei16.vv v8, v16, v28
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m4, ta, ma
-; CHECK-NEXT:    vid.v v16
-; CHECK-NEXT:    vrsub.vi v16, v16, 15
+; CHECK-NEXT:    vrsub.vi v16, v20, 15
 ; CHECK-NEXT:    lui a0, 16
 ; CHECK-NEXT:    addi a0, a0, -1
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, mu
@@ -549,10 +547,10 @@ define <32 x half> @v16f16_2(<16 x half> %a) {
 ; CHECK-LABEL: v16f16_2:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    li a0, 32
-; CHECK-NEXT:    lui a1, %hi(.LCPI35_0)
-; CHECK-NEXT:    addi a1, a1, %lo(.LCPI35_0)
 ; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    vle16.v v16, (a1)
+; CHECK-NEXT:    vid.v v12
+; CHECK-NEXT:    li a0, 31
+; CHECK-NEXT:    vrsub.vx v16, v12, a0
 ; CHECK-NEXT:    vrgather.vv v12, v8, v16
 ; CHECK-NEXT:    vmv.v.v v8, v12
 ; CHECK-NEXT:    ret
@@ -720,10 +718,10 @@ define <32 x i8> @v32i8(<32 x i8> %a) {
 ; CHECK-LABEL: v32i8:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    li a0, 32
-; CHECK-NEXT:    lui a1, %hi(.LCPI46_0)
-; CHECK-NEXT:    addi a1, a1, %lo(.LCPI46_0)
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
-; CHECK-NEXT:    vle8.v v12, (a1)
+; CHECK-NEXT:    vid.v v10
+; CHECK-NEXT:    li a0, 31
+; CHECK-NEXT:    vrsub.vx v12, v10, a0
 ; CHECK-NEXT:    vrgather.vv v10, v8, v12
 ; CHECK-NEXT:    vmv.v.v v8, v10
 ; CHECK-NEXT:    ret

From 0c02f6176242c37e6328255e396ccb717188c588 Mon Sep 17 00:00:00 2001
From: nosba0957 <67941279+nosba0957@users.noreply.github.com>
Date: Fri, 16 Aug 2024 13:38:28 +0800
Subject: [PATCH 128/441] [RISCV][NFC] Fix typo: "wererenamed" to "were
 renamed" (#104530)

This commit corrects a typo in a comment from "wererenamed" to "were
renamed". No functional changes.
---
 llvm/lib/Target/RISCV/RISCVInstrInfo.td | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.td b/llvm/lib/Target/RISCV/RISCVInstrInfo.td
index 2dd7240ef941a73..41ef55544e7d2df 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.td
@@ -1056,7 +1056,7 @@ def : InstAlias<"sltu $rd, $rs1, $imm12",
 
 def : MnemonicAlias<"move", "mv">;
 
-// The SCALL and SBREAK instructions wererenamed to ECALL and EBREAK in
+// The SCALL and SBREAK instructions were renamed to ECALL and EBREAK in
 // version 2.1 of the user-level ISA. Like the GNU toolchain, we still accept
 // the old name for backwards compatibility.
 def : MnemonicAlias<"scall", "ecall">;

From 1900810b6fd7b913d04b23afd9cce7e83fff51d7 Mon Sep 17 00:00:00 2001
From: Wang Yaduo <wangyaduo@linux.alibaba.com>
Date: Fri, 16 Aug 2024 14:23:00 +0800
Subject: [PATCH 129/441] [RISCV] Simplify (srl (and X, Mask), Const) to
 TH_EXTU (#102802)

---
 llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp |  9 +++++++++
 llvm/test/CodeGen/RISCV/ctlz-cttz-ctpop.ll  | 14 ++++++--------
 llvm/test/CodeGen/RISCV/rv64xtheadbb.ll     | 16 ++++++++++++++++
 3 files changed, 31 insertions(+), 8 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
index 35681c620eed51e..0f7b90f5f5f1349 100644
--- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
@@ -1164,6 +1164,15 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) {
     }
 
     unsigned LShAmt = Subtarget->getXLen() - TrailingOnes;
+    if (Subtarget->hasVendorXTHeadBb()) {
+      SDNode *THEXTU = CurDAG->getMachineNode(
+          RISCV::TH_EXTU, DL, VT, N0->getOperand(0),
+          CurDAG->getTargetConstant(TrailingOnes - 1, DL, VT),
+          CurDAG->getTargetConstant(ShAmt, DL, VT));
+      ReplaceNode(Node, THEXTU);
+      return;
+    }
+
     SDNode *SLLI =
         CurDAG->getMachineNode(RISCV::SLLI, DL, VT, N0->getOperand(0),
                                CurDAG->getTargetConstant(LShAmt, DL, VT));
diff --git a/llvm/test/CodeGen/RISCV/ctlz-cttz-ctpop.ll b/llvm/test/CodeGen/RISCV/ctlz-cttz-ctpop.ll
index 8caa64c9572ce78..2c691a2de4c4de7 100644
--- a/llvm/test/CodeGen/RISCV/ctlz-cttz-ctpop.ll
+++ b/llvm/test/CodeGen/RISCV/ctlz-cttz-ctpop.ll
@@ -2367,10 +2367,9 @@ define i16 @test_ctpop_i16(i16 %a) nounwind {
 ; RV32XTHEADBB-NEXT:    add a0, a2, a0
 ; RV32XTHEADBB-NEXT:    srli a1, a0, 4
 ; RV32XTHEADBB-NEXT:    add a0, a0, a1
-; RV32XTHEADBB-NEXT:    andi a1, a0, 15
-; RV32XTHEADBB-NEXT:    slli a0, a0, 20
-; RV32XTHEADBB-NEXT:    srli a0, a0, 28
-; RV32XTHEADBB-NEXT:    add a0, a1, a0
+; RV32XTHEADBB-NEXT:    th.extu a1, a0, 11, 8
+; RV32XTHEADBB-NEXT:    andi a0, a0, 15
+; RV32XTHEADBB-NEXT:    add a0, a0, a1
 ; RV32XTHEADBB-NEXT:    ret
 ;
 ; RV64XTHEADBB-LABEL: test_ctpop_i16:
@@ -2388,10 +2387,9 @@ define i16 @test_ctpop_i16(i16 %a) nounwind {
 ; RV64XTHEADBB-NEXT:    add a0, a2, a0
 ; RV64XTHEADBB-NEXT:    srli a1, a0, 4
 ; RV64XTHEADBB-NEXT:    add a0, a0, a1
-; RV64XTHEADBB-NEXT:    andi a1, a0, 15
-; RV64XTHEADBB-NEXT:    slli a0, a0, 52
-; RV64XTHEADBB-NEXT:    srli a0, a0, 60
-; RV64XTHEADBB-NEXT:    add a0, a1, a0
+; RV64XTHEADBB-NEXT:    th.extu a1, a0, 11, 8
+; RV64XTHEADBB-NEXT:    andi a0, a0, 15
+; RV64XTHEADBB-NEXT:    add a0, a0, a1
 ; RV64XTHEADBB-NEXT:    ret
   %1 = call i16 @llvm.ctpop.i16(i16 %a)
   ret i16 %1
diff --git a/llvm/test/CodeGen/RISCV/rv64xtheadbb.ll b/llvm/test/CodeGen/RISCV/rv64xtheadbb.ll
index 6cdab888ffcde7f..8ce4c44d8efcc2a 100644
--- a/llvm/test/CodeGen/RISCV/rv64xtheadbb.ll
+++ b/llvm/test/CodeGen/RISCV/rv64xtheadbb.ll
@@ -702,6 +702,22 @@ define i64 @zext_bf_i64(i64 %a) nounwind {
   ret i64 %and
 }
 
+define i64 @zext_bf2_i64(i64 %a) nounwind {
+; RV64I-LABEL: zext_bf2_i64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slli a0, a0, 48
+; RV64I-NEXT:    srli a0, a0, 49
+; RV64I-NEXT:    ret
+;
+; RV64XTHEADBB-LABEL: zext_bf2_i64:
+; RV64XTHEADBB:       # %bb.0:
+; RV64XTHEADBB-NEXT:    th.extu a0, a0, 15, 1
+; RV64XTHEADBB-NEXT:    ret
+  %t0 = and i64 %a, 65535
+  %result = lshr i64 %t0, 1
+  ret i64 %result
+}
+
 define i64 @zext_i64_srliw(i64 %a) nounwind {
 ; RV64I-LABEL: zext_i64_srliw:
 ; RV64I:       # %bb.0:

From 22219873987587d8b5d793ab5dea982a0887ac7c Mon Sep 17 00:00:00 2001
From: Timm Baeder <tbaeder@redhat.com>
Date: Fri, 16 Aug 2024 08:37:46 +0200
Subject: [PATCH 130/441] [clang][Interp] Add scopes to conditional operator
 subexpressions (#104418)

We would otherwise try to destroy the variables from both branches after
the conditional operator was done.

However, doing so breaks the case where we create internal temporaries.
For those, add allocateTemporary(), which attaches the lifetime of the
temporary to the topmost scope. Since they never have record type, this
shouldn't create any issues.
---
 clang/lib/AST/Interp/Compiler.cpp | 63 +++++++++++++++++++++----------
 clang/lib/AST/Interp/Compiler.h   |  1 +
 clang/test/AST/Interp/records.cpp | 26 +++++++++++++
 3 files changed, 70 insertions(+), 20 deletions(-)

diff --git a/clang/lib/AST/Interp/Compiler.cpp b/clang/lib/AST/Interp/Compiler.cpp
index d32b595f9f07240..5e1f507ca2b1785 100644
--- a/clang/lib/AST/Interp/Compiler.cpp
+++ b/clang/lib/AST/Interp/Compiler.cpp
@@ -533,10 +533,8 @@ bool Compiler<Emitter>::VisitCastExpr(const CastExpr *CE) {
     // We're creating a complex value here, so we need to
     // allocate storage for it.
     if (!Initializing) {
-      std::optional<unsigned> LocalIndex = allocateLocal(CE);
-      if (!LocalIndex)
-        return false;
-      if (!this->emitGetPtrLocal(*LocalIndex, CE))
+      unsigned LocalIndex = allocateTemporary(CE);
+      if (!this->emitGetPtrLocal(LocalIndex, CE))
         return false;
     }
 
@@ -671,10 +669,8 @@ bool Compiler<Emitter>::VisitImaginaryLiteral(const ImaginaryLiteral *E) {
     return true;
 
   if (!Initializing) {
-    std::optional<unsigned> LocalIndex = allocateLocal(E);
-    if (!LocalIndex)
-      return false;
-    if (!this->emitGetPtrLocal(*LocalIndex, E))
+    unsigned LocalIndex = allocateTemporary(E);
+    if (!this->emitGetPtrLocal(LocalIndex, E))
       return false;
   }
 
@@ -986,10 +982,8 @@ template <class Emitter>
 bool Compiler<Emitter>::VisitComplexBinOp(const BinaryOperator *E) {
   // Prepare storage for result.
   if (!Initializing) {
-    std::optional<unsigned> LocalIndex = allocateLocal(E);
-    if (!LocalIndex)
-      return false;
-    if (!this->emitGetPtrLocal(*LocalIndex, E))
+    unsigned LocalIndex = allocateTemporary(E);
+    if (!this->emitGetPtrLocal(LocalIndex, E))
       return false;
   }
 
@@ -1045,10 +1039,7 @@ bool Compiler<Emitter>::VisitComplexBinOp(const BinaryOperator *E) {
 
     if (!LHSIsComplex) {
       // This is using the RHS type for the fake-complex LHS.
-      if (auto LHSO = allocateLocal(RHS))
-        LHSOffset = *LHSO;
-      else
-        return false;
+      LHSOffset = allocateTemporary(RHS);
 
       if (!this->emitGetPtrLocal(LHSOffset, E))
         return false;
@@ -1881,15 +1872,26 @@ bool Compiler<Emitter>::VisitAbstractConditionalOperator(
   if (!this->jumpFalse(LabelFalse))
     return false;
 
-  if (!this->delegate(TrueExpr))
-    return false;
+  {
+    LocalScope<Emitter> S(this);
+    if (!this->delegate(TrueExpr))
+      return false;
+    if (!S.destroyLocals())
+      return false;
+  }
+
   if (!this->jump(LabelEnd))
     return false;
 
   this->emitLabel(LabelFalse);
 
-  if (!this->delegate(FalseExpr))
-    return false;
+  {
+    LocalScope<Emitter> S(this);
+    if (!this->delegate(FalseExpr))
+      return false;
+    if (!S.destroyLocals())
+      return false;
+  }
 
   this->fallthrough(LabelEnd);
   this->emitLabel(LabelEnd);
@@ -3549,6 +3551,27 @@ Compiler<Emitter>::allocateLocal(DeclTy &&Src, const ValueDecl *ExtendingDecl) {
   return Local.Offset;
 }
 
+template <class Emitter>
+unsigned Compiler<Emitter>::allocateTemporary(const Expr *E) {
+  QualType Ty = E->getType();
+  assert(!Ty->isRecordType());
+
+  Descriptor *D = P.createDescriptor(
+      E, Ty.getTypePtr(), Descriptor::InlineDescMD, Ty.isConstQualified(),
+      /*IsTemporary=*/true, /*IsMutable=*/false, /*Init=*/nullptr);
+  assert(D);
+
+  Scope::Local Local = this->createLocal(D);
+  VariableScope<Emitter> *S = VarScope;
+  assert(S);
+  // Attach to topmost scope.
+  while (S->getParent())
+    S = S->getParent();
+  assert(S && !S->getParent());
+  S->addLocal(Local);
+  return Local.Offset;
+}
+
 template <class Emitter>
 const RecordType *Compiler<Emitter>::getRecordTy(QualType Ty) {
   if (const PointerType *PT = dyn_cast<PointerType>(Ty))
diff --git a/clang/lib/AST/Interp/Compiler.h b/clang/lib/AST/Interp/Compiler.h
index 112219c49e8bddb..74bfce5f241f28e 100644
--- a/clang/lib/AST/Interp/Compiler.h
+++ b/clang/lib/AST/Interp/Compiler.h
@@ -297,6 +297,7 @@ class Compiler : public ConstStmtVisitor<Compiler<Emitter>, bool>,
   /// Allocates a space storing a local given its type.
   std::optional<unsigned>
   allocateLocal(DeclTy &&Decl, const ValueDecl *ExtendingDecl = nullptr);
+  unsigned allocateTemporary(const Expr *E);
 
 private:
   friend class VariableScope<Emitter>;
diff --git a/clang/test/AST/Interp/records.cpp b/clang/test/AST/Interp/records.cpp
index f51f9771b38aaa6..7e3cf5b94518f7c 100644
--- a/clang/test/AST/Interp/records.cpp
+++ b/clang/test/AST/Interp/records.cpp
@@ -1627,3 +1627,29 @@ namespace DtorDestroysFieldsAfterSelf {
   static_assert(foo() == 15);
 }
 #endif
+
+namespace ExprWithCleanups {
+  struct A { A(); ~A(); int get(); };
+  constexpr int get() {return false ? A().get() : 1;}
+  static_assert(get() == 1, "");
+
+
+  struct S {
+    int V;
+    constexpr S(int V) : V(V) {}
+    constexpr int get() {
+      return V;
+    }
+  };
+  constexpr int get(bool b) {
+    S a = b ? S(1) : S(2);
+
+    return a.get();
+  }
+  static_assert(get(true) == 1, "");
+  static_assert(get(false) == 2, "");
+
+
+  constexpr auto F = true ? 1i : 2i;
+  static_assert(F == 1i, "");
+}

From 698b42ccff69fde6509c6ad0baf262c257c039bb Mon Sep 17 00:00:00 2001
From: Kareem Ergawy <kareem.ergawy@amd.com>
Date: Fri, 16 Aug 2024 09:27:13 +0200
Subject: [PATCH 131/441] [flang][stack-arrays] Collect analysis results for
 OMP ws loops (#103590)

We missed collecting the analysis results for regions terminated with
`omp.yield`. This result in missing some opportunities for malloc
optimizations inside omp regions.
---
 .../lib/Optimizer/Transforms/StackArrays.cpp  |  2 ++
 flang/test/Transforms/stack-arrays-hlfir.f90  | 28 +++++++++++++++++++
 2 files changed, 30 insertions(+)

diff --git a/flang/lib/Optimizer/Transforms/StackArrays.cpp b/flang/lib/Optimizer/Transforms/StackArrays.cpp
index bdc2d9cd9c6c43f..a5a95138ac12819 100644
--- a/flang/lib/Optimizer/Transforms/StackArrays.cpp
+++ b/flang/lib/Optimizer/Transforms/StackArrays.cpp
@@ -457,10 +457,12 @@ StackArraysAnalysisWrapper::analyseFunction(mlir::Operation *func) {
     if (lattice)
       (void)point.join(*lattice);
   };
+
   func->walk([&](mlir::func::ReturnOp child) { joinOperationLattice(child); });
   func->walk([&](fir::UnreachableOp child) { joinOperationLattice(child); });
   func->walk(
       [&](mlir::omp::TerminatorOp child) { joinOperationLattice(child); });
+  func->walk([&](mlir::omp::YieldOp child) { joinOperationLattice(child); });
 
   llvm::DenseSet<mlir::Value> freedValues;
   point.appendFreedValues(freedValues);
diff --git a/flang/test/Transforms/stack-arrays-hlfir.f90 b/flang/test/Transforms/stack-arrays-hlfir.f90
index 50261b3078466d0..296edbaf35417e8 100644
--- a/flang/test/Transforms/stack-arrays-hlfir.f90
+++ b/flang/test/Transforms/stack-arrays-hlfir.f90
@@ -53,3 +53,31 @@ end subroutine omp_temp_array
 ! CHECK-NEXT:    }
 ! CHECK:         return
 ! CHECK-NEXT:  }
+
+subroutine omp_target_wsloop
+  implicit none
+  integer (8) :: lV, i
+  integer (8), dimension (2) :: iaVS
+
+  lV = 202
+
+  !$omp target teams distribute
+  do i = 1, 10
+    iaVS = [lV, lV]
+  end do
+  !$omp end target teams distribute
+end subroutine omp_target_wsloop
+! CHECK-LABEL: func.func @_QPomp_target_wsloop{{.*}} {
+! CHECK:         omp.target {{.*}} {
+! CHECK-NOT:       fir.allocmem
+! CHECK-NOT:       fir.freemem
+! CHECK:           fir.alloca !fir.array<2xi64>
+! CHECK:         omp.teams {
+! CHECK:         omp.distribute {
+! CHECK:         omp.loop_nest {{.*}} {
+! CHECK-NOT:       fir.allocmem
+! CHECK-NOT:       fir.freemem
+! CHECK:           omp.yield
+! CHECK-NEXT:    }
+! CHECK:         return
+! CHECK-NEXT:  }

From 5cdd2042a01881decd71fbfb7a465e4927a854a3 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke@igalia.com>
Date: Fri, 16 Aug 2024 14:57:38 +0800
Subject: [PATCH 132/441] [RISCV] Remove -riscv-v-vector-bits-max from reverse
 tests. NFC

There doesn't seem to be any difference in the output anymore between
the options.
---
 .../rvv/fixed-vectors-shuffle-reverse.ll      | 246 +++++-------------
 1 file changed, 60 insertions(+), 186 deletions(-)

diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-reverse.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-reverse.ll
index d6e66577fa97a78..bab2e9ba1963745 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-reverse.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-reverse.ll
@@ -1,10 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zfh,+zvfh -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,NO-ZVBB,RV32-BITS-UNKNOWN
-; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zfh,+zvfh -riscv-v-vector-bits-max=256 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,NO-ZVBB,RV32-BITS-256
-; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zfh,+zvfh -riscv-v-vector-bits-max=512 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,NO-ZVBB,RV32-BITS-512
-; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zfh,+zvfh -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,NO-ZVBB,RV64-BITS-UNKNOWN
-; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zfh,+zvfh -riscv-v-vector-bits-max=256 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,NO-ZVBB,RV64-BITS-256
-; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zfh,+zvfh -riscv-v-vector-bits-max=512 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,NO-ZVBB,RV64-BITS-512
+; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zfh,+zvfh -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,NO-ZVBB,RV32
+; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zfh,+zvfh -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,NO-ZVBB,RV64
 ; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zfh,+zvfh,+zvbb -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVBB,RV32-ZVBB
 ; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zfh,+zvfh,+zvbb -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVBB,RV64-ZVBB
 
@@ -648,65 +644,25 @@ define <8 x double> @reverse_v8f64(<8 x double> %a) {
 
 
 define <3 x i64> @reverse_v3i64(<3 x i64> %a) {
-; RV32-BITS-UNKNOWN-LABEL: reverse_v3i64:
-; RV32-BITS-UNKNOWN:       # %bb.0:
-; RV32-BITS-UNKNOWN-NEXT:    lui a0, %hi(.LCPI44_0)
-; RV32-BITS-UNKNOWN-NEXT:    addi a0, a0, %lo(.LCPI44_0)
-; RV32-BITS-UNKNOWN-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-BITS-UNKNOWN-NEXT:    vle16.v v12, (a0)
-; RV32-BITS-UNKNOWN-NEXT:    vrgatherei16.vv v10, v8, v12
-; RV32-BITS-UNKNOWN-NEXT:    vmv.v.v v8, v10
-; RV32-BITS-UNKNOWN-NEXT:    ret
-;
-; RV32-BITS-256-LABEL: reverse_v3i64:
-; RV32-BITS-256:       # %bb.0:
-; RV32-BITS-256-NEXT:    lui a0, %hi(.LCPI44_0)
-; RV32-BITS-256-NEXT:    addi a0, a0, %lo(.LCPI44_0)
-; RV32-BITS-256-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-BITS-256-NEXT:    vle16.v v12, (a0)
-; RV32-BITS-256-NEXT:    vrgatherei16.vv v10, v8, v12
-; RV32-BITS-256-NEXT:    vmv.v.v v8, v10
-; RV32-BITS-256-NEXT:    ret
-;
-; RV32-BITS-512-LABEL: reverse_v3i64:
-; RV32-BITS-512:       # %bb.0:
-; RV32-BITS-512-NEXT:    lui a0, %hi(.LCPI44_0)
-; RV32-BITS-512-NEXT:    addi a0, a0, %lo(.LCPI44_0)
-; RV32-BITS-512-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; RV32-BITS-512-NEXT:    vle16.v v12, (a0)
-; RV32-BITS-512-NEXT:    vrgatherei16.vv v10, v8, v12
-; RV32-BITS-512-NEXT:    vmv.v.v v8, v10
-; RV32-BITS-512-NEXT:    ret
-;
-; RV64-BITS-UNKNOWN-LABEL: reverse_v3i64:
-; RV64-BITS-UNKNOWN:       # %bb.0:
-; RV64-BITS-UNKNOWN-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
-; RV64-BITS-UNKNOWN-NEXT:    vid.v v10
-; RV64-BITS-UNKNOWN-NEXT:    vrsub.vi v12, v10, 2
-; RV64-BITS-UNKNOWN-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
-; RV64-BITS-UNKNOWN-NEXT:    vrgatherei16.vv v10, v8, v12
-; RV64-BITS-UNKNOWN-NEXT:    vmv.v.v v8, v10
-; RV64-BITS-UNKNOWN-NEXT:    ret
-;
-; RV64-BITS-256-LABEL: reverse_v3i64:
-; RV64-BITS-256:       # %bb.0:
-; RV64-BITS-256-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
-; RV64-BITS-256-NEXT:    vid.v v10
-; RV64-BITS-256-NEXT:    vrsub.vi v12, v10, 2
-; RV64-BITS-256-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
-; RV64-BITS-256-NEXT:    vrgatherei16.vv v10, v8, v12
-; RV64-BITS-256-NEXT:    vmv.v.v v8, v10
-; RV64-BITS-256-NEXT:    ret
-;
-; RV64-BITS-512-LABEL: reverse_v3i64:
-; RV64-BITS-512:       # %bb.0:
-; RV64-BITS-512-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
-; RV64-BITS-512-NEXT:    vid.v v10
-; RV64-BITS-512-NEXT:    vrsub.vi v12, v10, 2
-; RV64-BITS-512-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
-; RV64-BITS-512-NEXT:    vrgatherei16.vv v10, v8, v12
-; RV64-BITS-512-NEXT:    vmv.v.v v8, v10
-; RV64-BITS-512-NEXT:    ret
+; RV32-LABEL: reverse_v3i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    lui a0, %hi(.LCPI44_0)
+; RV32-NEXT:    addi a0, a0, %lo(.LCPI44_0)
+; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV32-NEXT:    vle16.v v12, (a0)
+; RV32-NEXT:    vrgatherei16.vv v10, v8, v12
+; RV32-NEXT:    vmv.v.v v8, v10
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: reverse_v3i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; RV64-NEXT:    vid.v v10
+; RV64-NEXT:    vrsub.vi v12, v10, 2
+; RV64-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
+; RV64-NEXT:    vrgatherei16.vv v10, v8, v12
+; RV64-NEXT:    vmv.v.v v8, v10
+; RV64-NEXT:    ret
 ;
 ; RV32-ZVBB-LABEL: reverse_v3i64:
 ; RV32-ZVBB:       # %bb.0:
@@ -732,65 +688,25 @@ define <3 x i64> @reverse_v3i64(<3 x i64> %a) {
 }
 
 define <6 x i64> @reverse_v6i64(<6 x i64> %a) {
-; RV32-BITS-UNKNOWN-LABEL: reverse_v6i64:
-; RV32-BITS-UNKNOWN:       # %bb.0:
-; RV32-BITS-UNKNOWN-NEXT:    lui a0, %hi(.LCPI45_0)
-; RV32-BITS-UNKNOWN-NEXT:    addi a0, a0, %lo(.LCPI45_0)
-; RV32-BITS-UNKNOWN-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
-; RV32-BITS-UNKNOWN-NEXT:    vle16.v v16, (a0)
-; RV32-BITS-UNKNOWN-NEXT:    vrgatherei16.vv v12, v8, v16
-; RV32-BITS-UNKNOWN-NEXT:    vmv.v.v v8, v12
-; RV32-BITS-UNKNOWN-NEXT:    ret
-;
-; RV32-BITS-256-LABEL: reverse_v6i64:
-; RV32-BITS-256:       # %bb.0:
-; RV32-BITS-256-NEXT:    lui a0, %hi(.LCPI45_0)
-; RV32-BITS-256-NEXT:    addi a0, a0, %lo(.LCPI45_0)
-; RV32-BITS-256-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
-; RV32-BITS-256-NEXT:    vle16.v v16, (a0)
-; RV32-BITS-256-NEXT:    vrgatherei16.vv v12, v8, v16
-; RV32-BITS-256-NEXT:    vmv.v.v v8, v12
-; RV32-BITS-256-NEXT:    ret
-;
-; RV32-BITS-512-LABEL: reverse_v6i64:
-; RV32-BITS-512:       # %bb.0:
-; RV32-BITS-512-NEXT:    lui a0, %hi(.LCPI45_0)
-; RV32-BITS-512-NEXT:    addi a0, a0, %lo(.LCPI45_0)
-; RV32-BITS-512-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
-; RV32-BITS-512-NEXT:    vle16.v v16, (a0)
-; RV32-BITS-512-NEXT:    vrgatherei16.vv v12, v8, v16
-; RV32-BITS-512-NEXT:    vmv.v.v v8, v12
-; RV32-BITS-512-NEXT:    ret
-;
-; RV64-BITS-UNKNOWN-LABEL: reverse_v6i64:
-; RV64-BITS-UNKNOWN:       # %bb.0:
-; RV64-BITS-UNKNOWN-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; RV64-BITS-UNKNOWN-NEXT:    vid.v v12
-; RV64-BITS-UNKNOWN-NEXT:    vrsub.vi v16, v12, 5
-; RV64-BITS-UNKNOWN-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
-; RV64-BITS-UNKNOWN-NEXT:    vrgatherei16.vv v12, v8, v16
-; RV64-BITS-UNKNOWN-NEXT:    vmv.v.v v8, v12
-; RV64-BITS-UNKNOWN-NEXT:    ret
-;
-; RV64-BITS-256-LABEL: reverse_v6i64:
-; RV64-BITS-256:       # %bb.0:
-; RV64-BITS-256-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; RV64-BITS-256-NEXT:    vid.v v12
-; RV64-BITS-256-NEXT:    vrsub.vi v16, v12, 5
-; RV64-BITS-256-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
-; RV64-BITS-256-NEXT:    vrgatherei16.vv v12, v8, v16
-; RV64-BITS-256-NEXT:    vmv.v.v v8, v12
-; RV64-BITS-256-NEXT:    ret
-;
-; RV64-BITS-512-LABEL: reverse_v6i64:
-; RV64-BITS-512:       # %bb.0:
-; RV64-BITS-512-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; RV64-BITS-512-NEXT:    vid.v v12
-; RV64-BITS-512-NEXT:    vrsub.vi v16, v12, 5
-; RV64-BITS-512-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
-; RV64-BITS-512-NEXT:    vrgatherei16.vv v12, v8, v16
-; RV64-BITS-512-NEXT:    vmv.v.v v8, v12
-; RV64-BITS-512-NEXT:    ret
+; RV32-LABEL: reverse_v6i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    lui a0, %hi(.LCPI45_0)
+; RV32-NEXT:    addi a0, a0, %lo(.LCPI45_0)
+; RV32-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
+; RV32-NEXT:    vle16.v v16, (a0)
+; RV32-NEXT:    vrgatherei16.vv v12, v8, v16
+; RV32-NEXT:    vmv.v.v v8, v12
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: reverse_v6i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; RV64-NEXT:    vid.v v12
+; RV64-NEXT:    vrsub.vi v16, v12, 5
+; RV64-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; RV64-NEXT:    vrgatherei16.vv v12, v8, v16
+; RV64-NEXT:    vmv.v.v v8, v12
+; RV64-NEXT:    ret
 ;
 ; RV32-ZVBB-LABEL: reverse_v6i64:
 ; RV32-ZVBB:       # %bb.0:
@@ -816,68 +732,26 @@ define <6 x i64> @reverse_v6i64(<6 x i64> %a) {
 }
 
 define <12 x i64> @reverse_v12i64(<12 x i64> %a) {
-; RV32-BITS-UNKNOWN-LABEL: reverse_v12i64:
-; RV32-BITS-UNKNOWN:       # %bb.0:
-; RV32-BITS-UNKNOWN-NEXT:    li a0, 32
-; RV32-BITS-UNKNOWN-NEXT:    lui a1, %hi(.LCPI46_0)
-; RV32-BITS-UNKNOWN-NEXT:    addi a1, a1, %lo(.LCPI46_0)
-; RV32-BITS-UNKNOWN-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
-; RV32-BITS-UNKNOWN-NEXT:    vle16.v v24, (a1)
-; RV32-BITS-UNKNOWN-NEXT:    vrgatherei16.vv v16, v8, v24
-; RV32-BITS-UNKNOWN-NEXT:    vmv.v.v v8, v16
-; RV32-BITS-UNKNOWN-NEXT:    ret
-;
-; RV32-BITS-256-LABEL: reverse_v12i64:
-; RV32-BITS-256:       # %bb.0:
-; RV32-BITS-256-NEXT:    li a0, 32
-; RV32-BITS-256-NEXT:    lui a1, %hi(.LCPI46_0)
-; RV32-BITS-256-NEXT:    addi a1, a1, %lo(.LCPI46_0)
-; RV32-BITS-256-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
-; RV32-BITS-256-NEXT:    vle16.v v24, (a1)
-; RV32-BITS-256-NEXT:    vrgatherei16.vv v16, v8, v24
-; RV32-BITS-256-NEXT:    vmv.v.v v8, v16
-; RV32-BITS-256-NEXT:    ret
-;
-; RV32-BITS-512-LABEL: reverse_v12i64:
-; RV32-BITS-512:       # %bb.0:
-; RV32-BITS-512-NEXT:    li a0, 32
-; RV32-BITS-512-NEXT:    lui a1, %hi(.LCPI46_0)
-; RV32-BITS-512-NEXT:    addi a1, a1, %lo(.LCPI46_0)
-; RV32-BITS-512-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
-; RV32-BITS-512-NEXT:    vle16.v v24, (a1)
-; RV32-BITS-512-NEXT:    vrgatherei16.vv v16, v8, v24
-; RV32-BITS-512-NEXT:    vmv.v.v v8, v16
-; RV32-BITS-512-NEXT:    ret
-;
-; RV64-BITS-UNKNOWN-LABEL: reverse_v12i64:
-; RV64-BITS-UNKNOWN:       # %bb.0:
-; RV64-BITS-UNKNOWN-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
-; RV64-BITS-UNKNOWN-NEXT:    vid.v v16
-; RV64-BITS-UNKNOWN-NEXT:    vrsub.vi v24, v16, 11
-; RV64-BITS-UNKNOWN-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
-; RV64-BITS-UNKNOWN-NEXT:    vrgatherei16.vv v16, v8, v24
-; RV64-BITS-UNKNOWN-NEXT:    vmv.v.v v8, v16
-; RV64-BITS-UNKNOWN-NEXT:    ret
-;
-; RV64-BITS-256-LABEL: reverse_v12i64:
-; RV64-BITS-256:       # %bb.0:
-; RV64-BITS-256-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
-; RV64-BITS-256-NEXT:    vid.v v16
-; RV64-BITS-256-NEXT:    vrsub.vi v24, v16, 11
-; RV64-BITS-256-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
-; RV64-BITS-256-NEXT:    vrgatherei16.vv v16, v8, v24
-; RV64-BITS-256-NEXT:    vmv.v.v v8, v16
-; RV64-BITS-256-NEXT:    ret
-;
-; RV64-BITS-512-LABEL: reverse_v12i64:
-; RV64-BITS-512:       # %bb.0:
-; RV64-BITS-512-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
-; RV64-BITS-512-NEXT:    vid.v v16
-; RV64-BITS-512-NEXT:    vrsub.vi v24, v16, 11
-; RV64-BITS-512-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
-; RV64-BITS-512-NEXT:    vrgatherei16.vv v16, v8, v24
-; RV64-BITS-512-NEXT:    vmv.v.v v8, v16
-; RV64-BITS-512-NEXT:    ret
+; RV32-LABEL: reverse_v12i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    li a0, 32
+; RV32-NEXT:    lui a1, %hi(.LCPI46_0)
+; RV32-NEXT:    addi a1, a1, %lo(.LCPI46_0)
+; RV32-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
+; RV32-NEXT:    vle16.v v24, (a1)
+; RV32-NEXT:    vrgatherei16.vv v16, v8, v24
+; RV32-NEXT:    vmv.v.v v8, v16
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: reverse_v12i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
+; RV64-NEXT:    vid.v v16
+; RV64-NEXT:    vrsub.vi v24, v16, 11
+; RV64-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
+; RV64-NEXT:    vrgatherei16.vv v16, v8, v24
+; RV64-NEXT:    vmv.v.v v8, v16
+; RV64-NEXT:    ret
 ;
 ; RV32-ZVBB-LABEL: reverse_v12i64:
 ; RV32-ZVBB:       # %bb.0:

From e4b24e68a2b55a59dc29700ae6a281c2d4665af0 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke@igalia.com>
Date: Fri, 16 Aug 2024 15:29:03 +0800
Subject: [PATCH 133/441] [RISCV] Use shufflevector in shuffle reverse tests.
 NFC

Fixed length @llvm.vector.reverses are converted to vector_shuffle
nodes in SelectionDAGBuilder anyway, and this matches the name of the
test file.
---
 .../rvv/fixed-vectors-shuffle-reverse.ll      | 142 ++++++------------
 1 file changed, 47 insertions(+), 95 deletions(-)

diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-reverse.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-reverse.ll
index bab2e9ba1963745..2ea51515187e23b 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-reverse.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-reverse.ll
@@ -25,7 +25,7 @@ define <2 x i1> @reverse_v2i1(<2 x i1> %a) {
 ; ZVBB-NEXT:    vbrev.v v8, v0
 ; ZVBB-NEXT:    vsrl.vi v0, v8, 6
 ; ZVBB-NEXT:    ret
-  %res = call <2 x i1> @llvm.vector.reverse.v2i1(<2 x i1> %a)
+  %res = shufflevector <2 x i1> %a, <2 x i1> poison, <2 x i32> <i32 1, i32 0>
   ret <2 x i1> %res
 }
 
@@ -47,7 +47,7 @@ define <4 x i1> @reverse_v4i1(<4 x i1> %a) {
 ; ZVBB-NEXT:    vbrev.v v8, v0
 ; ZVBB-NEXT:    vsrl.vi v0, v8, 4
 ; ZVBB-NEXT:    ret
-  %res = call <4 x i1> @llvm.vector.reverse.v4i1(<4 x i1> %a)
+  %res = shufflevector <4 x i1> %a, <4 x i1> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
   ret <4 x i1> %res
 }
 
@@ -68,7 +68,7 @@ define <8 x i1> @reverse_v8i1(<8 x i1> %a) {
 ; ZVBB-NEXT:    vsetivli zero, 1, e8, mf8, ta, ma
 ; ZVBB-NEXT:    vbrev.v v0, v0
 ; ZVBB-NEXT:    ret
-  %res = call <8 x i1> @llvm.vector.reverse.v8i1(<8 x i1> %a)
+  %res = shufflevector <8 x i1> %a, <8 x i1> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <8 x i1> %res
 }
 
@@ -89,7 +89,7 @@ define <16 x i1> @reverse_v16i1(<16 x i1> %a) {
 ; ZVBB-NEXT:    vsetivli zero, 1, e16, mf4, ta, ma
 ; ZVBB-NEXT:    vbrev.v v0, v0
 ; ZVBB-NEXT:    ret
-  %res = call <16 x i1> @llvm.vector.reverse.v16i1(<16 x i1> %a)
+  %res = shufflevector <16 x i1> %a, <16 x i1> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <16 x i1> %res
 }
 
@@ -112,7 +112,7 @@ define <32 x i1> @reverse_v32i1(<32 x i1> %a) {
 ; ZVBB-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
 ; ZVBB-NEXT:    vbrev.v v0, v0
 ; ZVBB-NEXT:    ret
-  %res = call <32 x i1> @llvm.vector.reverse.v32i1(<32 x i1> %a)
+  %res = shufflevector <32 x i1> %a, <32 x i1> poison, <32 x i32> <i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <32 x i1> %res
 }
 
@@ -135,7 +135,7 @@ define <64 x i1> @reverse_v64i1(<64 x i1> %a) {
 ; ZVBB-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; ZVBB-NEXT:    vbrev.v v0, v0
 ; ZVBB-NEXT:    ret
-  %res = call <64 x i1> @llvm.vector.reverse.v64i1(<64 x i1> %a)
+  %res = shufflevector <64 x i1> %a, <64 x i1> poison, <64 x i32> <i32 63, i32 62, i32 61, i32 60, i32 59, i32 58, i32 57, i32 56, i32 55, i32 54, i32 53, i32 52, i32 51, i32 50, i32 49, i32 48, i32 47, i32 46, i32 45, i32 44, i32 43, i32 42, i32 41, i32 40, i32 39, i32 38, i32 37, i32 36, i32 35, i32 34, i32 33, i32 32, i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <64 x i1> %res
 }
 
@@ -152,7 +152,7 @@ define <128 x i1> @reverse_v128i1(<128 x i1> %a) {
 ; CHECK-NEXT:    vrgather.vv v24, v16, v8
 ; CHECK-NEXT:    vmsne.vi v0, v24, 0
 ; CHECK-NEXT:    ret
-  %res = call <128 x i1> @llvm.vector.reverse.v128i1(<128 x i1> %a)
+  %res = shufflevector <128 x i1> %a, <128 x i1> poison, <128 x i32> <i32 127, i32 126, i32 125, i32 124, i32 123, i32 122, i32 121, i32 120, i32 119, i32 118, i32 117, i32 116, i32 115, i32 114, i32 113, i32 112, i32 111, i32 110, i32 109, i32 108, i32 107, i32 106, i32 105, i32 104, i32 103, i32 102, i32 101, i32 100, i32 99, i32 98, i32 97, i32 96, i32 95, i32 94, i32 93, i32 92, i32 91, i32 90, i32 89, i32 88, i32 87, i32 86, i32 85, i32 84, i32 83, i32 82, i32 81, i32 80, i32 79, i32 78, i32 77, i32 76, i32 75, i32 74, i32 73, i32 72, i32 71, i32 70, i32 69, i32 68, i32 67, i32 66, i32 65, i32 64, i32 63, i32 62, i32 61, i32 60, i32 59, i32 58, i32 57, i32 56, i32 55, i32 54, i32 53, i32 52, i32 51, i32 50, i32 49, i32 48, i32 47, i32 46, i32 45, i32 44, i32 43, i32 42, i32 41, i32 40, i32 39, i32 38, i32 37, i32 36, i32 35, i32 34, i32 33, i32 32, i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <128 x i1> %res
 }
 
@@ -160,7 +160,7 @@ define <1 x i8> @reverse_v1i8(<1 x i8> %a) {
 ; CHECK-LABEL: reverse_v1i8:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ret
-  %res = call <1 x i8> @llvm.vector.reverse.v1i8(<1 x i8> %a)
+  %res = shufflevector <1 x i8> %a, <1 x i8> poison, <1 x i32> <i32 0>
   ret <1 x i8> %res
 }
 
@@ -178,7 +178,7 @@ define <2 x i8> @reverse_v2i8(<2 x i8> %a) {
 ; ZVBB-NEXT:    vsetivli zero, 1, e16, mf4, ta, ma
 ; ZVBB-NEXT:    vrev8.v v8, v8
 ; ZVBB-NEXT:    ret
-  %res = call <2 x i8> @llvm.vector.reverse.v2i8(<2 x i8> %a)
+  %res = shufflevector <2 x i8> %a, <2 x i8> poison, <2 x i32> <i32 1, i32 0>
   ret <2 x i8> %res
 }
 
@@ -191,7 +191,7 @@ define <4 x i8> @reverse_v4i8(<4 x i8> %a) {
 ; CHECK-NEXT:    vrgather.vv v9, v8, v10
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
-  %res = call <4 x i8> @llvm.vector.reverse.v4i8(<4 x i8> %a)
+  %res = shufflevector <4 x i8> %a, <4 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
   ret <4 x i8> %res
 }
 
@@ -204,7 +204,7 @@ define <8 x i8> @reverse_v8i8(<8 x i8> %a) {
 ; CHECK-NEXT:    vrgather.vv v9, v8, v10
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
-  %res = call <8 x i8> @llvm.vector.reverse.v8i8(<8 x i8> %a)
+  %res = shufflevector <8 x i8> %a, <8 x i8> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <8 x i8> %res
 }
 
@@ -217,7 +217,7 @@ define <16 x i8> @reverse_v16i8(<16 x i8> %a) {
 ; CHECK-NEXT:    vrgather.vv v9, v8, v10
 ; CHECK-NEXT:    vmv.v.v v8, v9
 ; CHECK-NEXT:    ret
-  %res = call <16 x i8> @llvm.vector.reverse.v16i8(<16 x i8> %a)
+  %res = shufflevector <16 x i8> %a, <16 x i8> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <16 x i8> %res
 }
 
@@ -232,7 +232,7 @@ define <32 x i8> @reverse_v32i8(<32 x i8> %a) {
 ; CHECK-NEXT:    vrgather.vv v10, v8, v12
 ; CHECK-NEXT:    vmv.v.v v8, v10
 ; CHECK-NEXT:    ret
-  %res = call <32 x i8> @llvm.vector.reverse.v32i8(<32 x i8> %a)
+  %res = shufflevector <32 x i8> %a, <32 x i8> poison, <32 x i32> <i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <32 x i8> %res
 }
 
@@ -247,7 +247,7 @@ define <64 x i8> @reverse_v64i8(<64 x i8> %a) {
 ; CHECK-NEXT:    vrgather.vv v12, v8, v16
 ; CHECK-NEXT:    vmv.v.v v8, v12
 ; CHECK-NEXT:    ret
-  %res = call <64 x i8> @llvm.vector.reverse.v64i8(<64 x i8> %a)
+  %res = shufflevector <64 x i8> %a, <64 x i8> poison, <64 x i32> <i32 63, i32 62, i32 61, i32 60, i32 59, i32 58, i32 57, i32 56, i32 55, i32 54, i32 53, i32 52, i32 51, i32 50, i32 49, i32 48, i32 47, i32 46, i32 45, i32 44, i32 43, i32 42, i32 41, i32 40, i32 39, i32 38, i32 37, i32 36, i32 35, i32 34, i32 33, i32 32, i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <64 x i8> %res
 }
 
@@ -255,7 +255,7 @@ define <1 x i16> @reverse_v1i16(<1 x i16> %a) {
 ; CHECK-LABEL: reverse_v1i16:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ret
-  %res = call <1 x i16> @llvm.vector.reverse.v1i16(<1 x i16> %a)
+  %res = shufflevector <1 x i16> %a, <1 x i16> poison, <1 x i32> <i32 0>
   ret <1 x i16> %res
 }
 
@@ -273,7 +273,7 @@ define <2 x i16> @reverse_v2i16(<2 x i16> %a) {
 ; ZVBB-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
 ; ZVBB-NEXT:    vror.vi v8, v8, 16
 ; ZVBB-NEXT:    ret
-  %res = call <2 x i16> @llvm.vector.reverse.v2i16(<2 x i16> %a)
+  %res = shufflevector <2 x i16> %a, <2 x i16> poison, <2 x i32> <i32 1, i32 0>
   ret <2 x i16> %res
 }
 
@@ -286,7 +286,7 @@ define <4 x i16> @reverse_v4i16(<4 x i16> %a) {
 ; CHECK-NEXT:    vrgather.vv v9, v8, v10
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
-  %res = call <4 x i16> @llvm.vector.reverse.v4i16(<4 x i16> %a)
+  %res = shufflevector <4 x i16> %a, <4 x i16> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
   ret <4 x i16> %res
 }
 
@@ -299,7 +299,7 @@ define <8 x i16> @reverse_v8i16(<8 x i16> %a) {
 ; CHECK-NEXT:    vrgather.vv v9, v8, v10
 ; CHECK-NEXT:    vmv.v.v v8, v9
 ; CHECK-NEXT:    ret
-  %res = call <8 x i16> @llvm.vector.reverse.v8i16(<8 x i16> %a)
+  %res = shufflevector <8 x i16> %a, <8 x i16> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <8 x i16> %res
 }
 
@@ -312,7 +312,7 @@ define <16 x i16> @reverse_v16i16(<16 x i16> %a) {
 ; CHECK-NEXT:    vrgather.vv v10, v8, v12
 ; CHECK-NEXT:    vmv.v.v v8, v10
 ; CHECK-NEXT:    ret
-  %res = call <16 x i16> @llvm.vector.reverse.v16i16(<16 x i16> %a)
+  %res = shufflevector <16 x i16> %a, <16 x i16> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <16 x i16> %res
 }
 
@@ -327,7 +327,7 @@ define <32 x i16> @reverse_v32i16(<32 x i16> %a) {
 ; CHECK-NEXT:    vrgather.vv v12, v8, v16
 ; CHECK-NEXT:    vmv.v.v v8, v12
 ; CHECK-NEXT:    ret
-  %res = call <32 x i16> @llvm.vector.reverse.v32i16(<32 x i16> %a)
+  %res = shufflevector <32 x i16> %a, <32 x i16> poison, <32 x i32> <i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <32 x i16> %res
 }
 
@@ -335,7 +335,7 @@ define <1 x i32> @reverse_v1i32(<1 x i32> %a) {
 ; CHECK-LABEL: reverse_v1i32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ret
-  %res = call <1 x i32> @llvm.vector.reverse.v1i32(<1 x i32> %a)
+  %res = shufflevector <1 x i32> %a, <1 x i32> poison, <1 x i32> <i32 0>
   ret <1 x i32> %res
 }
 
@@ -353,7 +353,7 @@ define <2 x i32> @reverse_v2i32(<2 x i32> %a) {
 ; ZVBB-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; ZVBB-NEXT:    vror.vi v8, v8, 32
 ; ZVBB-NEXT:    ret
-  %res = call <2 x i32> @llvm.vector.reverse.v2i32(<2 x i32> %a)
+  %res = shufflevector <2 x i32> %a, <2 x i32> poison, <2 x i32> <i32 1, i32 0>
   ret <2 x i32> %res
 }
 
@@ -366,7 +366,7 @@ define <4 x i32> @reverse_v4i32(<4 x i32> %a) {
 ; CHECK-NEXT:    vrgather.vv v9, v8, v10
 ; CHECK-NEXT:    vmv.v.v v8, v9
 ; CHECK-NEXT:    ret
-  %res = call <4 x i32> @llvm.vector.reverse.v4i32(<4 x i32> %a)
+  %res = shufflevector <4 x i32> %a, <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
   ret <4 x i32> %res
 }
 
@@ -380,7 +380,7 @@ define <8 x i32> @reverse_v8i32(<8 x i32> %a) {
 ; CHECK-NEXT:    vrgatherei16.vv v10, v8, v12
 ; CHECK-NEXT:    vmv.v.v v8, v10
 ; CHECK-NEXT:    ret
-  %res = call <8 x i32> @llvm.vector.reverse.v8i32(<8 x i32> %a)
+  %res = shufflevector <8 x i32> %a, <8 x i32> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <8 x i32> %res
 }
 
@@ -394,7 +394,7 @@ define <16 x i32> @reverse_v16i32(<16 x i32> %a) {
 ; CHECK-NEXT:    vrgatherei16.vv v12, v8, v16
 ; CHECK-NEXT:    vmv.v.v v8, v12
 ; CHECK-NEXT:    ret
-  %res = call <16 x i32> @llvm.vector.reverse.v16i32(<16 x i32> %a)
+  %res = shufflevector <16 x i32> %a, <16 x i32> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <16 x i32> %res
 }
 
@@ -402,7 +402,7 @@ define <1 x i64> @reverse_v1i64(<1 x i64> %a) {
 ; CHECK-LABEL: reverse_v1i64:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ret
-  %res = call <1 x i64> @llvm.vector.reverse.v1i64(<1 x i64> %a)
+  %res = shufflevector <1 x i64> %a, <1 x i64> poison, <1 x i32> <i32 0>
   ret <1 x i64> %res
 }
 
@@ -414,7 +414,7 @@ define <2 x i64> @reverse_v2i64(<2 x i64> %a) {
 ; CHECK-NEXT:    vslideup.vi v9, v8, 1
 ; CHECK-NEXT:    vmv.v.v v8, v9
 ; CHECK-NEXT:    ret
-  %res = call <2 x i64> @llvm.vector.reverse.v2i64(<2 x i64> %a)
+  %res = shufflevector <2 x i64> %a, <2 x i64> poison, <2 x i32> <i32 1, i32 0>
   ret <2 x i64> %res
 }
 
@@ -428,7 +428,7 @@ define <4 x i64> @reverse_v4i64(<4 x i64> %a) {
 ; CHECK-NEXT:    vrgatherei16.vv v10, v8, v12
 ; CHECK-NEXT:    vmv.v.v v8, v10
 ; CHECK-NEXT:    ret
-  %res = call <4 x i64> @llvm.vector.reverse.v4i64(<4 x i64> %a)
+  %res = shufflevector <4 x i64> %a, <4 x i64> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
   ret <4 x i64> %res
 }
 
@@ -442,7 +442,7 @@ define <8 x i64> @reverse_v8i64(<8 x i64> %a) {
 ; CHECK-NEXT:    vrgatherei16.vv v12, v8, v16
 ; CHECK-NEXT:    vmv.v.v v8, v12
 ; CHECK-NEXT:    ret
-  %res = call <8 x i64> @llvm.vector.reverse.v8i64(<8 x i64> %a)
+  %res = shufflevector <8 x i64> %a, <8 x i64> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <8 x i64> %res
 }
 
@@ -451,7 +451,7 @@ define <1 x half> @reverse_v1f16(<1 x half> %a) {
 ; CHECK-LABEL: reverse_v1f16:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ret
-  %res = call <1 x half> @llvm.vector.reverse.v1f16(<1 x half> %a)
+  %res = shufflevector <1 x half> %a, <1 x half> poison, <1 x i32> <i32 0>
   ret <1 x half> %res
 }
 
@@ -469,7 +469,7 @@ define <2 x half> @reverse_v2f16(<2 x half> %a) {
 ; ZVBB-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
 ; ZVBB-NEXT:    vror.vi v8, v8, 16
 ; ZVBB-NEXT:    ret
-  %res = call <2 x half> @llvm.vector.reverse.v2f16(<2 x half> %a)
+  %res = shufflevector <2 x half> %a, <2 x half> poison, <2 x i32> <i32 1, i32 0>
   ret <2 x half> %res
 }
 
@@ -482,7 +482,7 @@ define <4 x half> @reverse_v4f16(<4 x half> %a) {
 ; CHECK-NEXT:    vrgather.vv v9, v8, v10
 ; CHECK-NEXT:    vmv1r.v v8, v9
 ; CHECK-NEXT:    ret
-  %res = call <4 x half> @llvm.vector.reverse.v4f16(<4 x half> %a)
+  %res = shufflevector <4 x half> %a, <4 x half> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
   ret <4 x half> %res
 }
 
@@ -495,7 +495,7 @@ define <8 x half> @reverse_v8f16(<8 x half> %a) {
 ; CHECK-NEXT:    vrgather.vv v9, v8, v10
 ; CHECK-NEXT:    vmv.v.v v8, v9
 ; CHECK-NEXT:    ret
-  %res = call <8 x half> @llvm.vector.reverse.v8f16(<8 x half> %a)
+  %res = shufflevector <8 x half> %a, <8 x half> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <8 x half> %res
 }
 
@@ -508,7 +508,7 @@ define <16 x half> @reverse_v16f16(<16 x half> %a) {
 ; CHECK-NEXT:    vrgather.vv v10, v8, v12
 ; CHECK-NEXT:    vmv.v.v v8, v10
 ; CHECK-NEXT:    ret
-  %res = call <16 x half> @llvm.vector.reverse.v16f16(<16 x half> %a)
+  %res = shufflevector <16 x half> %a, <16 x half> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <16 x half> %res
 }
 
@@ -523,7 +523,7 @@ define <32 x half> @reverse_v32f16(<32 x half> %a) {
 ; CHECK-NEXT:    vrgather.vv v12, v8, v16
 ; CHECK-NEXT:    vmv.v.v v8, v12
 ; CHECK-NEXT:    ret
-  %res = call <32 x half> @llvm.vector.reverse.v32f16(<32 x half> %a)
+  %res = shufflevector <32 x half> %a, <32 x half> poison, <32 x i32> <i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <32 x half> %res
 }
 
@@ -531,7 +531,7 @@ define <1 x float> @reverse_v1f32(<1 x float> %a) {
 ; CHECK-LABEL: reverse_v1f32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ret
-  %res = call <1 x float> @llvm.vector.reverse.v1f32(<1 x float> %a)
+  %res = shufflevector <1 x float> %a, <1 x float> poison, <1 x i32> <i32 0>
   ret <1 x float> %res
 }
 
@@ -549,7 +549,7 @@ define <2 x float> @reverse_v2f32(<2 x float> %a) {
 ; ZVBB-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
 ; ZVBB-NEXT:    vror.vi v8, v8, 32
 ; ZVBB-NEXT:    ret
-  %res = call <2 x float> @llvm.vector.reverse.v2f32(<2 x float> %a)
+  %res = shufflevector <2 x float> %a, <2 x float> poison, <2 x i32> <i32 1, i32 0>
   ret <2 x float> %res
 }
 
@@ -562,7 +562,7 @@ define <4 x float> @reverse_v4f32(<4 x float> %a) {
 ; CHECK-NEXT:    vrgather.vv v9, v8, v10
 ; CHECK-NEXT:    vmv.v.v v8, v9
 ; CHECK-NEXT:    ret
-  %res = call <4 x float> @llvm.vector.reverse.v4f32(<4 x float> %a)
+  %res = shufflevector <4 x float> %a, <4 x float> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
   ret <4 x float> %res
 }
 
@@ -576,7 +576,7 @@ define <8 x float> @reverse_v8f32(<8 x float> %a) {
 ; CHECK-NEXT:    vrgatherei16.vv v10, v8, v12
 ; CHECK-NEXT:    vmv.v.v v8, v10
 ; CHECK-NEXT:    ret
-  %res = call <8 x float> @llvm.vector.reverse.v8f32(<8 x float> %a)
+  %res = shufflevector <8 x float> %a, <8 x float> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <8 x float> %res
 }
 
@@ -590,7 +590,7 @@ define <16 x float> @reverse_v16f32(<16 x float> %a) {
 ; CHECK-NEXT:    vrgatherei16.vv v12, v8, v16
 ; CHECK-NEXT:    vmv.v.v v8, v12
 ; CHECK-NEXT:    ret
-  %res = call <16 x float> @llvm.vector.reverse.v16f32(<16 x float> %a)
+  %res = shufflevector <16 x float> %a, <16 x float> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <16 x float> %res
 }
 
@@ -598,7 +598,7 @@ define <1 x double> @reverse_v1f64(<1 x double> %a) {
 ; CHECK-LABEL: reverse_v1f64:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ret
-  %res = call <1 x double> @llvm.vector.reverse.v1f64(<1 x double> %a)
+  %res = shufflevector <1 x double> %a, <1 x double> poison, <1 x i32> <i32 0>
   ret <1 x double> %res
 }
 
@@ -610,7 +610,7 @@ define <2 x double> @reverse_v2f64(<2 x double> %a) {
 ; CHECK-NEXT:    vslideup.vi v9, v8, 1
 ; CHECK-NEXT:    vmv.v.v v8, v9
 ; CHECK-NEXT:    ret
-  %res = call <2 x double> @llvm.vector.reverse.v2f64(<2 x double> %a)
+  %res = shufflevector <2 x double> %a, <2 x double> poison, <2 x i32> <i32 1, i32 0>
   ret <2 x double> %res
 }
 
@@ -624,7 +624,7 @@ define <4 x double> @reverse_v4f64(<4 x double> %a) {
 ; CHECK-NEXT:    vrgatherei16.vv v10, v8, v12
 ; CHECK-NEXT:    vmv.v.v v8, v10
 ; CHECK-NEXT:    ret
-  %res = call <4 x double> @llvm.vector.reverse.v4f64(<4 x double> %a)
+  %res = shufflevector <4 x double> %a, <4 x double> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
   ret <4 x double> %res
 }
 
@@ -638,7 +638,7 @@ define <8 x double> @reverse_v8f64(<8 x double> %a) {
 ; CHECK-NEXT:    vrgatherei16.vv v12, v8, v16
 ; CHECK-NEXT:    vmv.v.v v8, v12
 ; CHECK-NEXT:    ret
-  %res = call <8 x double> @llvm.vector.reverse.v8f64(<8 x double> %a)
+  %res = shufflevector <8 x double> %a, <8 x double> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <8 x double> %res
 }
 
@@ -683,7 +683,7 @@ define <3 x i64> @reverse_v3i64(<3 x i64> %a) {
 ; RV64-ZVBB-NEXT:    vrgatherei16.vv v10, v8, v12
 ; RV64-ZVBB-NEXT:    vmv.v.v v8, v10
 ; RV64-ZVBB-NEXT:    ret
-  %res = call <3 x i64> @llvm.vector.reverse.v3i64(<3 x i64> %a)
+  %res = shufflevector <3 x i64> %a, <3 x i64> poison, <3 x i32> <i32 2, i32 1, i32 0>
   ret <3 x i64> %res
 }
 
@@ -727,7 +727,7 @@ define <6 x i64> @reverse_v6i64(<6 x i64> %a) {
 ; RV64-ZVBB-NEXT:    vrgatherei16.vv v12, v8, v16
 ; RV64-ZVBB-NEXT:    vmv.v.v v8, v12
 ; RV64-ZVBB-NEXT:    ret
-  %res = call <6 x i64> @llvm.vector.reverse.v6i64(<6 x i64> %a)
+  %res = shufflevector <6 x i64> %a, <6 x i64> poison, <6 x i32> <i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <6 x i64> %res
 }
 
@@ -773,54 +773,6 @@ define <12 x i64> @reverse_v12i64(<12 x i64> %a) {
 ; RV64-ZVBB-NEXT:    vrgatherei16.vv v16, v8, v24
 ; RV64-ZVBB-NEXT:    vmv.v.v v8, v16
 ; RV64-ZVBB-NEXT:    ret
-  %res = call <12 x i64> @llvm.vector.reverse.v12i64(<12 x i64> %a)
+  %res = shufflevector <12 x i64> %a, <12 x i64> poison, <12 x i32> <i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <12 x i64> %res
 }
-
-declare <2 x i1> @llvm.vector.reverse.v2i1(<2 x i1>)
-declare <4 x i1> @llvm.vector.reverse.v4i1(<4 x i1>)
-declare <8 x i1> @llvm.vector.reverse.v8i1(<8 x i1>)
-declare <16 x i1> @llvm.vector.reverse.v16i1(<16 x i1>)
-declare <32 x i1> @llvm.vector.reverse.v32i1(<32 x i1>)
-declare <64 x i1> @llvm.vector.reverse.v64i1(<64 x i1>)
-declare <128 x i1> @llvm.vector.reverse.v128i1(<128 x i1>)
-declare <1 x i8> @llvm.vector.reverse.v1i8(<1 x i8>)
-declare <2 x i8> @llvm.vector.reverse.v2i8(<2 x i8>)
-declare <4 x i8> @llvm.vector.reverse.v4i8(<4 x i8>)
-declare <8 x i8> @llvm.vector.reverse.v8i8(<8 x i8>)
-declare <16 x i8> @llvm.vector.reverse.v16i8(<16 x i8>)
-declare <32 x i8> @llvm.vector.reverse.v32i8(<32 x i8>)
-declare <64 x i8> @llvm.vector.reverse.v64i8(<64 x i8>)
-declare <1 x i16> @llvm.vector.reverse.v1i16(<1 x i16>)
-declare <2 x i16> @llvm.vector.reverse.v2i16(<2 x i16>)
-declare <4 x i16> @llvm.vector.reverse.v4i16(<4 x i16>)
-declare <8 x i16> @llvm.vector.reverse.v8i16(<8 x i16>)
-declare <16 x i16> @llvm.vector.reverse.v16i16(<16 x i16>)
-declare <32 x i16> @llvm.vector.reverse.v32i16(<32 x i16>)
-declare <1 x i32> @llvm.vector.reverse.v1i32(<1 x i32>)
-declare <2 x i32> @llvm.vector.reverse.v2i32(<2 x i32>)
-declare <4 x i32> @llvm.vector.reverse.v4i32(<4 x i32>)
-declare <8 x i32> @llvm.vector.reverse.v8i32(<8 x i32>)
-declare <16 x i32> @llvm.vector.reverse.v16i32(<16 x i32>)
-declare <1 x i64> @llvm.vector.reverse.v1i64(<1 x i64>)
-declare <2 x i64> @llvm.vector.reverse.v2i64(<2 x i64>)
-declare <4 x i64> @llvm.vector.reverse.v4i64(<4 x i64>)
-declare <8 x i64> @llvm.vector.reverse.v8i64(<8 x i64>)
-declare <1 x half> @llvm.vector.reverse.v1f16(<1 x half>)
-declare <2 x half> @llvm.vector.reverse.v2f16(<2 x half>)
-declare <4 x half> @llvm.vector.reverse.v4f16(<4 x half>)
-declare <8 x half> @llvm.vector.reverse.v8f16(<8 x half>)
-declare <16 x half> @llvm.vector.reverse.v16f16(<16 x half>)
-declare <32 x half> @llvm.vector.reverse.v32f16(<32 x half>)
-declare <1 x float> @llvm.vector.reverse.v1f32(<1 x float>)
-declare <2 x float> @llvm.vector.reverse.v2f32(<2 x float>)
-declare <4 x float> @llvm.vector.reverse.v4f32(<4 x float>)
-declare <8 x float> @llvm.vector.reverse.v8f32(<8 x float>)
-declare <16 x float> @llvm.vector.reverse.v16f32(<16 x float>)
-declare <1 x double> @llvm.vector.reverse.v1f64(<1 x double>)
-declare <2 x double> @llvm.vector.reverse.v2f64(<2 x double>)
-declare <4 x double> @llvm.vector.reverse.v4f64(<4 x double>)
-declare <8 x double> @llvm.vector.reverse.v8f64(<8 x double>)
-declare <3 x i64> @llvm.vector.reverse.v3i64(<3 x i64>)
-declare <6 x i64> @llvm.vector.reverse.v6i64(<6 x i64>)
-declare <12 x i64> @llvm.vector.reverse.v12i64(<12 x i64>)

From 8d71016eded39ee6a3d69a8219006f0327bea23c Mon Sep 17 00:00:00 2001
From: Luke Lau <luke@igalia.com>
Date: Fri, 16 Aug 2024 15:45:51 +0800
Subject: [PATCH 134/441] [RISCV] Merge shuffle reverse tests. NFC

shuffle-reverse.ll also tests reverse shuffles with two source
vectors, so copy them over.
---
 .../rvv/fixed-vectors-shuffle-reverse.ll      | 464 ++++++++++-
 .../test/CodeGen/RISCV/rvv/shuffle-reverse.ll | 734 ------------------
 2 files changed, 460 insertions(+), 738 deletions(-)
 delete mode 100644 llvm/test/CodeGen/RISCV/rvv/shuffle-reverse.ll

diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-reverse.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-reverse.ll
index 2ea51515187e23b..cb7fde9188e813d 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-reverse.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-reverse.ll
@@ -4,10 +4,6 @@
 ; RUN: llc -mtriple=riscv32 -mattr=+m,+v,+f,+d,+zfh,+zvfh,+zvbb -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVBB,RV32-ZVBB
 ; RUN: llc -mtriple=riscv64 -mattr=+m,+v,+f,+d,+zfh,+zvfh,+zvbb -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZVBB,RV64-ZVBB
 
-;
-; VECTOR_REVERSE - masks
-;
-
 define <2 x i1> @reverse_v2i1(<2 x i1> %a) {
 ; NO-ZVBB-LABEL: reverse_v2i1:
 ; NO-ZVBB:       # %bb.0:
@@ -776,3 +772,463 @@ define <12 x i64> @reverse_v12i64(<12 x i64> %a) {
   %res = shufflevector <12 x i64> %a, <12 x i64> poison, <12 x i32> <i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <12 x i64> %res
 }
+
+define <4 x i8> @reverse_v4i8_2(<2 x i8> %a, <2 x i8> %b) {
+; NO-ZVBB-LABEL: reverse_v4i8_2:
+; NO-ZVBB:       # %bb.0:
+; NO-ZVBB-NEXT:    vsetivli zero, 2, e8, mf8, ta, ma
+; NO-ZVBB-NEXT:    vslidedown.vi v10, v8, 1
+; NO-ZVBB-NEXT:    vslideup.vi v10, v8, 1
+; NO-ZVBB-NEXT:    vslidedown.vi v8, v9, 1
+; NO-ZVBB-NEXT:    vslideup.vi v8, v9, 1
+; NO-ZVBB-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
+; NO-ZVBB-NEXT:    vslideup.vi v8, v10, 2
+; NO-ZVBB-NEXT:    ret
+;
+; ZVBB-LABEL: reverse_v4i8_2:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetivli zero, 1, e16, mf4, ta, ma
+; ZVBB-NEXT:    vrev8.v v10, v8
+; ZVBB-NEXT:    vrev8.v v8, v9
+; ZVBB-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
+; ZVBB-NEXT:    vslideup.vi v8, v10, 2
+; ZVBB-NEXT:    ret
+  %res = shufflevector <2 x i8> %a, <2 x i8> %b, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  ret <4 x i8> %res
+}
+
+define <8 x i8> @reverse_v8i8_2(<4 x i8> %a, <4 x i8> %b) {
+; CHECK-LABEL: reverse_v8i8_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, mu
+; CHECK-NEXT:    vid.v v11
+; CHECK-NEXT:    vrsub.vi v12, v11, 7
+; CHECK-NEXT:    vrgather.vv v10, v8, v12
+; CHECK-NEXT:    vmv.v.i v0, 15
+; CHECK-NEXT:    vrsub.vi v8, v11, 3
+; CHECK-NEXT:    vrgather.vv v10, v9, v8, v0.t
+; CHECK-NEXT:    vmv1r.v v8, v10
+; CHECK-NEXT:    ret
+  %res = shufflevector <4 x i8> %a, <4 x i8> %b, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <8 x i8> %res
+}
+
+define <16 x i8> @reverse_v16i8_2(<8 x i8> %a, <8 x i8> %b) {
+; CHECK-LABEL: reverse_v16i8_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-NEXT:    vid.v v11
+; CHECK-NEXT:    vrsub.vi v12, v11, 15
+; CHECK-NEXT:    vrgather.vv v10, v8, v12
+; CHECK-NEXT:    li a0, 255
+; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vmv.s.x v0, a0
+; CHECK-NEXT:    vsetvli zero, zero, e8, m1, ta, mu
+; CHECK-NEXT:    vrsub.vi v8, v11, 7
+; CHECK-NEXT:    vrgather.vv v10, v9, v8, v0.t
+; CHECK-NEXT:    vmv.v.v v8, v10
+; CHECK-NEXT:    ret
+  %res = shufflevector <8 x i8> %a, <8 x i8> %b, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <16 x i8> %res
+}
+
+define <32 x i8> @reverse_v32i8_2(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: reverse_v32i8_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmv1r.v v12, v9
+; CHECK-NEXT:    li a0, 32
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
+; CHECK-NEXT:    vid.v v14
+; CHECK-NEXT:    li a0, 31
+; CHECK-NEXT:    vrsub.vx v16, v14, a0
+; CHECK-NEXT:    vrgather.vv v10, v8, v16
+; CHECK-NEXT:    vrsub.vi v8, v14, 15
+; CHECK-NEXT:    lui a0, 16
+; CHECK-NEXT:    addi a0, a0, -1
+; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
+; CHECK-NEXT:    vmv.s.x v0, a0
+; CHECK-NEXT:    vsetvli zero, zero, e8, m2, ta, mu
+; CHECK-NEXT:    vrgather.vv v10, v12, v8, v0.t
+; CHECK-NEXT:    vmv.v.v v8, v10
+; CHECK-NEXT:    ret
+  %res = shufflevector <16 x i8> %a, <16 x i8> %b,  <32 x i32> <i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <32 x i8> %res
+}
+
+define <4 x i16> @reverse_v4i16_2(<2 x i16> %a, <2 x i16> %b) {
+; NO-ZVBB-LABEL: reverse_v4i16_2:
+; NO-ZVBB:       # %bb.0:
+; NO-ZVBB-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
+; NO-ZVBB-NEXT:    vslidedown.vi v10, v8, 1
+; NO-ZVBB-NEXT:    vslideup.vi v10, v8, 1
+; NO-ZVBB-NEXT:    vslidedown.vi v8, v9, 1
+; NO-ZVBB-NEXT:    vslideup.vi v8, v9, 1
+; NO-ZVBB-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; NO-ZVBB-NEXT:    vslideup.vi v8, v10, 2
+; NO-ZVBB-NEXT:    ret
+;
+; ZVBB-LABEL: reverse_v4i16_2:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
+; ZVBB-NEXT:    vror.vi v10, v8, 16
+; ZVBB-NEXT:    vror.vi v8, v9, 16
+; ZVBB-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; ZVBB-NEXT:    vslideup.vi v8, v10, 2
+; ZVBB-NEXT:    ret
+  %res = shufflevector <2 x i16> %a, <2 x i16> %b, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  ret <4 x i16> %res
+}
+
+define <8 x i16> @reverse_v8i16_2(<4 x i16> %a, <4 x i16> %b) {
+; CHECK-LABEL: reverse_v8i16_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, mu
+; CHECK-NEXT:    vid.v v11
+; CHECK-NEXT:    vrsub.vi v12, v11, 7
+; CHECK-NEXT:    vrgather.vv v10, v8, v12
+; CHECK-NEXT:    vmv.v.i v0, 15
+; CHECK-NEXT:    vrsub.vi v8, v11, 3
+; CHECK-NEXT:    vrgather.vv v10, v9, v8, v0.t
+; CHECK-NEXT:    vmv.v.v v8, v10
+; CHECK-NEXT:    ret
+  %res = shufflevector <4 x i16> %a, <4 x i16> %b, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <8 x i16> %res
+}
+
+define <16 x i16> @reverse_v16i16_2(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: reverse_v16i16_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmv1r.v v12, v9
+; CHECK-NEXT:    vsetivli zero, 16, e16, m2, ta, mu
+; CHECK-NEXT:    vid.v v14
+; CHECK-NEXT:    vrsub.vi v16, v14, 15
+; CHECK-NEXT:    vrgather.vv v10, v8, v16
+; CHECK-NEXT:    vrsub.vi v8, v14, 7
+; CHECK-NEXT:    li a0, 255
+; CHECK-NEXT:    vmv.s.x v0, a0
+; CHECK-NEXT:    vrgather.vv v10, v12, v8, v0.t
+; CHECK-NEXT:    vmv.v.v v8, v10
+; CHECK-NEXT:    ret
+  %res = shufflevector <8 x i16> %a, <8 x i16> %b,  <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <16 x i16> %res
+}
+
+define <32 x i16> @reverse_v32i16_2(<16 x i16> %a, <16 x i16> %b) {
+; CHECK-LABEL: reverse_v32i16_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmv2r.v v16, v10
+; CHECK-NEXT:    li a0, 32
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
+; CHECK-NEXT:    vid.v v20
+; CHECK-NEXT:    li a0, 31
+; CHECK-NEXT:    vrsub.vx v24, v20, a0
+; CHECK-NEXT:    vrgather.vv v12, v8, v24
+; CHECK-NEXT:    vrsub.vi v8, v20, 15
+; CHECK-NEXT:    lui a0, 16
+; CHECK-NEXT:    addi a0, a0, -1
+; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
+; CHECK-NEXT:    vmv.s.x v0, a0
+; CHECK-NEXT:    vsetvli zero, zero, e16, m4, ta, mu
+; CHECK-NEXT:    vrgather.vv v12, v16, v8, v0.t
+; CHECK-NEXT:    vmv.v.v v8, v12
+; CHECK-NEXT:    ret
+  %res = shufflevector <16 x i16> %a, <16 x i16> %b,  <32 x i32> <i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <32 x i16> %res
+}
+
+define <4 x i32> @reverse_v4i32_2(<2 x i32> %a, < 2 x i32> %b) {
+; NO-ZVBB-LABEL: reverse_v4i32_2:
+; NO-ZVBB:       # %bb.0:
+; NO-ZVBB-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; NO-ZVBB-NEXT:    vslidedown.vi v10, v8, 1
+; NO-ZVBB-NEXT:    vslideup.vi v10, v8, 1
+; NO-ZVBB-NEXT:    vslidedown.vi v8, v9, 1
+; NO-ZVBB-NEXT:    vslideup.vi v8, v9, 1
+; NO-ZVBB-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; NO-ZVBB-NEXT:    vslideup.vi v8, v10, 2
+; NO-ZVBB-NEXT:    ret
+;
+; ZVBB-LABEL: reverse_v4i32_2:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; ZVBB-NEXT:    vror.vi v10, v8, 32
+; ZVBB-NEXT:    vror.vi v8, v9, 32
+; ZVBB-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; ZVBB-NEXT:    vslideup.vi v8, v10, 2
+; ZVBB-NEXT:    ret
+  %res = shufflevector <2 x i32> %a, <2 x i32> %b, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  ret <4 x i32> %res
+}
+
+define <8 x i32> @reverse_v8i32_2(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: reverse_v8i32_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmv1r.v v12, v9
+; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-NEXT:    vid.v v9
+; CHECK-NEXT:    vrsub.vi v13, v9, 7
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vrgatherei16.vv v10, v8, v13
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vrsub.vi v8, v9, 3
+; CHECK-NEXT:    vmv.v.i v0, 15
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
+; CHECK-NEXT:    vrgatherei16.vv v10, v12, v8, v0.t
+; CHECK-NEXT:    vmv.v.v v8, v10
+; CHECK-NEXT:    ret
+  %res = shufflevector <4 x i32> %a, <4 x i32> %b, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <8 x i32> %res
+}
+
+define <16 x i32> @reverse_v16i32_2(<8 x i32> %a, <8 x i32> %b) {
+; CHECK-LABEL: reverse_v16i32_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmv2r.v v16, v10
+; CHECK-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
+; CHECK-NEXT:    vid.v v10
+; CHECK-NEXT:    vrsub.vi v18, v10, 15
+; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vrgatherei16.vv v12, v8, v18
+; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vrsub.vi v8, v10, 7
+; CHECK-NEXT:    li a0, 255
+; CHECK-NEXT:    vmv.s.x v0, a0
+; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, mu
+; CHECK-NEXT:    vrgatherei16.vv v12, v16, v8, v0.t
+; CHECK-NEXT:    vmv.v.v v8, v12
+; CHECK-NEXT:    ret
+  %res = shufflevector <8 x i32> %a, <8 x i32> %b,  <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <16 x i32> %res
+}
+
+define <32 x i32> @reverse_v32i32_2(<16 x i32> %a, <16 x i32> %b) {
+; CHECK-LABEL: reverse_v32i32_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmv4r.v v24, v12
+; CHECK-NEXT:    vmv4r.v v16, v8
+; CHECK-NEXT:    li a0, 32
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
+; CHECK-NEXT:    vid.v v20
+; CHECK-NEXT:    li a0, 31
+; CHECK-NEXT:    vrsub.vx v28, v20, a0
+; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
+; CHECK-NEXT:    vrgatherei16.vv v8, v16, v28
+; CHECK-NEXT:    vsetvli zero, zero, e16, m4, ta, ma
+; CHECK-NEXT:    vrsub.vi v16, v20, 15
+; CHECK-NEXT:    lui a0, 16
+; CHECK-NEXT:    addi a0, a0, -1
+; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, mu
+; CHECK-NEXT:    vmv.s.x v0, a0
+; CHECK-NEXT:    vrgatherei16.vv v8, v24, v16, v0.t
+; CHECK-NEXT:    ret
+  %res = shufflevector <16 x i32> %a, <16 x i32> %b,  <32 x i32> <i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <32 x i32> %res
+}
+
+define <4 x i64> @reverse_v4i64_2(<2 x i64> %a, < 2 x i64> %b) {
+; CHECK-LABEL: reverse_v4i64_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; CHECK-NEXT:    vslidedown.vi v10, v8, 1
+; CHECK-NEXT:    vslideup.vi v10, v8, 1
+; CHECK-NEXT:    vslidedown.vi v8, v9, 1
+; CHECK-NEXT:    vslideup.vi v8, v9, 1
+; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; CHECK-NEXT:    vslideup.vi v8, v10, 2
+; CHECK-NEXT:    ret
+  %res = shufflevector <2 x i64> %a, <2 x i64> %b, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  ret <4 x i64> %res
+}
+
+define <8 x i64> @reverse_v8i64_2(<4 x i64> %a, <4 x i64> %b) {
+; CHECK-LABEL: reverse_v8i64_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmv2r.v v16, v10
+; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-NEXT:    vid.v v10
+; CHECK-NEXT:    vrsub.vi v11, v10, 7
+; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; CHECK-NEXT:    vrgatherei16.vv v12, v8, v11
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vrsub.vi v8, v10, 3
+; CHECK-NEXT:    vmv.v.i v0, 15
+; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
+; CHECK-NEXT:    vrgatherei16.vv v12, v16, v8, v0.t
+; CHECK-NEXT:    vmv.v.v v8, v12
+; CHECK-NEXT:    ret
+  %res = shufflevector <4 x i64> %a, <4 x i64> %b, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <8 x i64> %res
+}
+
+define <4 x half> @reverse_v4f16_2(<2 x half> %a, <2 x half> %b) {
+; NO-ZVBB-LABEL: reverse_v4f16_2:
+; NO-ZVBB:       # %bb.0:
+; NO-ZVBB-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
+; NO-ZVBB-NEXT:    vslidedown.vi v10, v8, 1
+; NO-ZVBB-NEXT:    vslideup.vi v10, v8, 1
+; NO-ZVBB-NEXT:    vslidedown.vi v8, v9, 1
+; NO-ZVBB-NEXT:    vslideup.vi v8, v9, 1
+; NO-ZVBB-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; NO-ZVBB-NEXT:    vslideup.vi v8, v10, 2
+; NO-ZVBB-NEXT:    ret
+;
+; ZVBB-LABEL: reverse_v4f16_2:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
+; ZVBB-NEXT:    vror.vi v10, v8, 16
+; ZVBB-NEXT:    vror.vi v8, v9, 16
+; ZVBB-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; ZVBB-NEXT:    vslideup.vi v8, v10, 2
+; ZVBB-NEXT:    ret
+  %res = shufflevector <2 x half> %a, <2 x half> %b, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  ret <4 x half> %res
+}
+
+define <8 x half> @reverse_v8f16_2(<4 x half> %a, <4 x half> %b) {
+; CHECK-LABEL: reverse_v8f16_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, mu
+; CHECK-NEXT:    vid.v v11
+; CHECK-NEXT:    vrsub.vi v12, v11, 7
+; CHECK-NEXT:    vrgather.vv v10, v8, v12
+; CHECK-NEXT:    vmv.v.i v0, 15
+; CHECK-NEXT:    vrsub.vi v8, v11, 3
+; CHECK-NEXT:    vrgather.vv v10, v9, v8, v0.t
+; CHECK-NEXT:    vmv.v.v v8, v10
+; CHECK-NEXT:    ret
+  %res = shufflevector <4 x half> %a, <4 x half> %b, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <8 x half> %res
+}
+
+define <16 x half> @reverse_v16f16_2(<8 x half> %a, <8 x half> %b) {
+; CHECK-LABEL: reverse_v16f16_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmv1r.v v12, v9
+; CHECK-NEXT:    vsetivli zero, 16, e16, m2, ta, mu
+; CHECK-NEXT:    vid.v v14
+; CHECK-NEXT:    vrsub.vi v16, v14, 15
+; CHECK-NEXT:    vrgather.vv v10, v8, v16
+; CHECK-NEXT:    vrsub.vi v8, v14, 7
+; CHECK-NEXT:    li a0, 255
+; CHECK-NEXT:    vmv.s.x v0, a0
+; CHECK-NEXT:    vrgather.vv v10, v12, v8, v0.t
+; CHECK-NEXT:    vmv.v.v v8, v10
+; CHECK-NEXT:    ret
+  %res = shufflevector <8 x half> %a, <8 x half> %b,  <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <16 x half> %res
+}
+
+define <32 x half> @reverse_v32f16_2(<16 x half> %a) {
+; CHECK-LABEL: reverse_v32f16_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a0, 32
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
+; CHECK-NEXT:    vid.v v12
+; CHECK-NEXT:    li a0, 31
+; CHECK-NEXT:    vrsub.vx v16, v12, a0
+; CHECK-NEXT:    vrgather.vv v12, v8, v16
+; CHECK-NEXT:    vmv.v.v v8, v12
+; CHECK-NEXT:    ret
+  %res = shufflevector <16 x half> %a, <16 x half> poison,  <32 x i32> <i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <32 x half> %res
+}
+
+define <4 x float> @reverse_v4f32_2(<2 x float> %a, <2 x float> %b) {
+; NO-ZVBB-LABEL: reverse_v4f32_2:
+; NO-ZVBB:       # %bb.0:
+; NO-ZVBB-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; NO-ZVBB-NEXT:    vslidedown.vi v10, v8, 1
+; NO-ZVBB-NEXT:    vslideup.vi v10, v8, 1
+; NO-ZVBB-NEXT:    vslidedown.vi v8, v9, 1
+; NO-ZVBB-NEXT:    vslideup.vi v8, v9, 1
+; NO-ZVBB-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; NO-ZVBB-NEXT:    vslideup.vi v8, v10, 2
+; NO-ZVBB-NEXT:    ret
+;
+; ZVBB-LABEL: reverse_v4f32_2:
+; ZVBB:       # %bb.0:
+; ZVBB-NEXT:    vsetivli zero, 1, e64, m1, ta, ma
+; ZVBB-NEXT:    vror.vi v10, v8, 32
+; ZVBB-NEXT:    vror.vi v8, v9, 32
+; ZVBB-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; ZVBB-NEXT:    vslideup.vi v8, v10, 2
+; ZVBB-NEXT:    ret
+  %res = shufflevector <2 x float> %a, <2 x float> %b, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  ret <4 x float> %res
+}
+
+define <8 x float> @reverse_v8f32_2(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: reverse_v8f32_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmv1r.v v12, v9
+; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-NEXT:    vid.v v9
+; CHECK-NEXT:    vrsub.vi v13, v9, 7
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vrgatherei16.vv v10, v8, v13
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vrsub.vi v8, v9, 3
+; CHECK-NEXT:    vmv.v.i v0, 15
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
+; CHECK-NEXT:    vrgatherei16.vv v10, v12, v8, v0.t
+; CHECK-NEXT:    vmv.v.v v8, v10
+; CHECK-NEXT:    ret
+  %res = shufflevector <4 x float> %a, <4 x float> %b, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <8 x float> %res
+}
+
+define <16 x float> @reverse_v16f32_2(<8 x float> %a, <8 x float> %b) {
+; CHECK-LABEL: reverse_v16f32_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmv2r.v v16, v10
+; CHECK-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
+; CHECK-NEXT:    vid.v v10
+; CHECK-NEXT:    vrsub.vi v18, v10, 15
+; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vrgatherei16.vv v12, v8, v18
+; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vrsub.vi v8, v10, 7
+; CHECK-NEXT:    li a0, 255
+; CHECK-NEXT:    vmv.s.x v0, a0
+; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, mu
+; CHECK-NEXT:    vrgatherei16.vv v12, v16, v8, v0.t
+; CHECK-NEXT:    vmv.v.v v8, v12
+; CHECK-NEXT:    ret
+  %res = shufflevector <8 x float> %a, <8 x float> %b, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <16 x float> %res
+}
+
+define <4 x double> @reverse_v4f64_2(<2 x double> %a, < 2 x double> %b) {
+; CHECK-LABEL: reverse_v4f64_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; CHECK-NEXT:    vslidedown.vi v10, v8, 1
+; CHECK-NEXT:    vslideup.vi v10, v8, 1
+; CHECK-NEXT:    vslidedown.vi v8, v9, 1
+; CHECK-NEXT:    vslideup.vi v8, v9, 1
+; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; CHECK-NEXT:    vslideup.vi v8, v10, 2
+; CHECK-NEXT:    ret
+  %res = shufflevector <2 x double> %a, <2 x double> %b, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  ret <4 x double> %res
+}
+
+define <8 x double> @reverse_v8f64_2(<4 x double> %a, <4 x double> %b) {
+; CHECK-LABEL: reverse_v8f64_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmv2r.v v16, v10
+; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
+; CHECK-NEXT:    vid.v v10
+; CHECK-NEXT:    vrsub.vi v11, v10, 7
+; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
+; CHECK-NEXT:    vrgatherei16.vv v12, v8, v11
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vrsub.vi v8, v10, 3
+; CHECK-NEXT:    vmv.v.i v0, 15
+; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
+; CHECK-NEXT:    vrgatherei16.vv v12, v16, v8, v0.t
+; CHECK-NEXT:    vmv.v.v v8, v12
+; CHECK-NEXT:    ret
+  %res = shufflevector <4 x double> %a, <4 x double> %b, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <8 x double> %res
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/shuffle-reverse.ll b/llvm/test/CodeGen/RISCV/rvv/shuffle-reverse.ll
deleted file mode 100644
index 9e62bb623a03bd0..000000000000000
--- a/llvm/test/CodeGen/RISCV/rvv/shuffle-reverse.ll
+++ /dev/null
@@ -1,734 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=riscv32 -mattr=+v,+f,+zfh,+zvfh,+d -verify-machineinstrs | FileCheck %s --check-prefixes=CHECK,RV32
-; RUN: llc < %s -mtriple=riscv64 -mattr=+v,+f,+zfh,+zvfh,+d -verify-machineinstrs | FileCheck %s --check-prefixes=CHECK,RV64
-
-define <2 x i8> @v2i8(<2 x i8> %a) {
-; CHECK-LABEL: v2i8:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 2, e8, mf8, ta, ma
-; CHECK-NEXT:    vslidedown.vi v9, v8, 1
-; CHECK-NEXT:    vslideup.vi v9, v8, 1
-; CHECK-NEXT:    vmv1r.v v8, v9
-; CHECK-NEXT:    ret
-  %v2i8 = shufflevector <2 x i8> %a, <2 x i8> undef, <2 x i32> <i32 1, i32 0>
-  ret <2 x i8> %v2i8
-}
-
-define <4 x i8> @v2i8_2(<2 x i8> %a, <2 x i8> %b) {
-; CHECK-LABEL: v2i8_2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 2, e8, mf8, ta, ma
-; CHECK-NEXT:    vslidedown.vi v10, v8, 1
-; CHECK-NEXT:    vslideup.vi v10, v8, 1
-; CHECK-NEXT:    vslidedown.vi v8, v9, 1
-; CHECK-NEXT:    vslideup.vi v8, v9, 1
-; CHECK-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
-; CHECK-NEXT:    vslideup.vi v8, v10, 2
-; CHECK-NEXT:    ret
-  %v4i8 = shufflevector <2 x i8> %a, <2 x i8> %b, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-  ret <4 x i8> %v4i8
-}
-
-define <4 x i8> @v4i8(<4 x i8> %a) {
-; CHECK-LABEL: v4i8:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
-; CHECK-NEXT:    vid.v v9
-; CHECK-NEXT:    vrsub.vi v10, v9, 3
-; CHECK-NEXT:    vrgather.vv v9, v8, v10
-; CHECK-NEXT:    vmv1r.v v8, v9
-; CHECK-NEXT:    ret
-  %v4i8 = shufflevector <4 x i8> %a, <4 x i8> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-  ret <4 x i8> %v4i8
-}
-
-define <8 x i8> @v4i8_2(<4 x i8> %a, <4 x i8> %b) {
-; CHECK-LABEL: v4i8_2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, mu
-; CHECK-NEXT:    vid.v v11
-; CHECK-NEXT:    vrsub.vi v12, v11, 7
-; CHECK-NEXT:    vrgather.vv v10, v8, v12
-; CHECK-NEXT:    vmv.v.i v0, 15
-; CHECK-NEXT:    vrsub.vi v8, v11, 3
-; CHECK-NEXT:    vrgather.vv v10, v9, v8, v0.t
-; CHECK-NEXT:    vmv1r.v v8, v10
-; CHECK-NEXT:    ret
-  %v8i8 = shufflevector <4 x i8> %a, <4 x i8> %b, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-  ret <8 x i8> %v8i8
-}
-
-define <8 x i8> @v8i8(<8 x i8> %a) {
-; CHECK-LABEL: v8i8:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
-; CHECK-NEXT:    vid.v v9
-; CHECK-NEXT:    vrsub.vi v10, v9, 7
-; CHECK-NEXT:    vrgather.vv v9, v8, v10
-; CHECK-NEXT:    vmv1r.v v8, v9
-; CHECK-NEXT:    ret
-  %v8i8 = shufflevector <8 x i8> %a, <8 x i8> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-  ret <8 x i8> %v8i8
-}
-
-define <16 x i8> @v8i8_2(<8 x i8> %a, <8 x i8> %b) {
-; CHECK-LABEL: v8i8_2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
-; CHECK-NEXT:    vid.v v11
-; CHECK-NEXT:    vrsub.vi v12, v11, 15
-; CHECK-NEXT:    vrgather.vv v10, v8, v12
-; CHECK-NEXT:    li a0, 255
-; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vmv.s.x v0, a0
-; CHECK-NEXT:    vsetvli zero, zero, e8, m1, ta, mu
-; CHECK-NEXT:    vrsub.vi v8, v11, 7
-; CHECK-NEXT:    vrgather.vv v10, v9, v8, v0.t
-; CHECK-NEXT:    vmv.v.v v8, v10
-; CHECK-NEXT:    ret
-  %v16i8 = shufflevector <8 x i8> %a, <8 x i8> %b, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-  ret <16 x i8> %v16i8
-}
-
-define <16 x i8> @v16i8(<16 x i8> %a) {
-; CHECK-LABEL: v16i8:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
-; CHECK-NEXT:    vid.v v9
-; CHECK-NEXT:    vrsub.vi v10, v9, 15
-; CHECK-NEXT:    vrgather.vv v9, v8, v10
-; CHECK-NEXT:    vmv.v.v v8, v9
-; CHECK-NEXT:    ret
-  %v16i8 = shufflevector <16 x i8> %a, <16 x i8> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-  ret <16 x i8> %v16i8
-}
-
-define <32 x i8> @v16i8_2(<16 x i8> %a, <16 x i8> %b) {
-; CHECK-LABEL: v16i8_2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v12, v9
-; CHECK-NEXT:    li a0, 32
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
-; CHECK-NEXT:    vid.v v14
-; CHECK-NEXT:    li a0, 31
-; CHECK-NEXT:    vrsub.vx v16, v14, a0
-; CHECK-NEXT:    vrgather.vv v10, v8, v16
-; CHECK-NEXT:    vrsub.vi v8, v14, 15
-; CHECK-NEXT:    lui a0, 16
-; CHECK-NEXT:    addi a0, a0, -1
-; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
-; CHECK-NEXT:    vmv.s.x v0, a0
-; CHECK-NEXT:    vsetvli zero, zero, e8, m2, ta, mu
-; CHECK-NEXT:    vrgather.vv v10, v12, v8, v0.t
-; CHECK-NEXT:    vmv.v.v v8, v10
-; CHECK-NEXT:    ret
-  %v32i8 = shufflevector <16 x i8> %a, <16 x i8> %b,  <32 x i32> <i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-  ret <32 x i8> %v32i8
-}
-
-define <2 x i16> @v2i16(<2 x i16> %a) {
-; CHECK-LABEL: v2i16:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
-; CHECK-NEXT:    vslidedown.vi v9, v8, 1
-; CHECK-NEXT:    vslideup.vi v9, v8, 1
-; CHECK-NEXT:    vmv1r.v v8, v9
-; CHECK-NEXT:    ret
-  %v2i16 = shufflevector <2 x i16> %a, <2 x i16> undef, <2 x i32> <i32 1, i32 0>
-  ret <2 x i16> %v2i16
-}
-
-define <4 x i16> @v2i16_2(<2 x i16> %a, <2 x i16> %b) {
-; CHECK-LABEL: v2i16_2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
-; CHECK-NEXT:    vslidedown.vi v10, v8, 1
-; CHECK-NEXT:    vslideup.vi v10, v8, 1
-; CHECK-NEXT:    vslidedown.vi v8, v9, 1
-; CHECK-NEXT:    vslideup.vi v8, v9, 1
-; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
-; CHECK-NEXT:    vslideup.vi v8, v10, 2
-; CHECK-NEXT:    ret
-  %v4i16 = shufflevector <2 x i16> %a, <2 x i16> %b, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-  ret <4 x i16> %v4i16
-}
-
-define <4 x i16> @v4i16(<4 x i16> %a) {
-; CHECK-LABEL: v4i16:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
-; CHECK-NEXT:    vid.v v9
-; CHECK-NEXT:    vrsub.vi v10, v9, 3
-; CHECK-NEXT:    vrgather.vv v9, v8, v10
-; CHECK-NEXT:    vmv1r.v v8, v9
-; CHECK-NEXT:    ret
-  %v4i16 = shufflevector <4 x i16> %a, <4 x i16> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-  ret <4 x i16> %v4i16
-}
-
-define <8 x i16> @v4i16_2(<4 x i16> %a, <4 x i16> %b) {
-; CHECK-LABEL: v4i16_2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, mu
-; CHECK-NEXT:    vid.v v11
-; CHECK-NEXT:    vrsub.vi v12, v11, 7
-; CHECK-NEXT:    vrgather.vv v10, v8, v12
-; CHECK-NEXT:    vmv.v.i v0, 15
-; CHECK-NEXT:    vrsub.vi v8, v11, 3
-; CHECK-NEXT:    vrgather.vv v10, v9, v8, v0.t
-; CHECK-NEXT:    vmv.v.v v8, v10
-; CHECK-NEXT:    ret
-  %v8i16 = shufflevector <4 x i16> %a, <4 x i16> %b, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-  ret <8 x i16> %v8i16
-}
-
-define <8 x i16> @v8i16(<8 x i16> %a) {
-; CHECK-LABEL: v8i16:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; CHECK-NEXT:    vid.v v9
-; CHECK-NEXT:    vrsub.vi v10, v9, 7
-; CHECK-NEXT:    vrgather.vv v9, v8, v10
-; CHECK-NEXT:    vmv.v.v v8, v9
-; CHECK-NEXT:    ret
-  %v8i16 = shufflevector <8 x i16> %a, <8 x i16> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-  ret <8 x i16> %v8i16
-}
-
-define <16 x i16> @v8i16_2(<8 x i16> %a, <8 x i16> %b) {
-; CHECK-LABEL: v8i16_2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v12, v9
-; CHECK-NEXT:    vsetivli zero, 16, e16, m2, ta, mu
-; CHECK-NEXT:    vid.v v14
-; CHECK-NEXT:    vrsub.vi v16, v14, 15
-; CHECK-NEXT:    vrgather.vv v10, v8, v16
-; CHECK-NEXT:    vrsub.vi v8, v14, 7
-; CHECK-NEXT:    li a0, 255
-; CHECK-NEXT:    vmv.s.x v0, a0
-; CHECK-NEXT:    vrgather.vv v10, v12, v8, v0.t
-; CHECK-NEXT:    vmv.v.v v8, v10
-; CHECK-NEXT:    ret
-  %v16i16 = shufflevector <8 x i16> %a, <8 x i16> %b,  <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-  ret <16 x i16> %v16i16
-}
-
-define <16 x i16> @v16i16(<16 x i16> %a) {
-; CHECK-LABEL: v16i16:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
-; CHECK-NEXT:    vid.v v10
-; CHECK-NEXT:    vrsub.vi v12, v10, 15
-; CHECK-NEXT:    vrgather.vv v10, v8, v12
-; CHECK-NEXT:    vmv.v.v v8, v10
-; CHECK-NEXT:    ret
-  %v16i16 = shufflevector <16 x i16> %a, <16 x i16> undef,  <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-  ret <16 x i16> %v16i16
-}
-
-define <32 x i16> @v16i16_2(<16 x i16> %a, <16 x i16> %b) {
-; CHECK-LABEL: v16i16_2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv2r.v v16, v10
-; CHECK-NEXT:    li a0, 32
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    vid.v v20
-; CHECK-NEXT:    li a0, 31
-; CHECK-NEXT:    vrsub.vx v24, v20, a0
-; CHECK-NEXT:    vrgather.vv v12, v8, v24
-; CHECK-NEXT:    vrsub.vi v8, v20, 15
-; CHECK-NEXT:    lui a0, 16
-; CHECK-NEXT:    addi a0, a0, -1
-; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
-; CHECK-NEXT:    vmv.s.x v0, a0
-; CHECK-NEXT:    vsetvli zero, zero, e16, m4, ta, mu
-; CHECK-NEXT:    vrgather.vv v12, v16, v8, v0.t
-; CHECK-NEXT:    vmv.v.v v8, v12
-; CHECK-NEXT:    ret
-  %v32i16 = shufflevector <16 x i16> %a, <16 x i16> %b,  <32 x i32> <i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-  ret <32 x i16> %v32i16
-}
-
-define <2 x i32> @v2i32(<2 x i32> %a) {
-; CHECK-LABEL: v2i32:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
-; CHECK-NEXT:    vslidedown.vi v9, v8, 1
-; CHECK-NEXT:    vslideup.vi v9, v8, 1
-; CHECK-NEXT:    vmv1r.v v8, v9
-; CHECK-NEXT:    ret
-  %v2i32 = shufflevector <2 x i32> %a, <2 x i32> undef, <2 x i32> <i32 1, i32 0>
-  ret <2 x i32> %v2i32
-}
-
-define <4 x i32> @v2i32_2(<2 x i32> %a, < 2 x i32> %b) {
-; CHECK-LABEL: v2i32_2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
-; CHECK-NEXT:    vslidedown.vi v10, v8, 1
-; CHECK-NEXT:    vslideup.vi v10, v8, 1
-; CHECK-NEXT:    vslidedown.vi v8, v9, 1
-; CHECK-NEXT:    vslideup.vi v8, v9, 1
-; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; CHECK-NEXT:    vslideup.vi v8, v10, 2
-; CHECK-NEXT:    ret
-  %v4i32 = shufflevector <2 x i32> %a, <2 x i32> %b, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-  ret <4 x i32> %v4i32
-}
-
-define <4 x i32> @v4i32(<4 x i32> %a) {
-; CHECK-LABEL: v4i32:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; CHECK-NEXT:    vid.v v9
-; CHECK-NEXT:    vrsub.vi v10, v9, 3
-; CHECK-NEXT:    vrgather.vv v9, v8, v10
-; CHECK-NEXT:    vmv.v.v v8, v9
-; CHECK-NEXT:    ret
-  %v4i32 = shufflevector <4 x i32> %a, <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-  ret <4 x i32> %v4i32
-}
-
-define <8 x i32> @v4i32_2(<4 x i32> %a, <4 x i32> %b) {
-; CHECK-LABEL: v4i32_2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v12, v9
-; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; CHECK-NEXT:    vid.v v9
-; CHECK-NEXT:    vrsub.vi v13, v9, 7
-; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; CHECK-NEXT:    vrgatherei16.vv v10, v8, v13
-; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
-; CHECK-NEXT:    vrsub.vi v8, v9, 3
-; CHECK-NEXT:    vmv.v.i v0, 15
-; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
-; CHECK-NEXT:    vrgatherei16.vv v10, v12, v8, v0.t
-; CHECK-NEXT:    vmv.v.v v8, v10
-; CHECK-NEXT:    ret
-  %v8i32 = shufflevector <4 x i32> %a, <4 x i32> %b, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-  ret <8 x i32> %v8i32
-}
-
-define <8 x i32> @v8i32(<8 x i32> %a) {
-; CHECK-LABEL: v8i32:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; CHECK-NEXT:    vid.v v10
-; CHECK-NEXT:    vrsub.vi v12, v10, 7
-; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; CHECK-NEXT:    vrgatherei16.vv v10, v8, v12
-; CHECK-NEXT:    vmv.v.v v8, v10
-; CHECK-NEXT:    ret
-  %v8i32 = shufflevector <8 x i32> %a, <8 x i32> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-  ret <8 x i32> %v8i32
-}
-
-define <16 x i32> @v8i32_2(<8 x i32> %a, <8 x i32> %b) {
-; CHECK-LABEL: v8i32_2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv2r.v v16, v10
-; CHECK-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
-; CHECK-NEXT:    vid.v v10
-; CHECK-NEXT:    vrsub.vi v18, v10, 15
-; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vrgatherei16.vv v12, v8, v18
-; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vrsub.vi v8, v10, 7
-; CHECK-NEXT:    li a0, 255
-; CHECK-NEXT:    vmv.s.x v0, a0
-; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, mu
-; CHECK-NEXT:    vrgatherei16.vv v12, v16, v8, v0.t
-; CHECK-NEXT:    vmv.v.v v8, v12
-; CHECK-NEXT:    ret
-  %v16i32 = shufflevector <8 x i32> %a, <8 x i32> %b,  <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-  ret <16 x i32> %v16i32
-}
-
-define <16 x i32> @v16i32(<16 x i32> %a) {
-; CHECK-LABEL: v16i32:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
-; CHECK-NEXT:    vid.v v12
-; CHECK-NEXT:    vrsub.vi v16, v12, 15
-; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vrgatherei16.vv v12, v8, v16
-; CHECK-NEXT:    vmv.v.v v8, v12
-; CHECK-NEXT:    ret
-  %v16i32 = shufflevector <16 x i32> %a, <16 x i32> undef,  <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-  ret <16 x i32> %v16i32
-}
-
-define <32 x i32> @v16i32_2(<16 x i32> %a, <16 x i32> %b) {
-; CHECK-LABEL: v16i32_2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv4r.v v24, v12
-; CHECK-NEXT:    vmv4r.v v16, v8
-; CHECK-NEXT:    li a0, 32
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    vid.v v20
-; CHECK-NEXT:    li a0, 31
-; CHECK-NEXT:    vrsub.vx v28, v20, a0
-; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
-; CHECK-NEXT:    vrgatherei16.vv v8, v16, v28
-; CHECK-NEXT:    vsetvli zero, zero, e16, m4, ta, ma
-; CHECK-NEXT:    vrsub.vi v16, v20, 15
-; CHECK-NEXT:    lui a0, 16
-; CHECK-NEXT:    addi a0, a0, -1
-; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, mu
-; CHECK-NEXT:    vmv.s.x v0, a0
-; CHECK-NEXT:    vrgatherei16.vv v8, v24, v16, v0.t
-; CHECK-NEXT:    ret
-  %v32i32 = shufflevector <16 x i32> %a, <16 x i32> %b,  <32 x i32> <i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-  ret <32 x i32> %v32i32
-}
-
-define <2 x i64> @v2i64(<2 x i64> %a) {
-; CHECK-LABEL: v2i64:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
-; CHECK-NEXT:    vslidedown.vi v9, v8, 1
-; CHECK-NEXT:    vslideup.vi v9, v8, 1
-; CHECK-NEXT:    vmv.v.v v8, v9
-; CHECK-NEXT:    ret
-  %v2i64 = shufflevector <2 x i64> %a, <2 x i64> undef, <2 x i32> <i32 1, i32 0>
-  ret <2 x i64> %v2i64
-}
-
-define <4 x i64> @v2i64_2(<2 x i64> %a, < 2 x i64> %b) {
-; CHECK-LABEL: v2i64_2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
-; CHECK-NEXT:    vslidedown.vi v10, v8, 1
-; CHECK-NEXT:    vslideup.vi v10, v8, 1
-; CHECK-NEXT:    vslidedown.vi v8, v9, 1
-; CHECK-NEXT:    vslideup.vi v8, v9, 1
-; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
-; CHECK-NEXT:    vslideup.vi v8, v10, 2
-; CHECK-NEXT:    ret
-  %v4i64 = shufflevector <2 x i64> %a, <2 x i64> %b, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-  ret <4 x i64> %v4i64
-}
-
-define <4 x i64> @v4i64(<4 x i64> %a) {
-; CHECK-LABEL: v4i64:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
-; CHECK-NEXT:    vid.v v10
-; CHECK-NEXT:    vrsub.vi v12, v10, 3
-; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
-; CHECK-NEXT:    vrgatherei16.vv v10, v8, v12
-; CHECK-NEXT:    vmv.v.v v8, v10
-; CHECK-NEXT:    ret
-  %v4i64 = shufflevector <4 x i64> %a, <4 x i64> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-  ret <4 x i64> %v4i64
-}
-
-define <8 x i64> @v4i64_2(<4 x i64> %a, <4 x i64> %b) {
-; CHECK-LABEL: v4i64_2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv2r.v v16, v10
-; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; CHECK-NEXT:    vid.v v10
-; CHECK-NEXT:    vrsub.vi v11, v10, 7
-; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
-; CHECK-NEXT:    vrgatherei16.vv v12, v8, v11
-; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
-; CHECK-NEXT:    vrsub.vi v8, v10, 3
-; CHECK-NEXT:    vmv.v.i v0, 15
-; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
-; CHECK-NEXT:    vrgatherei16.vv v12, v16, v8, v0.t
-; CHECK-NEXT:    vmv.v.v v8, v12
-; CHECK-NEXT:    ret
-  %v8i64 = shufflevector <4 x i64> %a, <4 x i64> %b, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-  ret <8 x i64> %v8i64
-}
-
-define <2 x half> @v2f16(<2 x half> %a) {
-; CHECK-LABEL: v2f16:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
-; CHECK-NEXT:    vslidedown.vi v9, v8, 1
-; CHECK-NEXT:    vslideup.vi v9, v8, 1
-; CHECK-NEXT:    vmv1r.v v8, v9
-; CHECK-NEXT:    ret
-  %v2f16 = shufflevector <2 x half> %a, <2 x half> undef, <2 x i32> <i32 1, i32 0>
-  ret <2 x half> %v2f16
-}
-
-define <4 x half> @v2f16_2(<2 x half> %a, <2 x half> %b) {
-; CHECK-LABEL: v2f16_2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
-; CHECK-NEXT:    vslidedown.vi v10, v8, 1
-; CHECK-NEXT:    vslideup.vi v10, v8, 1
-; CHECK-NEXT:    vslidedown.vi v8, v9, 1
-; CHECK-NEXT:    vslideup.vi v8, v9, 1
-; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
-; CHECK-NEXT:    vslideup.vi v8, v10, 2
-; CHECK-NEXT:    ret
-  %v4f16 = shufflevector <2 x half> %a, <2 x half> %b, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-  ret <4 x half> %v4f16
-}
-
-define <4 x half> @v4f16(<4 x half> %a) {
-; CHECK-LABEL: v4f16:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
-; CHECK-NEXT:    vid.v v9
-; CHECK-NEXT:    vrsub.vi v10, v9, 3
-; CHECK-NEXT:    vrgather.vv v9, v8, v10
-; CHECK-NEXT:    vmv1r.v v8, v9
-; CHECK-NEXT:    ret
-  %v4f16 = shufflevector <4 x half> %a, <4 x half> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-  ret <4 x half> %v4f16
-}
-
-define <8 x half> @v4f16_2(<4 x half> %a, <4 x half> %b) {
-; CHECK-LABEL: v4f16_2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, mu
-; CHECK-NEXT:    vid.v v11
-; CHECK-NEXT:    vrsub.vi v12, v11, 7
-; CHECK-NEXT:    vrgather.vv v10, v8, v12
-; CHECK-NEXT:    vmv.v.i v0, 15
-; CHECK-NEXT:    vrsub.vi v8, v11, 3
-; CHECK-NEXT:    vrgather.vv v10, v9, v8, v0.t
-; CHECK-NEXT:    vmv.v.v v8, v10
-; CHECK-NEXT:    ret
-  %v8f16 = shufflevector <4 x half> %a, <4 x half> %b, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-  ret <8 x half> %v8f16
-}
-
-define <8 x half> @v8f16(<8 x half> %a) {
-; CHECK-LABEL: v8f16:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; CHECK-NEXT:    vid.v v9
-; CHECK-NEXT:    vrsub.vi v10, v9, 7
-; CHECK-NEXT:    vrgather.vv v9, v8, v10
-; CHECK-NEXT:    vmv.v.v v8, v9
-; CHECK-NEXT:    ret
-  %v8f16 = shufflevector <8 x half> %a, <8 x half> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-  ret <8 x half> %v8f16
-}
-
-define <16 x half> @v8f16_2(<8 x half> %a, <8 x half> %b) {
-; CHECK-LABEL: v8f16_2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v12, v9
-; CHECK-NEXT:    vsetivli zero, 16, e16, m2, ta, mu
-; CHECK-NEXT:    vid.v v14
-; CHECK-NEXT:    vrsub.vi v16, v14, 15
-; CHECK-NEXT:    vrgather.vv v10, v8, v16
-; CHECK-NEXT:    vrsub.vi v8, v14, 7
-; CHECK-NEXT:    li a0, 255
-; CHECK-NEXT:    vmv.s.x v0, a0
-; CHECK-NEXT:    vrgather.vv v10, v12, v8, v0.t
-; CHECK-NEXT:    vmv.v.v v8, v10
-; CHECK-NEXT:    ret
-  %v16f16 = shufflevector <8 x half> %a, <8 x half> %b,  <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-  ret <16 x half> %v16f16
-}
-
-define <16 x half> @v16f16(<16 x half> %a) {
-; CHECK-LABEL: v16f16:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
-; CHECK-NEXT:    vid.v v10
-; CHECK-NEXT:    vrsub.vi v12, v10, 15
-; CHECK-NEXT:    vrgather.vv v10, v8, v12
-; CHECK-NEXT:    vmv.v.v v8, v10
-; CHECK-NEXT:    ret
-  %v16f16 = shufflevector <16 x half> %a, <16 x half> undef,  <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-  ret <16 x half> %v16f16
-}
-
-define <32 x half> @v16f16_2(<16 x half> %a) {
-; CHECK-LABEL: v16f16_2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a0, 32
-; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
-; CHECK-NEXT:    vid.v v12
-; CHECK-NEXT:    li a0, 31
-; CHECK-NEXT:    vrsub.vx v16, v12, a0
-; CHECK-NEXT:    vrgather.vv v12, v8, v16
-; CHECK-NEXT:    vmv.v.v v8, v12
-; CHECK-NEXT:    ret
-  %v32f16 = shufflevector <16 x half> %a, <16 x half> undef,  <32 x i32> <i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-  ret <32 x half> %v32f16
-}
-
-define <2 x float> @v2f32(<2 x float> %a) {
-; CHECK-LABEL: v2f32:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
-; CHECK-NEXT:    vslidedown.vi v9, v8, 1
-; CHECK-NEXT:    vslideup.vi v9, v8, 1
-; CHECK-NEXT:    vmv1r.v v8, v9
-; CHECK-NEXT:    ret
-  %v2f32 = shufflevector <2 x float> %a, <2 x float> undef, <2 x i32> <i32 1, i32 0>
-  ret <2 x float> %v2f32
-}
-
-define <4 x float> @v2f32_2(<2 x float> %a, <2 x float> %b) {
-; CHECK-LABEL: v2f32_2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
-; CHECK-NEXT:    vslidedown.vi v10, v8, 1
-; CHECK-NEXT:    vslideup.vi v10, v8, 1
-; CHECK-NEXT:    vslidedown.vi v8, v9, 1
-; CHECK-NEXT:    vslideup.vi v8, v9, 1
-; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; CHECK-NEXT:    vslideup.vi v8, v10, 2
-; CHECK-NEXT:    ret
-  %v4f32 = shufflevector <2 x float> %a, <2 x float> %b, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-  ret <4 x float> %v4f32
-}
-
-define <4 x float> @v4f32(<4 x float> %a) {
-; CHECK-LABEL: v4f32:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; CHECK-NEXT:    vid.v v9
-; CHECK-NEXT:    vrsub.vi v10, v9, 3
-; CHECK-NEXT:    vrgather.vv v9, v8, v10
-; CHECK-NEXT:    vmv.v.v v8, v9
-; CHECK-NEXT:    ret
-  %v4f32 = shufflevector <4 x float> %a, <4 x float> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-  ret <4 x float> %v4f32
-}
-
-define <8 x float> @v4f32_2(<4 x float> %a, <4 x float> %b) {
-; CHECK-LABEL: v4f32_2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv1r.v v12, v9
-; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; CHECK-NEXT:    vid.v v9
-; CHECK-NEXT:    vrsub.vi v13, v9, 7
-; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; CHECK-NEXT:    vrgatherei16.vv v10, v8, v13
-; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
-; CHECK-NEXT:    vrsub.vi v8, v9, 3
-; CHECK-NEXT:    vmv.v.i v0, 15
-; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, mu
-; CHECK-NEXT:    vrgatherei16.vv v10, v12, v8, v0.t
-; CHECK-NEXT:    vmv.v.v v8, v10
-; CHECK-NEXT:    ret
-  %v8f32 = shufflevector <4 x float> %a, <4 x float> %b, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-  ret <8 x float> %v8f32
-}
-
-define <8 x float> @v8f32(<8 x float> %a) {
-; CHECK-LABEL: v8f32:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; CHECK-NEXT:    vid.v v10
-; CHECK-NEXT:    vrsub.vi v12, v10, 7
-; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
-; CHECK-NEXT:    vrgatherei16.vv v10, v8, v12
-; CHECK-NEXT:    vmv.v.v v8, v10
-; CHECK-NEXT:    ret
-  %v8f32 = shufflevector <8 x float> %a, <8 x float> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-  ret <8 x float> %v8f32
-}
-
-define <16 x float> @v8f32_2(<8 x float> %a, <8 x float> %b) {
-; CHECK-LABEL: v8f32_2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv2r.v v16, v10
-; CHECK-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
-; CHECK-NEXT:    vid.v v10
-; CHECK-NEXT:    vrsub.vi v18, v10, 15
-; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vrgatherei16.vv v12, v8, v18
-; CHECK-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vrsub.vi v8, v10, 7
-; CHECK-NEXT:    li a0, 255
-; CHECK-NEXT:    vmv.s.x v0, a0
-; CHECK-NEXT:    vsetvli zero, zero, e32, m4, ta, mu
-; CHECK-NEXT:    vrgatherei16.vv v12, v16, v8, v0.t
-; CHECK-NEXT:    vmv.v.v v8, v12
-; CHECK-NEXT:    ret
-  %v16f32 = shufflevector <8 x float> %a, <8 x float> %b, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-  ret <16 x float> %v16f32
-}
-
-define <2 x double> @v2f64(<2 x double> %a) {
-; CHECK-LABEL: v2f64:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
-; CHECK-NEXT:    vslidedown.vi v9, v8, 1
-; CHECK-NEXT:    vslideup.vi v9, v8, 1
-; CHECK-NEXT:    vmv.v.v v8, v9
-; CHECK-NEXT:    ret
-  %v2f64 = shufflevector <2 x double> %a, <2 x double> undef, <2 x i32> <i32 1, i32 0>
-  ret <2 x double> %v2f64
-}
-
-define <4 x double> @v2f64_2(<2 x double> %a, < 2 x double> %b) {
-; CHECK-LABEL: v2f64_2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
-; CHECK-NEXT:    vslidedown.vi v10, v8, 1
-; CHECK-NEXT:    vslideup.vi v10, v8, 1
-; CHECK-NEXT:    vslidedown.vi v8, v9, 1
-; CHECK-NEXT:    vslideup.vi v8, v9, 1
-; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
-; CHECK-NEXT:    vslideup.vi v8, v10, 2
-; CHECK-NEXT:    ret
-  %v4f64 = shufflevector <2 x double> %a, <2 x double> %b, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-  ret <4 x double> %v4f64
-}
-
-define <4 x double> @v4f64(<4 x double> %a) {
-; CHECK-LABEL: v4f64:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
-; CHECK-NEXT:    vid.v v10
-; CHECK-NEXT:    vrsub.vi v12, v10, 3
-; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
-; CHECK-NEXT:    vrgatherei16.vv v10, v8, v12
-; CHECK-NEXT:    vmv.v.v v8, v10
-; CHECK-NEXT:    ret
-  %v4f64 = shufflevector <4 x double> %a, <4 x double> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-  ret <4 x double> %v4f64
-}
-
-define <8 x double> @v4f64_2(<4 x double> %a, <4 x double> %b) {
-; CHECK-LABEL: v4f64_2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmv2r.v v16, v10
-; CHECK-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
-; CHECK-NEXT:    vid.v v10
-; CHECK-NEXT:    vrsub.vi v11, v10, 7
-; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, ma
-; CHECK-NEXT:    vrgatherei16.vv v12, v8, v11
-; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
-; CHECK-NEXT:    vrsub.vi v8, v10, 3
-; CHECK-NEXT:    vmv.v.i v0, 15
-; CHECK-NEXT:    vsetvli zero, zero, e64, m4, ta, mu
-; CHECK-NEXT:    vrgatherei16.vv v12, v16, v8, v0.t
-; CHECK-NEXT:    vmv.v.v v8, v12
-; CHECK-NEXT:    ret
-  %v8f64 = shufflevector <4 x double> %a, <4 x double> %b, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-  ret <8 x double> %v8f64
-}
-
-define <32 x i8> @v32i8(<32 x i8> %a) {
-; CHECK-LABEL: v32i8:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a0, 32
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
-; CHECK-NEXT:    vid.v v10
-; CHECK-NEXT:    li a0, 31
-; CHECK-NEXT:    vrsub.vx v12, v10, a0
-; CHECK-NEXT:    vrgather.vv v10, v8, v12
-; CHECK-NEXT:    vmv.v.v v8, v10
-; CHECK-NEXT:    ret
-  %v32i8 = shufflevector <32 x i8> %a, <32 x i8> undef, <32 x i32> <i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
-  ret <32 x i8> %v32i8
-}
-
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; RV32: {{.*}}
-; RV64: {{.*}}

From 4c77cc634d49782aceff77f7ec4e6183ec223020 Mon Sep 17 00:00:00 2001
From: Luke Boyer <lukeboyer4@gmail.com>
Date: Fri, 16 Aug 2024 00:51:01 -0700
Subject: [PATCH 135/441] [mlir][IR] Fix `checkFoldResult` error message
 (#104559)

checkFoldResult error message has expected and actual backwards.
---
 mlir/lib/IR/Operation.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/mlir/lib/IR/Operation.cpp b/mlir/lib/IR/Operation.cpp
index b51357198b1ca00..8f28d61c732cdcd 100644
--- a/mlir/lib/IR/Operation.cpp
+++ b/mlir/lib/IR/Operation.cpp
@@ -619,8 +619,8 @@ static void checkFoldResultTypes(Operation *op,
     if (auto value = dyn_cast<Value>(ofr)) {
       if (value.getType() != opResult.getType()) {
         op->emitOpError() << "folder produced a value of incorrect type: "
-                          << opResult.getType()
-                          << ", expected: " << value.getType();
+                          << value.getType()
+                          << ", expected: " << opResult.getType();
         assert(false && "incorrect fold result type");
       }
     }

From bbc89faf4edf36ceeeef87150e8a7e7a1fd5051a Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Fri, 16 Aug 2024 09:58:47 +0200
Subject: [PATCH 136/441] [Passes] clang-format initialization files (NFC)

These regularly get touched when adding or removing passes, and
always cause issues with clang-format.
---
 llvm/include/llvm/InitializePasses.h | 386 +++++++++++++--------------
 llvm/include/llvm/LinkAllPasses.h    | 207 +++++++-------
 2 files changed, 297 insertions(+), 296 deletions(-)

diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h
index 0cd0d2edc236687..a4ac314bb590e57 100644
--- a/llvm/include/llvm/InitializePasses.h
+++ b/llvm/include/llvm/InitializePasses.h
@@ -19,304 +19,304 @@ namespace llvm {
 class PassRegistry;
 
 /// Initialize all passes linked into the Core library.
-void initializeCore(PassRegistry&);
+void initializeCore(PassRegistry &);
 
 /// Initialize all passes linked into the TransformUtils library.
-void initializeTransformUtils(PassRegistry&);
+void initializeTransformUtils(PassRegistry &);
 
 /// Initialize all passes linked into the ScalarOpts library.
-void initializeScalarOpts(PassRegistry&);
+void initializeScalarOpts(PassRegistry &);
 
 /// Initialize all passes linked into the Vectorize library.
-void initializeVectorization(PassRegistry&);
+void initializeVectorization(PassRegistry &);
 
 /// Initialize all passes linked into the InstCombine library.
-void initializeInstCombine(PassRegistry&);
+void initializeInstCombine(PassRegistry &);
 
 /// Initialize all passes linked into the IPO library.
-void initializeIPO(PassRegistry&);
+void initializeIPO(PassRegistry &);
 
 /// Initialize all passes linked into the Analysis library.
-void initializeAnalysis(PassRegistry&);
+void initializeAnalysis(PassRegistry &);
 
 /// Initialize all passes linked into the CodeGen library.
-void initializeCodeGen(PassRegistry&);
+void initializeCodeGen(PassRegistry &);
 
 /// Initialize all passes linked into the GlobalISel library.
-void initializeGlobalISel(PassRegistry&);
+void initializeGlobalISel(PassRegistry &);
 
 /// Initialize all passes linked into the CodeGen library.
-void initializeTarget(PassRegistry&);
+void initializeTarget(PassRegistry &);
 
-void initializeAAResultsWrapperPassPass(PassRegistry&);
-void initializeAlwaysInlinerLegacyPassPass(PassRegistry&);
+void initializeAAResultsWrapperPassPass(PassRegistry &);
+void initializeAlwaysInlinerLegacyPassPass(PassRegistry &);
 void initializeAssignmentTrackingAnalysisPass(PassRegistry &);
-void initializeAssumptionCacheTrackerPass(PassRegistry&);
+void initializeAssumptionCacheTrackerPass(PassRegistry &);
 void initializeAtomicExpandLegacyPass(PassRegistry &);
 void initializeBasicBlockPathCloningPass(PassRegistry &);
 void initializeBasicBlockSectionsProfileReaderWrapperPassPass(PassRegistry &);
 void initializeBasicBlockSectionsPass(PassRegistry &);
-void initializeBarrierNoopPass(PassRegistry&);
-void initializeBasicAAWrapperPassPass(PassRegistry&);
-void initializeBlockFrequencyInfoWrapperPassPass(PassRegistry&);
-void initializeBranchFolderPassPass(PassRegistry&);
-void initializeBranchProbabilityInfoWrapperPassPass(PassRegistry&);
-void initializeBranchRelaxationPass(PassRegistry&);
-void initializeBreakCriticalEdgesPass(PassRegistry&);
-void initializeBreakFalseDepsPass(PassRegistry&);
+void initializeBarrierNoopPass(PassRegistry &);
+void initializeBasicAAWrapperPassPass(PassRegistry &);
+void initializeBlockFrequencyInfoWrapperPassPass(PassRegistry &);
+void initializeBranchFolderPassPass(PassRegistry &);
+void initializeBranchProbabilityInfoWrapperPassPass(PassRegistry &);
+void initializeBranchRelaxationPass(PassRegistry &);
+void initializeBreakCriticalEdgesPass(PassRegistry &);
+void initializeBreakFalseDepsPass(PassRegistry &);
 void initializeCanonicalizeFreezeInLoopsPass(PassRegistry &);
-void initializeCFGSimplifyPassPass(PassRegistry&);
-void initializeCFGuardPass(PassRegistry&);
-void initializeCFGuardLongjmpPass(PassRegistry&);
-void initializeCFIFixupPass(PassRegistry&);
-void initializeCFIInstrInserterPass(PassRegistry&);
+void initializeCFGSimplifyPassPass(PassRegistry &);
+void initializeCFGuardPass(PassRegistry &);
+void initializeCFGuardLongjmpPass(PassRegistry &);
+void initializeCFIFixupPass(PassRegistry &);
+void initializeCFIInstrInserterPass(PassRegistry &);
 void initializeCallBrPreparePass(PassRegistry &);
-void initializeCallGraphDOTPrinterPass(PassRegistry&);
-void initializeCallGraphViewerPass(PassRegistry&);
-void initializeCallGraphWrapperPassPass(PassRegistry&);
+void initializeCallGraphDOTPrinterPass(PassRegistry &);
+void initializeCallGraphViewerPass(PassRegistry &);
+void initializeCallGraphWrapperPassPass(PassRegistry &);
 void initializeCheckDebugMachineModulePass(PassRegistry &);
 void initializeCodeGenPrepareLegacyPassPass(PassRegistry &);
-void initializeComplexDeinterleavingLegacyPassPass(PassRegistry&);
-void initializeConstantHoistingLegacyPassPass(PassRegistry&);
+void initializeComplexDeinterleavingLegacyPassPass(PassRegistry &);
+void initializeConstantHoistingLegacyPassPass(PassRegistry &);
 void initializeCycleInfoWrapperPassPass(PassRegistry &);
-void initializeDAEPass(PassRegistry&);
-void initializeDAHPass(PassRegistry&);
-void initializeDCELegacyPassPass(PassRegistry&);
+void initializeDAEPass(PassRegistry &);
+void initializeDAHPass(PassRegistry &);
+void initializeDCELegacyPassPass(PassRegistry &);
 void initializeDXILMetadataAnalysisWrapperPassPass(PassRegistry &);
 void initializeDXILMetadataAnalysisWrapperPrinterPass(PassRegistry &);
 void initializeDXILResourceWrapperPassPass(PassRegistry &);
-void initializeDeadMachineInstructionElimPass(PassRegistry&);
+void initializeDeadMachineInstructionElimPass(PassRegistry &);
 void initializeDebugifyMachineModulePass(PassRegistry &);
-void initializeDependenceAnalysisWrapperPassPass(PassRegistry&);
-void initializeDetectDeadLanesPass(PassRegistry&);
+void initializeDependenceAnalysisWrapperPassPass(PassRegistry &);
+void initializeDetectDeadLanesPass(PassRegistry &);
 void initializeDomOnlyPrinterWrapperPassPass(PassRegistry &);
 void initializeDomOnlyViewerWrapperPassPass(PassRegistry &);
 void initializeDomPrinterWrapperPassPass(PassRegistry &);
 void initializeDomViewerWrapperPassPass(PassRegistry &);
-void initializeDominanceFrontierWrapperPassPass(PassRegistry&);
-void initializeDominatorTreeWrapperPassPass(PassRegistry&);
+void initializeDominanceFrontierWrapperPassPass(PassRegistry &);
+void initializeDominatorTreeWrapperPassPass(PassRegistry &);
 void initializeDwarfEHPrepareLegacyPassPass(PassRegistry &);
-void initializeEarlyCSELegacyPassPass(PassRegistry&);
-void initializeEarlyCSEMemSSALegacyPassPass(PassRegistry&);
-void initializeEarlyIfConverterPass(PassRegistry&);
+void initializeEarlyCSELegacyPassPass(PassRegistry &);
+void initializeEarlyCSEMemSSALegacyPassPass(PassRegistry &);
+void initializeEarlyIfConverterPass(PassRegistry &);
 void initializeEarlyIfPredicatorPass(PassRegistry &);
-void initializeEarlyMachineLICMPass(PassRegistry&);
-void initializeEarlyTailDuplicatePass(PassRegistry&);
-void initializeEdgeBundlesPass(PassRegistry&);
+void initializeEarlyMachineLICMPass(PassRegistry &);
+void initializeEarlyTailDuplicatePass(PassRegistry &);
+void initializeEdgeBundlesPass(PassRegistry &);
 void initializeEHContGuardCatchretPass(PassRegistry &);
-void initializeExpandLargeFpConvertLegacyPassPass(PassRegistry&);
-void initializeExpandLargeDivRemLegacyPassPass(PassRegistry&);
+void initializeExpandLargeFpConvertLegacyPassPass(PassRegistry &);
+void initializeExpandLargeDivRemLegacyPassPass(PassRegistry &);
 void initializeExpandMemCmpLegacyPassPass(PassRegistry &);
-void initializeExpandPostRAPass(PassRegistry&);
-void initializeExpandReductionsPass(PassRegistry&);
+void initializeExpandPostRAPass(PassRegistry &);
+void initializeExpandReductionsPass(PassRegistry &);
 void initializeExpandVariadicsPass(PassRegistry &);
 void initializeExpandVectorPredicationPass(PassRegistry &);
-void initializeExternalAAWrapperPassPass(PassRegistry&);
-void initializeFEntryInserterPass(PassRegistry&);
-void initializeFinalizeISelPass(PassRegistry&);
-void initializeFinalizeMachineBundlesPass(PassRegistry&);
+void initializeExternalAAWrapperPassPass(PassRegistry &);
+void initializeFEntryInserterPass(PassRegistry &);
+void initializeFinalizeISelPass(PassRegistry &);
+void initializeFinalizeMachineBundlesPass(PassRegistry &);
 void initializeFixIrreduciblePass(PassRegistry &);
-void initializeFixupStatepointCallerSavedPass(PassRegistry&);
+void initializeFixupStatepointCallerSavedPass(PassRegistry &);
 void initializeFlattenCFGLegacyPassPass(PassRegistry &);
-void initializeFuncletLayoutPass(PassRegistry&);
+void initializeFuncletLayoutPass(PassRegistry &);
 void initializeGCEmptyBasicBlocksPass(PassRegistry &);
-void initializeGCMachineCodeAnalysisPass(PassRegistry&);
-void initializeGCModuleInfoPass(PassRegistry&);
-void initializeGVNLegacyPassPass(PassRegistry&);
-void initializeGlobalMergePass(PassRegistry&);
+void initializeGCMachineCodeAnalysisPass(PassRegistry &);
+void initializeGCModuleInfoPass(PassRegistry &);
+void initializeGVNLegacyPassPass(PassRegistry &);
+void initializeGlobalMergePass(PassRegistry &);
 void initializeGlobalsAAWrapperPassPass(PassRegistry &);
-void initializeHardwareLoopsLegacyPass(PassRegistry&);
+void initializeHardwareLoopsLegacyPass(PassRegistry &);
 void initializeMIRProfileLoaderPassPass(PassRegistry &);
-void initializeIRSimilarityIdentifierWrapperPassPass(PassRegistry&);
-void initializeIRTranslatorPass(PassRegistry&);
-void initializeIVUsersWrapperPassPass(PassRegistry&);
-void initializeIfConverterPass(PassRegistry&);
-void initializeImmutableModuleSummaryIndexWrapperPassPass(PassRegistry&);
-void initializeImplicitNullChecksPass(PassRegistry&);
+void initializeIRSimilarityIdentifierWrapperPassPass(PassRegistry &);
+void initializeIRTranslatorPass(PassRegistry &);
+void initializeIVUsersWrapperPassPass(PassRegistry &);
+void initializeIfConverterPass(PassRegistry &);
+void initializeImmutableModuleSummaryIndexWrapperPassPass(PassRegistry &);
+void initializeImplicitNullChecksPass(PassRegistry &);
 void initializeIndirectBrExpandLegacyPassPass(PassRegistry &);
-void initializeInferAddressSpacesPass(PassRegistry&);
+void initializeInferAddressSpacesPass(PassRegistry &);
 void initializeInstSimplifyLegacyPassPass(PassRegistry &);
-void initializeInstructionCombiningPassPass(PassRegistry&);
-void initializeInstructionSelectPass(PassRegistry&);
-void initializeInterleavedAccessPass(PassRegistry&);
+void initializeInstructionCombiningPassPass(PassRegistry &);
+void initializeInstructionSelectPass(PassRegistry &);
+void initializeInterleavedAccessPass(PassRegistry &);
 void initializeInterleavedLoadCombinePass(PassRegistry &);
-void initializeJMCInstrumenterPass(PassRegistry&);
+void initializeJMCInstrumenterPass(PassRegistry &);
 void initializeKCFIPass(PassRegistry &);
-void initializeLCSSAVerificationPassPass(PassRegistry&);
-void initializeLCSSAWrapperPassPass(PassRegistry&);
-void initializeLazyBlockFrequencyInfoPassPass(PassRegistry&);
-void initializeLazyBranchProbabilityInfoPassPass(PassRegistry&);
-void initializeLazyMachineBlockFrequencyInfoPassPass(PassRegistry&);
-void initializeLazyValueInfoWrapperPassPass(PassRegistry&);
-void initializeLegacyLICMPassPass(PassRegistry&);
-void initializeLegalizerPass(PassRegistry&);
+void initializeLCSSAVerificationPassPass(PassRegistry &);
+void initializeLCSSAWrapperPassPass(PassRegistry &);
+void initializeLazyBlockFrequencyInfoPassPass(PassRegistry &);
+void initializeLazyBranchProbabilityInfoPassPass(PassRegistry &);
+void initializeLazyMachineBlockFrequencyInfoPassPass(PassRegistry &);
+void initializeLazyValueInfoWrapperPassPass(PassRegistry &);
+void initializeLegacyLICMPassPass(PassRegistry &);
+void initializeLegalizerPass(PassRegistry &);
 void initializeGISelCSEAnalysisWrapperPassPass(PassRegistry &);
 void initializeGISelKnownBitsAnalysisPass(PassRegistry &);
-void initializeLiveDebugValuesPass(PassRegistry&);
-void initializeLiveDebugVariablesPass(PassRegistry&);
+void initializeLiveDebugValuesPass(PassRegistry &);
+void initializeLiveDebugVariablesPass(PassRegistry &);
 void initializeLiveIntervalsWrapperPassPass(PassRegistry &);
-void initializeLiveRangeShrinkPass(PassRegistry&);
-void initializeLiveRegMatrixPass(PassRegistry&);
-void initializeLiveStacksPass(PassRegistry&);
+void initializeLiveRangeShrinkPass(PassRegistry &);
+void initializeLiveRegMatrixPass(PassRegistry &);
+void initializeLiveStacksPass(PassRegistry &);
 void initializeLiveVariablesWrapperPassPass(PassRegistry &);
 void initializeLoadStoreOptPass(PassRegistry &);
-void initializeLoadStoreVectorizerLegacyPassPass(PassRegistry&);
-void initializeLocalStackSlotPassPass(PassRegistry&);
-void initializeLocalizerPass(PassRegistry&);
-void initializeLoopDataPrefetchLegacyPassPass(PassRegistry&);
+void initializeLoadStoreVectorizerLegacyPassPass(PassRegistry &);
+void initializeLocalStackSlotPassPass(PassRegistry &);
+void initializeLocalizerPass(PassRegistry &);
+void initializeLoopDataPrefetchLegacyPassPass(PassRegistry &);
 void initializeLoopExtractorLegacyPassPass(PassRegistry &);
-void initializeLoopInfoWrapperPassPass(PassRegistry&);
-void initializeLoopPassPass(PassRegistry&);
-void initializeLoopSimplifyPass(PassRegistry&);
-void initializeLoopStrengthReducePass(PassRegistry&);
-void initializeLoopUnrollPass(PassRegistry&);
-void initializeLowerAtomicLegacyPassPass(PassRegistry&);
-void initializeLowerConstantIntrinsicsPass(PassRegistry&);
-void initializeLowerEmuTLSPass(PassRegistry&);
+void initializeLoopInfoWrapperPassPass(PassRegistry &);
+void initializeLoopPassPass(PassRegistry &);
+void initializeLoopSimplifyPass(PassRegistry &);
+void initializeLoopStrengthReducePass(PassRegistry &);
+void initializeLoopUnrollPass(PassRegistry &);
+void initializeLowerAtomicLegacyPassPass(PassRegistry &);
+void initializeLowerConstantIntrinsicsPass(PassRegistry &);
+void initializeLowerEmuTLSPass(PassRegistry &);
 void initializeLowerGlobalDtorsLegacyPassPass(PassRegistry &);
-void initializeLowerIntrinsicsPass(PassRegistry&);
-void initializeLowerInvokeLegacyPassPass(PassRegistry&);
+void initializeLowerIntrinsicsPass(PassRegistry &);
+void initializeLowerInvokeLegacyPassPass(PassRegistry &);
 void initializeLowerSwitchLegacyPassPass(PassRegistry &);
 void initializeKCFIPass(PassRegistry &);
 void initializeMIRAddFSDiscriminatorsPass(PassRegistry &);
 void initializeMIRCanonicalizerPass(PassRegistry &);
 void initializeMIRNamerPass(PassRegistry &);
-void initializeMIRPrintingPassPass(PassRegistry&);
+void initializeMIRPrintingPassPass(PassRegistry &);
 void initializeMachineBlockFrequencyInfoWrapperPassPass(PassRegistry &);
-void initializeMachineBlockPlacementPass(PassRegistry&);
-void initializeMachineBlockPlacementStatsPass(PassRegistry&);
+void initializeMachineBlockPlacementPass(PassRegistry &);
+void initializeMachineBlockPlacementStatsPass(PassRegistry &);
 void initializeMachineBranchProbabilityInfoWrapperPassPass(PassRegistry &);
 void initializeMachineCFGPrinterPass(PassRegistry &);
-void initializeMachineCSEPass(PassRegistry&);
-void initializeMachineCombinerPass(PassRegistry&);
-void initializeMachineCopyPropagationPass(PassRegistry&);
+void initializeMachineCSEPass(PassRegistry &);
+void initializeMachineCombinerPass(PassRegistry &);
+void initializeMachineCopyPropagationPass(PassRegistry &);
 void initializeMachineCycleInfoPrinterPassPass(PassRegistry &);
 void initializeMachineCycleInfoWrapperPassPass(PassRegistry &);
-void initializeMachineDominanceFrontierPass(PassRegistry&);
+void initializeMachineDominanceFrontierPass(PassRegistry &);
 void initializeMachineDominatorTreeWrapperPassPass(PassRegistry &);
-void initializeMachineFunctionPrinterPassPass(PassRegistry&);
+void initializeMachineFunctionPrinterPassPass(PassRegistry &);
 void initializeMachineFunctionSplitterPass(PassRegistry &);
-void initializeMachineLateInstrsCleanupPass(PassRegistry&);
-void initializeMachineLICMPass(PassRegistry&);
+void initializeMachineLateInstrsCleanupPass(PassRegistry &);
+void initializeMachineLICMPass(PassRegistry &);
 void initializeMachineLoopInfoWrapperPassPass(PassRegistry &);
 void initializeMachineModuleInfoWrapperPassPass(PassRegistry &);
-void initializeMachineOptimizationRemarkEmitterPassPass(PassRegistry&);
-void initializeMachineOutlinerPass(PassRegistry&);
-void initializeMachinePipelinerPass(PassRegistry&);
+void initializeMachineOptimizationRemarkEmitterPassPass(PassRegistry &);
+void initializeMachineOutlinerPass(PassRegistry &);
+void initializeMachinePipelinerPass(PassRegistry &);
 void initializeMachinePostDominatorTreeWrapperPassPass(PassRegistry &);
-void initializeMachineRegionInfoPassPass(PassRegistry&);
+void initializeMachineRegionInfoPassPass(PassRegistry &);
 void initializeMachineSanitizerBinaryMetadataPass(PassRegistry &);
-void initializeMachineSchedulerPass(PassRegistry&);
-void initializeMachineSinkingPass(PassRegistry&);
-void initializeMachineTraceMetricsPass(PassRegistry&);
+void initializeMachineSchedulerPass(PassRegistry &);
+void initializeMachineSinkingPass(PassRegistry &);
+void initializeMachineTraceMetricsPass(PassRegistry &);
 void initializeMachineUniformityInfoPrinterPassPass(PassRegistry &);
 void initializeMachineUniformityAnalysisPassPass(PassRegistry &);
 void initializeMachineVerifierLegacyPassPass(PassRegistry &);
-void initializeMemoryDependenceWrapperPassPass(PassRegistry&);
-void initializeMemorySSAWrapperPassPass(PassRegistry&);
+void initializeMemoryDependenceWrapperPassPass(PassRegistry &);
+void initializeMemorySSAWrapperPassPass(PassRegistry &);
 void initializeMergeICmpsLegacyPassPass(PassRegistry &);
-void initializeModuleSummaryIndexWrapperPassPass(PassRegistry&);
-void initializeModuloScheduleTestPass(PassRegistry&);
-void initializeNaryReassociateLegacyPassPass(PassRegistry&);
+void initializeModuleSummaryIndexWrapperPassPass(PassRegistry &);
+void initializeModuloScheduleTestPass(PassRegistry &);
+void initializeNaryReassociateLegacyPassPass(PassRegistry &);
 void initializeObjCARCContractLegacyPassPass(PassRegistry &);
-void initializeOptimizationRemarkEmitterWrapperPassPass(PassRegistry&);
-void initializeOptimizePHIsPass(PassRegistry&);
-void initializePEIPass(PassRegistry&);
-void initializePHIEliminationPass(PassRegistry&);
-void initializePartiallyInlineLibCallsLegacyPassPass(PassRegistry&);
-void initializePatchableFunctionPass(PassRegistry&);
-void initializePeepholeOptimizerPass(PassRegistry&);
-void initializePhiValuesWrapperPassPass(PassRegistry&);
-void initializePhysicalRegisterUsageInfoPass(PassRegistry&);
+void initializeOptimizationRemarkEmitterWrapperPassPass(PassRegistry &);
+void initializeOptimizePHIsPass(PassRegistry &);
+void initializePEIPass(PassRegistry &);
+void initializePHIEliminationPass(PassRegistry &);
+void initializePartiallyInlineLibCallsLegacyPassPass(PassRegistry &);
+void initializePatchableFunctionPass(PassRegistry &);
+void initializePeepholeOptimizerPass(PassRegistry &);
+void initializePhiValuesWrapperPassPass(PassRegistry &);
+void initializePhysicalRegisterUsageInfoPass(PassRegistry &);
 void initializePlaceBackedgeSafepointsLegacyPassPass(PassRegistry &);
 void initializePostDomOnlyPrinterWrapperPassPass(PassRegistry &);
 void initializePostDomOnlyViewerWrapperPassPass(PassRegistry &);
 void initializePostDomPrinterWrapperPassPass(PassRegistry &);
 void initializePostDomViewerWrapperPassPass(PassRegistry &);
-void initializePostDominatorTreeWrapperPassPass(PassRegistry&);
-void initializePostInlineEntryExitInstrumenterPass(PassRegistry&);
-void initializePostMachineSchedulerPass(PassRegistry&);
-void initializePostRAHazardRecognizerPass(PassRegistry&);
-void initializePostRAMachineSinkingPass(PassRegistry&);
-void initializePostRASchedulerPass(PassRegistry&);
-void initializePreISelIntrinsicLoweringLegacyPassPass(PassRegistry&);
-void initializePrintFunctionPassWrapperPass(PassRegistry&);
-void initializePrintModulePassWrapperPass(PassRegistry&);
-void initializeProcessImplicitDefsPass(PassRegistry&);
-void initializeProfileSummaryInfoWrapperPassPass(PassRegistry&);
-void initializePromoteLegacyPassPass(PassRegistry&);
-void initializeRABasicPass(PassRegistry&);
+void initializePostDominatorTreeWrapperPassPass(PassRegistry &);
+void initializePostInlineEntryExitInstrumenterPass(PassRegistry &);
+void initializePostMachineSchedulerPass(PassRegistry &);
+void initializePostRAHazardRecognizerPass(PassRegistry &);
+void initializePostRAMachineSinkingPass(PassRegistry &);
+void initializePostRASchedulerPass(PassRegistry &);
+void initializePreISelIntrinsicLoweringLegacyPassPass(PassRegistry &);
+void initializePrintFunctionPassWrapperPass(PassRegistry &);
+void initializePrintModulePassWrapperPass(PassRegistry &);
+void initializeProcessImplicitDefsPass(PassRegistry &);
+void initializeProfileSummaryInfoWrapperPassPass(PassRegistry &);
+void initializePromoteLegacyPassPass(PassRegistry &);
+void initializeRABasicPass(PassRegistry &);
 void initializePseudoProbeInserterPass(PassRegistry &);
-void initializeRAGreedyPass(PassRegistry&);
-void initializeReachingDefAnalysisPass(PassRegistry&);
-void initializeReassociateLegacyPassPass(PassRegistry&);
+void initializeRAGreedyPass(PassRegistry &);
+void initializeReachingDefAnalysisPass(PassRegistry &);
+void initializeReassociateLegacyPassPass(PassRegistry &);
 void initializeRegAllocEvictionAdvisorAnalysisPass(PassRegistry &);
-void initializeRegAllocFastPass(PassRegistry&);
+void initializeRegAllocFastPass(PassRegistry &);
 void initializeRegAllocPriorityAdvisorAnalysisPass(PassRegistry &);
 void initializeRegAllocScoringPass(PassRegistry &);
-void initializeRegBankSelectPass(PassRegistry&);
-void initializeRegUsageInfoCollectorPass(PassRegistry&);
-void initializeRegUsageInfoPropagationPass(PassRegistry&);
-void initializeRegionInfoPassPass(PassRegistry&);
-void initializeRegionOnlyPrinterPass(PassRegistry&);
-void initializeRegionOnlyViewerPass(PassRegistry&);
-void initializeRegionPrinterPass(PassRegistry&);
-void initializeRegionViewerPass(PassRegistry&);
-void initializeRegisterCoalescerPass(PassRegistry&);
-void initializeRemoveRedundantDebugValuesPass(PassRegistry&);
-void initializeRenameIndependentSubregsPass(PassRegistry&);
+void initializeRegBankSelectPass(PassRegistry &);
+void initializeRegUsageInfoCollectorPass(PassRegistry &);
+void initializeRegUsageInfoPropagationPass(PassRegistry &);
+void initializeRegionInfoPassPass(PassRegistry &);
+void initializeRegionOnlyPrinterPass(PassRegistry &);
+void initializeRegionOnlyViewerPass(PassRegistry &);
+void initializeRegionPrinterPass(PassRegistry &);
+void initializeRegionViewerPass(PassRegistry &);
+void initializeRegisterCoalescerPass(PassRegistry &);
+void initializeRemoveRedundantDebugValuesPass(PassRegistry &);
+void initializeRenameIndependentSubregsPass(PassRegistry &);
 void initializeReplaceWithVeclibLegacyPass(PassRegistry &);
-void initializeResetMachineFunctionPass(PassRegistry&);
-void initializeSCEVAAWrapperPassPass(PassRegistry&);
-void initializeSROALegacyPassPass(PassRegistry&);
-void initializeSafeStackLegacyPassPass(PassRegistry&);
-void initializeSafepointIRVerifierPass(PassRegistry&);
+void initializeResetMachineFunctionPass(PassRegistry &);
+void initializeSCEVAAWrapperPassPass(PassRegistry &);
+void initializeSROALegacyPassPass(PassRegistry &);
+void initializeSafeStackLegacyPassPass(PassRegistry &);
+void initializeSafepointIRVerifierPass(PassRegistry &);
 void initializeSelectOptimizePass(PassRegistry &);
-void initializeScalarEvolutionWrapperPassPass(PassRegistry&);
+void initializeScalarEvolutionWrapperPassPass(PassRegistry &);
 void initializeScalarizeMaskedMemIntrinLegacyPassPass(PassRegistry &);
-void initializeScavengerTestPass(PassRegistry&);
-void initializeScopedNoAliasAAWrapperPassPass(PassRegistry&);
+void initializeScavengerTestPass(PassRegistry &);
+void initializeScopedNoAliasAAWrapperPassPass(PassRegistry &);
 void initializeSeparateConstOffsetFromGEPLegacyPassPass(PassRegistry &);
-void initializeShadowStackGCLoweringPass(PassRegistry&);
-void initializeShrinkWrapPass(PassRegistry&);
-void initializeSingleLoopExtractorPass(PassRegistry&);
-void initializeSinkingLegacyPassPass(PassRegistry&);
-void initializeSjLjEHPreparePass(PassRegistry&);
+void initializeShadowStackGCLoweringPass(PassRegistry &);
+void initializeShrinkWrapPass(PassRegistry &);
+void initializeSingleLoopExtractorPass(PassRegistry &);
+void initializeSinkingLegacyPassPass(PassRegistry &);
+void initializeSjLjEHPreparePass(PassRegistry &);
 void initializeSlotIndexesWrapperPassPass(PassRegistry &);
-void initializeSpeculativeExecutionLegacyPassPass(PassRegistry&);
-void initializeSpillPlacementPass(PassRegistry&);
-void initializeStackColoringPass(PassRegistry&);
+void initializeSpeculativeExecutionLegacyPassPass(PassRegistry &);
+void initializeSpillPlacementPass(PassRegistry &);
+void initializeStackColoringPass(PassRegistry &);
 void initializeStackFrameLayoutAnalysisPassPass(PassRegistry &);
-void initializeStackMapLivenessPass(PassRegistry&);
-void initializeStackProtectorPass(PassRegistry&);
+void initializeStackMapLivenessPass(PassRegistry &);
+void initializeStackProtectorPass(PassRegistry &);
 void initializeStackSafetyGlobalInfoWrapperPassPass(PassRegistry &);
 void initializeStackSafetyInfoWrapperPassPass(PassRegistry &);
-void initializeStackSlotColoringPass(PassRegistry&);
+void initializeStackSlotColoringPass(PassRegistry &);
 void initializeStraightLineStrengthReduceLegacyPassPass(PassRegistry &);
 void initializeStripDebugMachineModulePass(PassRegistry &);
 void initializeStructurizeCFGLegacyPassPass(PassRegistry &);
-void initializeTailCallElimPass(PassRegistry&);
-void initializeTailDuplicatePass(PassRegistry&);
-void initializeTargetLibraryInfoWrapperPassPass(PassRegistry&);
-void initializeTargetPassConfigPass(PassRegistry&);
-void initializeTargetTransformInfoWrapperPassPass(PassRegistry&);
+void initializeTailCallElimPass(PassRegistry &);
+void initializeTailDuplicatePass(PassRegistry &);
+void initializeTargetLibraryInfoWrapperPassPass(PassRegistry &);
+void initializeTargetPassConfigPass(PassRegistry &);
+void initializeTargetTransformInfoWrapperPassPass(PassRegistry &);
 void initializeTLSVariableHoistLegacyPassPass(PassRegistry &);
 void initializeTwoAddressInstructionLegacyPassPass(PassRegistry &);
-void initializeTypeBasedAAWrapperPassPass(PassRegistry&);
-void initializeTypePromotionLegacyPass(PassRegistry&);
+void initializeTypeBasedAAWrapperPassPass(PassRegistry &);
+void initializeTypePromotionLegacyPass(PassRegistry &);
 void initializeInitUndefPass(PassRegistry &);
 void initializeUniformityInfoWrapperPassPass(PassRegistry &);
 void initializeUnifyLoopExitsLegacyPassPass(PassRegistry &);
-void initializeUnpackMachineBundlesPass(PassRegistry&);
-void initializeUnreachableBlockElimLegacyPassPass(PassRegistry&);
-void initializeUnreachableMachineBlockElimPass(PassRegistry&);
-void initializeVerifierLegacyPassPass(PassRegistry&);
-void initializeVirtRegMapPass(PassRegistry&);
-void initializeVirtRegRewriterPass(PassRegistry&);
-void initializeWasmEHPreparePass(PassRegistry&);
-void initializeWinEHPreparePass(PassRegistry&);
-void initializeWriteBitcodePassPass(PassRegistry&);
-void initializeXRayInstrumentationPass(PassRegistry&);
+void initializeUnpackMachineBundlesPass(PassRegistry &);
+void initializeUnreachableBlockElimLegacyPassPass(PassRegistry &);
+void initializeUnreachableMachineBlockElimPass(PassRegistry &);
+void initializeVerifierLegacyPassPass(PassRegistry &);
+void initializeVirtRegMapPass(PassRegistry &);
+void initializeVirtRegRewriterPass(PassRegistry &);
+void initializeWasmEHPreparePass(PassRegistry &);
+void initializeWinEHPreparePass(PassRegistry &);
+void initializeWriteBitcodePassPass(PassRegistry &);
+void initializeXRayInstrumentationPass(PassRegistry &);
 
 } // end namespace llvm
 
diff --git a/llvm/include/llvm/LinkAllPasses.h b/llvm/include/llvm/LinkAllPasses.h
index 967238e8d03b6e0..e6a70dfd1ea6f83 100644
--- a/llvm/include/llvm/LinkAllPasses.h
+++ b/llvm/include/llvm/LinkAllPasses.h
@@ -48,110 +48,111 @@
 #include <cstdlib>
 
 namespace {
-  struct ForcePassLinking {
-    ForcePassLinking() {
-      // We must reference the passes in such a way that compilers will not
-      // delete it all as dead code, even with whole program optimization,
-      // yet is effectively a NO-OP. As the compiler isn't smart enough
-      // to know that getenv() never returns -1, this will do the job.
-      // This is so that globals in the translation units where these functions
-      // are defined are forced to be initialized, populating various
-      // registries.
-      if (std::getenv("bar") != (char*) -1)
-        return;
+struct ForcePassLinking {
+  ForcePassLinking() {
+    // We must reference the passes in such a way that compilers will not
+    // delete it all as dead code, even with whole program optimization,
+    // yet is effectively a NO-OP. As the compiler isn't smart enough
+    // to know that getenv() never returns -1, this will do the job.
+    // This is so that globals in the translation units where these functions
+    // are defined are forced to be initialized, populating various
+    // registries.
+    if (std::getenv("bar") != (char *)-1)
+      return;
 
-      (void)llvm::createAtomicExpandLegacyPass();
-      (void) llvm::createBasicAAWrapperPass();
-      (void) llvm::createSCEVAAWrapperPass();
-      (void) llvm::createTypeBasedAAWrapperPass();
-      (void) llvm::createScopedNoAliasAAWrapperPass();
-      (void) llvm::createBreakCriticalEdgesPass();
-      (void) llvm::createCallGraphDOTPrinterPass();
-      (void) llvm::createCallGraphViewerPass();
-      (void) llvm::createCFGSimplificationPass();
-      (void) llvm::createStructurizeCFGPass();
-      (void) llvm::createDXILResourceWrapperPassPass();
-      (void) llvm::createDeadArgEliminationPass();
-      (void) llvm::createDeadCodeEliminationPass();
-      (void) llvm::createDependenceAnalysisWrapperPass();
-      (void) llvm::createDomOnlyPrinterWrapperPassPass();
-      (void) llvm::createDomPrinterWrapperPassPass();
-      (void) llvm::createDomOnlyViewerWrapperPassPass();
-      (void) llvm::createDomViewerWrapperPassPass();
-      (void) llvm::createAlwaysInlinerLegacyPass();
-      (void) llvm::createGlobalsAAWrapperPass();
-      (void) llvm::createInstSimplifyLegacyPass();
-      (void) llvm::createInstructionCombiningPass();
-      (void) llvm::createJMCInstrumenterPass();
-      (void) llvm::createKCFIPass();
-      (void) llvm::createLCSSAPass();
-      (void) llvm::createLICMPass();
-      (void) llvm::createLazyValueInfoPass();
-      (void) llvm::createLoopExtractorPass();
-      (void) llvm::createLoopSimplifyPass();
-      (void) llvm::createLoopStrengthReducePass();
-      (void)llvm::createLoopUnrollPass();
-      (void) llvm::createLowerGlobalDtorsLegacyPass();
-      (void) llvm::createLowerInvokePass();
-      (void) llvm::createLowerSwitchPass();
-      (void) llvm::createNaryReassociatePass();
-      (void) llvm::createObjCARCContractPass();
-      (void) llvm::createPromoteMemoryToRegisterPass();
-      (void)llvm::createPostDomOnlyPrinterWrapperPassPass();
-      (void)llvm::createPostDomPrinterWrapperPassPass();
-      (void)llvm::createPostDomOnlyViewerWrapperPassPass();
-      (void)llvm::createPostDomViewerWrapperPassPass();
-      (void) llvm::createReassociatePass();
-      (void) llvm::createRegionInfoPass();
-      (void) llvm::createRegionOnlyPrinterPass();
-      (void) llvm::createRegionOnlyViewerPass();
-      (void) llvm::createRegionPrinterPass();
-      (void) llvm::createRegionViewerPass();
-      (void) llvm::createSafeStackPass();
-      (void) llvm::createSROAPass();
-      (void) llvm::createSingleLoopExtractorPass();
-      (void) llvm::createTailCallEliminationPass();
-      (void)llvm::createTLSVariableHoistPass();
-      (void) llvm::createConstantHoistingPass();
-      (void)llvm::createCodeGenPrepareLegacyPass();
-      (void) llvm::createPostInlineEntryExitInstrumenterPass();
-      (void) llvm::createEarlyCSEPass();
-      (void) llvm::createGVNPass();
-      (void) llvm::createPostDomTree();
-      (void) llvm::createMergeICmpsLegacyPass();
-      (void) llvm::createExpandLargeDivRemPass();
-      (void)llvm::createExpandMemCmpLegacyPass();
-      std::string buf;
-      llvm::raw_string_ostream os(buf);
-      (void) llvm::createPrintModulePass(os);
-      (void) llvm::createPrintFunctionPass(os);
-      (void) llvm::createSinkingPass();
-      (void) llvm::createLowerAtomicPass();
-      (void) llvm::createLoadStoreVectorizerPass();
-      (void) llvm::createPartiallyInlineLibCallsPass();
-      (void) llvm::createSeparateConstOffsetFromGEPPass();
-      (void) llvm::createSpeculativeExecutionPass();
-      (void) llvm::createSpeculativeExecutionIfHasBranchDivergencePass();
-      (void) llvm::createStraightLineStrengthReducePass();
-      (void)llvm::createScalarizeMaskedMemIntrinLegacyPass();
-      (void) llvm::createHardwareLoopsLegacyPass();
-      (void) llvm::createUnifyLoopExitsPass();
-      (void) llvm::createFixIrreduciblePass();
-      (void)llvm::createSelectOptimizePass();
+    (void)llvm::createAtomicExpandLegacyPass();
+    (void)llvm::createBasicAAWrapperPass();
+    (void)llvm::createSCEVAAWrapperPass();
+    (void)llvm::createTypeBasedAAWrapperPass();
+    (void)llvm::createScopedNoAliasAAWrapperPass();
+    (void)llvm::createBreakCriticalEdgesPass();
+    (void)llvm::createCallGraphDOTPrinterPass();
+    (void)llvm::createCallGraphViewerPass();
+    (void)llvm::createCFGSimplificationPass();
+    (void)llvm::createStructurizeCFGPass();
+    (void)llvm::createDXILResourceWrapperPassPass();
+    (void)llvm::createDeadArgEliminationPass();
+    (void)llvm::createDeadCodeEliminationPass();
+    (void)llvm::createDependenceAnalysisWrapperPass();
+    (void)llvm::createDomOnlyPrinterWrapperPassPass();
+    (void)llvm::createDomPrinterWrapperPassPass();
+    (void)llvm::createDomOnlyViewerWrapperPassPass();
+    (void)llvm::createDomViewerWrapperPassPass();
+    (void)llvm::createAlwaysInlinerLegacyPass();
+    (void)llvm::createGlobalsAAWrapperPass();
+    (void)llvm::createInstSimplifyLegacyPass();
+    (void)llvm::createInstructionCombiningPass();
+    (void)llvm::createJMCInstrumenterPass();
+    (void)llvm::createKCFIPass();
+    (void)llvm::createLCSSAPass();
+    (void)llvm::createLICMPass();
+    (void)llvm::createLazyValueInfoPass();
+    (void)llvm::createLoopExtractorPass();
+    (void)llvm::createLoopSimplifyPass();
+    (void)llvm::createLoopStrengthReducePass();
+    (void)llvm::createLoopUnrollPass();
+    (void)llvm::createLowerGlobalDtorsLegacyPass();
+    (void)llvm::createLowerInvokePass();
+    (void)llvm::createLowerSwitchPass();
+    (void)llvm::createNaryReassociatePass();
+    (void)llvm::createObjCARCContractPass();
+    (void)llvm::createPromoteMemoryToRegisterPass();
+    (void)llvm::createPostDomOnlyPrinterWrapperPassPass();
+    (void)llvm::createPostDomPrinterWrapperPassPass();
+    (void)llvm::createPostDomOnlyViewerWrapperPassPass();
+    (void)llvm::createPostDomViewerWrapperPassPass();
+    (void)llvm::createReassociatePass();
+    (void)llvm::createRegionInfoPass();
+    (void)llvm::createRegionOnlyPrinterPass();
+    (void)llvm::createRegionOnlyViewerPass();
+    (void)llvm::createRegionPrinterPass();
+    (void)llvm::createRegionViewerPass();
+    (void)llvm::createSafeStackPass();
+    (void)llvm::createSROAPass();
+    (void)llvm::createSingleLoopExtractorPass();
+    (void)llvm::createTailCallEliminationPass();
+    (void)llvm::createTLSVariableHoistPass();
+    (void)llvm::createConstantHoistingPass();
+    (void)llvm::createCodeGenPrepareLegacyPass();
+    (void)llvm::createPostInlineEntryExitInstrumenterPass();
+    (void)llvm::createEarlyCSEPass();
+    (void)llvm::createGVNPass();
+    (void)llvm::createPostDomTree();
+    (void)llvm::createMergeICmpsLegacyPass();
+    (void)llvm::createExpandLargeDivRemPass();
+    (void)llvm::createExpandMemCmpLegacyPass();
+    std::string buf;
+    llvm::raw_string_ostream os(buf);
+    (void)llvm::createPrintModulePass(os);
+    (void)llvm::createPrintFunctionPass(os);
+    (void)llvm::createSinkingPass();
+    (void)llvm::createLowerAtomicPass();
+    (void)llvm::createLoadStoreVectorizerPass();
+    (void)llvm::createPartiallyInlineLibCallsPass();
+    (void)llvm::createSeparateConstOffsetFromGEPPass();
+    (void)llvm::createSpeculativeExecutionPass();
+    (void)llvm::createSpeculativeExecutionIfHasBranchDivergencePass();
+    (void)llvm::createStraightLineStrengthReducePass();
+    (void)llvm::createScalarizeMaskedMemIntrinLegacyPass();
+    (void)llvm::createHardwareLoopsLegacyPass();
+    (void)llvm::createUnifyLoopExitsPass();
+    (void)llvm::createFixIrreduciblePass();
+    (void)llvm::createSelectOptimizePass();
 
-      (void)new llvm::ScalarEvolutionWrapperPass();
-      llvm::Function::Create(nullptr, llvm::GlobalValue::ExternalLinkage)->viewCFGOnly();
-      llvm::RGPassManager RGM;
-      llvm::TargetLibraryInfoImpl TLII;
-      llvm::TargetLibraryInfo TLI(TLII);
-      llvm::AliasAnalysis AA(TLI);
-      llvm::BatchAAResults BAA(AA);
-      llvm::AliasSetTracker X(BAA);
-      X.add(llvm::MemoryLocation()); // for -print-alias-sets
-      (void) llvm::AreStatisticsEnabled();
-      (void) llvm::sys::RunningOnValgrind();
-    }
-  } ForcePassLinking; // Force link by creating a global definition.
-}
+    (void)new llvm::ScalarEvolutionWrapperPass();
+    llvm::Function::Create(nullptr, llvm::GlobalValue::ExternalLinkage)
+        ->viewCFGOnly();
+    llvm::RGPassManager RGM;
+    llvm::TargetLibraryInfoImpl TLII;
+    llvm::TargetLibraryInfo TLI(TLII);
+    llvm::AliasAnalysis AA(TLI);
+    llvm::BatchAAResults BAA(AA);
+    llvm::AliasSetTracker X(BAA);
+    X.add(llvm::MemoryLocation()); // for -print-alias-sets
+    (void)llvm::AreStatisticsEnabled();
+    (void)llvm::sys::RunningOnValgrind();
+  }
+} ForcePassLinking; // Force link by creating a global definition.
+} // namespace
 
 #endif

From 7bef16a04dc42e7b0a7b8b27fb661b474f46a7ca Mon Sep 17 00:00:00 2001
From: Luke Lau <luke@igalia.com>
Date: Fri, 16 Aug 2024 16:10:39 +0800
Subject: [PATCH 137/441] [RISCV] Merge bitrotate crash test into shuffle
 reverse tests. NFC

Use vscale_range instead of having a separate test to run with +zvl1024b.
---
 ...fixed-vectors-shuffle-reverse-bitrotate.ll | 23 -------------------
 .../rvv/fixed-vectors-shuffle-reverse.ll      | 18 +++++++++++++++
 2 files changed, 18 insertions(+), 23 deletions(-)
 delete mode 100644 llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-reverse-bitrotate.ll

diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-reverse-bitrotate.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-reverse-bitrotate.ll
deleted file mode 100644
index a81f740f1739340..000000000000000
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-reverse-bitrotate.ll
+++ /dev/null
@@ -1,23 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
-; RUN: llc -mtriple=riscv32 -mattr=+v,+zvl1024b -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=riscv64 -mattr=+v,+zvl1024b -verify-machineinstrs < %s | FileCheck %s
-
-; There is no corresponding v1i256 type, so make sure we don't crash if we try
-; to lower via lowerBitreverseShuffle.
-define <256 x i1> @reverse_v256i1(<256 x i1> %a) {
-; CHECK-LABEL: reverse_v256i1:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a0, 256
-; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
-; CHECK-NEXT:    vmv.v.i v8, 0
-; CHECK-NEXT:    vmerge.vim v8, v8, 1, v0
-; CHECK-NEXT:    vid.v v10
-; CHECK-NEXT:    vrsub.vi v10, v10, -1
-; CHECK-NEXT:    vrgather.vv v12, v8, v10
-; CHECK-NEXT:    vmsne.vi v0, v12, 0
-; CHECK-NEXT:    ret
-  %res = call <256 x i1> @llvm.vector.reverse.v256i1(<256 x i1> %a)
-  ret <256 x i1> %res
-}
-
-declare <256 x i1> @llvm.vector.reverse.v256i1(<256 x i1>)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-reverse.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-reverse.ll
index cb7fde9188e813d..e773d93fad47961 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-reverse.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-reverse.ll
@@ -1232,3 +1232,21 @@ define <8 x double> @reverse_v8f64_2(<4 x double> %a, <4 x double> %b) {
   %res = shufflevector <4 x double> %a, <4 x double> %b, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   ret <8 x double> %res
 }
+
+; There is no corresponding v1i256 type, so make sure we don't crash if we try
+; to lower via lowerBitreverseShuffle.
+define <256 x i1> @reverse_v256i1(<256 x i1> %a) vscale_range(16, 1024) {
+; CHECK-LABEL: reverse_v256i1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a0, 256
+; CHECK-NEXT:    vsetvli zero, a0, e8, m2, ta, ma
+; CHECK-NEXT:    vmv.v.i v8, 0
+; CHECK-NEXT:    vmerge.vim v8, v8, 1, v0
+; CHECK-NEXT:    vid.v v10
+; CHECK-NEXT:    vrsub.vi v10, v10, -1
+; CHECK-NEXT:    vrgather.vv v12, v8, v10
+; CHECK-NEXT:    vmsne.vi v0, v12, 0
+; CHECK-NEXT:    ret
+  %res = shufflevector <256 x i1> %a, <256 x i1> poison, <256 x i32> <i32 255, i32 254, i32 253, i32 252, i32 251, i32 250, i32 249, i32 248, i32 247, i32 246, i32 245, i32 244, i32 243, i32 242, i32 241, i32 240, i32 239, i32 238, i32 237, i32 236, i32 235, i32 234, i32 233, i32 232, i32 231, i32 230, i32 229, i32 228, i32 227, i32 226, i32 225, i32 224, i32 223, i32 222, i32 221, i32 220, i32 219, i32 218, i32 217, i32 216, i32 215, i32 214, i32 213, i32 212, i32 211, i32 210, i32 209, i32 208, i32 207, i32 206, i32 205, i32 204, i32 203, i32 202, i32 201, i32 200, i32 199, i32 198, i32 197, i32 196, i32 195, i32 194, i32 193, i32 192, i32 191, i32 190, i32 189, i32 188, i32 187, i32 186, i32 185, i32 184, i32 183, i32 182, i32 181, i32 180, i32 179, i32 178, i32 177, i32 176, i32 175, i32 174, i32 173, i32 172, i32 171, i32 170, i32 169, i32 168, i32 167, i32 166, i32 165, i32 164, i32 163, i32 162, i32 161, i32 160, i32 159, i32 158, i32 157, i32 156, i32 155, i32 154, i32 153, i32 152, i32 151, i32 150, i32 149, i32 148, i32 147, i32 146, i32 145, i32 144, i32 143, i32 142, i32 141, i32 140, i32 139, i32 138, i32 137, i32 136, i32 135, i32 134, i32 133, i32 132, i32 131, i32 130, i32 129, i32 128, i32 127, i32 126, i32 125, i32 124, i32 123, i32 122, i32 121, i32 120, i32 119, i32 118, i32 117, i32 116, i32 115, i32 114, i32 113, i32 112, i32 111, i32 110, i32 109, i32 108, i32 107, i32 106, i32 105, i32 104, i32 103, i32 102, i32 101, i32 100, i32 99, i32 98, i32 97, i32 96, i32 95, i32 94, i32 93, i32 92, i32 91, i32 90, i32 89, i32 88, i32 87, i32 86, i32 85, i32 84, i32 83, i32 82, i32 81, i32 80, i32 79, i32 78, i32 77, i32 76, i32 75, i32 74, i32 73, i32 72, i32 71, i32 70, i32 69, i32 68, i32 67, i32 66, i32 65, i32 64, i32 63, i32 62, i32 61, i32 60, i32 59, i32 58, i32 57, i32 56, i32 55, i32 54, i32 53, i32 52, i32 51, i32 50, i32 49, i32 48, i32 47, i32 46, i32 45, i32 44, i32 43, i32 42, i32 41, i32 40, i32 39, i32 38, i32 37, i32 36, i32 35, i32 34, i32 33, i32 32, i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <256 x i1> %res
+}

From 59d7df4255cc0de5d448aa7908f1f8407860c802 Mon Sep 17 00:00:00 2001
From: Freddy Ye <freddy.ye@intel.com>
Date: Fri, 16 Aug 2024 16:26:07 +0800
Subject: [PATCH 138/441] [NFC][X86] Refactor: merge avx512_binop_all2 into
 avx512_binop_all (#104561)

---
 llvm/lib/Target/X86/X86InstrAVX512.td | 52 +++++++++------------------
 1 file changed, 16 insertions(+), 36 deletions(-)

diff --git a/llvm/lib/Target/X86/X86InstrAVX512.td b/llvm/lib/Target/X86/X86InstrAVX512.td
index e616a8a37c64872..a606962a5538091 100644
--- a/llvm/lib/Target/X86/X86InstrAVX512.td
+++ b/llvm/lib/Target/X86/X86InstrAVX512.td
@@ -4855,27 +4855,29 @@ multiclass avx512_binop_all<bits<8> opc, string OpcodeStr,
                             X86SchedWriteWidths sched,
                             AVX512VLVectorVTInfo _SrcVTInfo,
                             AVX512VLVectorVTInfo _DstVTInfo,
-                            SDNode OpNode, Predicate prd,  bit IsCommutable = 0> {
+                            SDNode OpNode, Predicate prd,
+                            X86VectorVTInfo _VTInfo512 = _SrcVTInfo.info512,
+                            X86VectorVTInfo _VTInfo256 = _SrcVTInfo.info256,
+                            X86VectorVTInfo _VTInfo128 = _SrcVTInfo.info128> {
   let Predicates = [prd] in
     defm NAME#Z : avx512_binop_rm2<opc, OpcodeStr, sched.ZMM, OpNode,
                                  _SrcVTInfo.info512, _DstVTInfo.info512,
-                                 v8i64_info, IsCommutable>,
-                                  EVEX_V512, EVEX_CD8<64, CD8VF>, REX_W;
+                                 _VTInfo512>, EVEX_V512;
   let Predicates = [HasVLX, prd] in {
     defm NAME#Z256 : avx512_binop_rm2<opc, OpcodeStr, sched.YMM, OpNode,
                                       _SrcVTInfo.info256, _DstVTInfo.info256,
-                                      v4i64x_info, IsCommutable>,
-                                      EVEX_V256, EVEX_CD8<64, CD8VF>, REX_W;
+                                      _VTInfo256>, EVEX_V256;
     defm NAME#Z128 : avx512_binop_rm2<opc, OpcodeStr, sched.XMM, OpNode,
                                       _SrcVTInfo.info128, _DstVTInfo.info128,
-                                      v2i64x_info, IsCommutable>,
-                                     EVEX_V128, EVEX_CD8<64, CD8VF>, REX_W;
+                                      _VTInfo128>, EVEX_V128;
   }
 }
 
 defm VPMULTISHIFTQB : avx512_binop_all<0x83, "vpmultishiftqb", SchedWriteVecALU,
-                                avx512vl_i8_info, avx512vl_i8_info,
-                                X86multishift, HasVBMI, 0>, T8;
+                                       avx512vl_i8_info, avx512vl_i8_info,
+                                       X86multishift, HasVBMI, v8i64_info,
+                                       v4i64x_info, v2i64x_info>, T8,
+                                       EVEX_CD8<64, CD8VF>, REX_W;
 
 multiclass avx512_packs_rmb<bits<8> opc, string OpcodeStr, SDNode OpNode,
                             X86VectorVTInfo _Src, X86VectorVTInfo _Dst,
@@ -12670,34 +12672,12 @@ defm VP2INTERSECTD : avx512_vp2intersect<SchedWriteVecALU, avx512vl_i32_info>;
 defm VP2INTERSECTQ : avx512_vp2intersect<SchedWriteVecALU, avx512vl_i64_info>, REX_W;
 }
 
-multiclass avx512_binop_all2<bits<8> opc, string OpcodeStr,
-                             X86SchedWriteWidths sched,
-                             AVX512VLVectorVTInfo _SrcVTInfo,
-                             AVX512VLVectorVTInfo _DstVTInfo,
-                             SDNode OpNode, Predicate prd,
-                             bit IsCommutable = 0> {
-  let Predicates = [prd] in
-    defm NAME#Z : avx512_binop_rm2<opc, OpcodeStr, sched.ZMM, OpNode,
-                                   _SrcVTInfo.info512, _DstVTInfo.info512,
-                                   _SrcVTInfo.info512, IsCommutable>,
-                                   EVEX_V512, EVEX_CD8<32, CD8VF>;
-  let Predicates = [HasVLX, prd] in {
-    defm NAME#Z256 : avx512_binop_rm2<opc, OpcodeStr, sched.YMM, OpNode,
-                                      _SrcVTInfo.info256, _DstVTInfo.info256,
-                                      _SrcVTInfo.info256, IsCommutable>,
-                                     EVEX_V256, EVEX_CD8<32, CD8VF>;
-    defm NAME#Z128 : avx512_binop_rm2<opc, OpcodeStr, sched.XMM, OpNode,
-                                      _SrcVTInfo.info128, _DstVTInfo.info128,
-                                      _SrcVTInfo.info128, IsCommutable>,
-                                      EVEX_V128, EVEX_CD8<32, CD8VF>;
-  }
-}
-
 let ExeDomain = SSEPackedSingle in
-defm VCVTNE2PS2BF16 : avx512_binop_all2<0x72, "vcvtne2ps2bf16",
-                                        SchedWriteCvtPD2PS, //FIXME: Should be SchedWriteCvtPS2BF
-                                        avx512vl_f32_info, avx512vl_bf16_info,
-                                        X86cvtne2ps2bf16, HasBF16, 0>, T8, XD;
+defm VCVTNE2PS2BF16 : avx512_binop_all<0x72, "vcvtne2ps2bf16",
+                                       SchedWriteCvtPD2PS, //FIXME: Should be SchedWriteCvtPS2BF
+                                       avx512vl_f32_info, avx512vl_bf16_info,
+                                       X86cvtne2ps2bf16, HasBF16>, T8, XD,
+                                       EVEX_CD8<32, CD8VF>;
 
 // Truncate Float to BFloat16
 multiclass avx512_cvtps2bf16<bits<8> opc, string OpcodeStr,

From 8ca5ff2743f6020e29ee923114e2762b53bd32b1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Martin=20Storsj=C3=B6?= <martin@martin.st>
Date: Fri, 16 Aug 2024 11:38:22 +0300
Subject: [PATCH 139/441] Revert "[clang][driver] Fix -print-target-triple OS
 version for apple targets" (#104563)

This reverts llvm/llvm-project#104037 /
7227b44f928a87b5d7fb05bd1539fdfb6d4958dc.

This change had the unintended consequence of making e.g. `clang -target
armv7-windows-gnu --print-target-triple` output
`thumbv7-unknown-windows-gnu` rather than `armv7-unknown-windows-gnu`.
---
 clang/lib/Driver/Driver.cpp                   | 13 ++----
 .../test/Driver/darwin-print-target-triple.c  | 42 -------------------
 2 files changed, 3 insertions(+), 52 deletions(-)
 delete mode 100644 clang/test/Driver/darwin-print-target-triple.c

diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index 5b95019c25cab67..e12416e51f8d248 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -2271,7 +2271,8 @@ bool Driver::HandleImmediateArgs(Compilation &C) {
     return false;
   }
 
-  auto initializeTargets = [&]() {
+  if (C.getArgs().hasArg(options::OPT_print_libgcc_file_name)) {
+    ToolChain::RuntimeLibType RLT = TC.GetRuntimeLibType(C.getArgs());
     const llvm::Triple Triple(TC.ComputeEffectiveClangTriple(C.getArgs()));
     // The 'Darwin' toolchain is initialized only when its arguments are
     // computed. Get the default arguments for OFK_None to ensure that
@@ -2281,12 +2282,6 @@ bool Driver::HandleImmediateArgs(Compilation &C) {
     // FIXME: For some more esoteric targets the default toolchain is not the
     //        correct one.
     C.getArgsForToolChain(&TC, Triple.getArchName(), Action::OFK_None);
-    return Triple;
-  };
-
-  if (C.getArgs().hasArg(options::OPT_print_libgcc_file_name)) {
-    ToolChain::RuntimeLibType RLT = TC.GetRuntimeLibType(C.getArgs());
-    const llvm::Triple Triple = initializeTargets();
     RegisterEffectiveTriple TripleRAII(TC, Triple);
     switch (RLT) {
     case ToolChain::RLT_CompilerRT:
@@ -2330,9 +2325,7 @@ bool Driver::HandleImmediateArgs(Compilation &C) {
   }
 
   if (C.getArgs().hasArg(options::OPT_print_target_triple)) {
-    initializeTargets();
-    llvm::Triple Triple(TC.ComputeEffectiveClangTriple(C.getArgs()));
-    llvm::outs() << Triple.getTriple() << "\n";
+    llvm::outs() << TC.getTripleString() << "\n";
     return false;
   }
 
diff --git a/clang/test/Driver/darwin-print-target-triple.c b/clang/test/Driver/darwin-print-target-triple.c
deleted file mode 100644
index 4f5fdfe9d0db340..000000000000000
--- a/clang/test/Driver/darwin-print-target-triple.c
+++ /dev/null
@@ -1,42 +0,0 @@
-// Test the output of -print-target-triple on Darwin.
-// See https://github.com/llvm/llvm-project/issues/61762
-
-//
-// All platforms
-//
-
-// RUN: %clang -print-target-triple \
-// RUN:     --target=x86_64-apple-macos -mmacos-version-min=15 \
-// RUN:     -resource-dir=%S/Inputs/resource_dir 2>&1 \
-// RUN:   | FileCheck --check-prefix=CHECK-CLANGRT-MACOS %s
-// CHECK-CLANGRT-MACOS: x86_64-apple-macosx15.0.0
-
-// RUN: %clang -print-target-triple \
-// RUN:     --target=arm64-apple-ios -mios-version-min=9 \
-// RUN:     -resource-dir=%S/Inputs/resource_dir 2>&1 \
-// RUN:   | FileCheck --check-prefix=CHECK-CLANGRT-IOS %s
-// CHECK-CLANGRT-IOS: arm64-apple-ios9.0.0
-
-// RUN: %clang -print-target-triple \
-// RUN:     --target=arm64-apple-watchos -mwatchos-version-min=3 \
-// RUN:     -resource-dir=%S/Inputs/resource_dir 2>&1 \
-// RUN:   | FileCheck --check-prefix=CHECK-CLANGRT-WATCHOS %s
-// CHECK-CLANGRT-WATCHOS: arm64-apple-watchos3.0.0
-
-// RUN: %clang -print-target-triple \
-// RUN:     --target=armv7k-apple-watchos -mwatchos-version-min=3 \
-// RUN:     -resource-dir=%S/Inputs/resource_dir 2>&1 \
-// RUN:   | FileCheck --check-prefix=CHECK-CLANGRT-WATCHOS-ARMV7K %s
-// CHECK-CLANGRT-WATCHOS-ARMV7K: thumbv7-apple-watchos3.0.0
-
-// RUN: %clang -print-target-triple \
-// RUN:     --target=arm64-apple-tvos -mtvos-version-min=1\
-// RUN:     -resource-dir=%S/Inputs/resource_dir 2>&1 \
-// RUN:   | FileCheck --check-prefix=CHECK-CLANGRT-TVOS %s
-// CHECK-CLANGRT-TVOS: arm64-apple-tvos1.0.0
-
-// RUN: %clang -print-target-triple \
-// RUN:     --target=arm64-apple-driverkit \
-// RUN:     -resource-dir=%S/Inputs/resource_dir 2>&1 \
-// RUN:   | FileCheck --check-prefix=CHECK-CLANGRT-DRIVERKIT %s
-// CHECK-CLANGRT-DRIVERKIT: arm64-apple-driverkit19.0.0

From fc1b01963857b5c04980c713145a71d6b858ad8a Mon Sep 17 00:00:00 2001
From: v01dXYZ <14996868+v01dXYZ@users.noreply.github.com>
Date: Fri, 16 Aug 2024 10:46:20 +0200
Subject: [PATCH 140/441] [DAG] SD Pattern Match: Operands patterns with VP
 Context  (#103308)

Currently, when using a VP match context with `sd_context_match`, only Opcode matching is possible (`m_Opc(Opcode)`).

This PR suggest a way to make patterns with Operands (eg `m_Node`, `m_Add`, ...) works with a VP context.

This PR blocks another PR https://github.com/llvm/llvm-project/pull/102877.

Co-authored-by: v01dxyz <v01dxyz@v01d.xyz>
---
 llvm/include/llvm/CodeGen/SDPatternMatch.h    | 20 ++++++---
 llvm/lib/CodeGen/SelectionDAG/MatchContext.h  |  6 +++
 .../CodeGen/SelectionDAGPatternMatchTest.cpp  | 43 +++++++++++++++++++
 3 files changed, 62 insertions(+), 7 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/SDPatternMatch.h b/llvm/include/llvm/CodeGen/SDPatternMatch.h
index 88ddd43a2a8913c..b1aa87ca2d3e134 100644
--- a/llvm/include/llvm/CodeGen/SDPatternMatch.h
+++ b/llvm/include/llvm/CodeGen/SDPatternMatch.h
@@ -47,6 +47,8 @@ class BasicMatchContext {
   bool match(SDValue N, unsigned Opcode) const {
     return N->getOpcode() == Opcode;
   }
+
+  unsigned getNumOperands(SDValue N) const { return N->getNumOperands(); }
 };
 
 template <typename Pattern, typename MatchContext>
@@ -390,7 +392,8 @@ template <unsigned OpIdx, typename... OpndPreds> struct Operands_match {
   template <typename MatchContext>
   bool match(const MatchContext &Ctx, SDValue N) {
     // Returns false if there are more operands than predicates;
-    return N->getNumOperands() == OpIdx;
+    // Ignores the last two operands if both the Context and the Node are VP
+    return Ctx.getNumOperands(N) == OpIdx;
   }
 };
 
@@ -424,8 +427,9 @@ template <bool ExcludeChain> struct EffectiveOperands {
   unsigned Size = 0;
   unsigned FirstIndex = 0;
 
-  explicit EffectiveOperands(SDValue N) {
-    const unsigned TotalNumOps = N->getNumOperands();
+  template <typename MatchContext>
+  explicit EffectiveOperands(SDValue N, const MatchContext &Ctx) {
+    const unsigned TotalNumOps = Ctx.getNumOperands(N);
     FirstIndex = TotalNumOps;
     for (unsigned I = 0; I < TotalNumOps; ++I) {
       // Count the number of non-chain and non-glue nodes (we ignore chain
@@ -444,7 +448,9 @@ template <> struct EffectiveOperands<false> {
   unsigned Size = 0;
   unsigned FirstIndex = 0;
 
-  explicit EffectiveOperands(SDValue N) : Size(N->getNumOperands()) {}
+  template <typename MatchContext>
+  explicit EffectiveOperands(SDValue N, const MatchContext &Ctx)
+      : Size(Ctx.getNumOperands(N)) {}
 };
 
 // === Ternary operations ===
@@ -463,7 +469,7 @@ struct TernaryOpc_match {
   template <typename MatchContext>
   bool match(const MatchContext &Ctx, SDValue N) {
     if (sd_context_match(N, Ctx, m_Opc(Opcode))) {
-      EffectiveOperands<ExcludeChain> EO(N);
+      EffectiveOperands<ExcludeChain> EO(N, Ctx);
       assert(EO.Size == 3);
       return ((Op0.match(Ctx, N->getOperand(EO.FirstIndex)) &&
                Op1.match(Ctx, N->getOperand(EO.FirstIndex + 1))) ||
@@ -515,7 +521,7 @@ struct BinaryOpc_match {
   template <typename MatchContext>
   bool match(const MatchContext &Ctx, SDValue N) {
     if (sd_context_match(N, Ctx, m_Opc(Opcode))) {
-      EffectiveOperands<ExcludeChain> EO(N);
+      EffectiveOperands<ExcludeChain> EO(N, Ctx);
       assert(EO.Size == 2);
       return (LHS.match(Ctx, N->getOperand(EO.FirstIndex)) &&
               RHS.match(Ctx, N->getOperand(EO.FirstIndex + 1))) ||
@@ -667,7 +673,7 @@ template <typename Opnd_P, bool ExcludeChain = false> struct UnaryOpc_match {
   template <typename MatchContext>
   bool match(const MatchContext &Ctx, SDValue N) {
     if (sd_context_match(N, Ctx, m_Opc(Opcode))) {
-      EffectiveOperands<ExcludeChain> EO(N);
+      EffectiveOperands<ExcludeChain> EO(N, Ctx);
       assert(EO.Size == 1);
       return Opnd.match(Ctx, N->getOperand(EO.FirstIndex));
     }
diff --git a/llvm/lib/CodeGen/SelectionDAG/MatchContext.h b/llvm/lib/CodeGen/SelectionDAG/MatchContext.h
index d81a0c2e87e7d28..8f03532af99e865 100644
--- a/llvm/lib/CodeGen/SelectionDAG/MatchContext.h
+++ b/llvm/lib/CodeGen/SelectionDAG/MatchContext.h
@@ -45,6 +45,8 @@ class EmptyMatchContext {
                                 bool LegalOnly = false) const {
     return TLI.isOperationLegalOrCustom(Op, VT, LegalOnly);
   }
+
+  unsigned getNumOperands(SDValue N) const { return N->getNumOperands(); }
 };
 
 class VPMatchContext {
@@ -169,6 +171,10 @@ class VPMatchContext {
     unsigned VPOp = ISD::getVPForBaseOpcode(Op);
     return TLI.isOperationLegalOrCustom(VPOp, VT, LegalOnly);
   }
+
+  unsigned getNumOperands(SDValue N) const {
+    return N->isVPOpcode() ? N->getNumOperands() - 2 : N->getNumOperands();
+  }
 };
 
 } // namespace llvm
diff --git a/llvm/unittests/CodeGen/SelectionDAGPatternMatchTest.cpp b/llvm/unittests/CodeGen/SelectionDAGPatternMatchTest.cpp
index b9414be98623af2..c04fc5621ab4992 100644
--- a/llvm/unittests/CodeGen/SelectionDAGPatternMatchTest.cpp
+++ b/llvm/unittests/CodeGen/SelectionDAGPatternMatchTest.cpp
@@ -393,6 +393,10 @@ struct VPMatchContext : public SDPatternMatch::BasicMatchContext {
     auto BaseOpc = ISD::getBaseOpcodeForVP(OpVal->getOpcode(), false);
     return BaseOpc.has_value() && *BaseOpc == Opc;
   }
+
+  unsigned getNumOperands(SDValue N) const {
+    return N->isVPOpcode() ? N->getNumOperands() - 2 : N->getNumOperands();
+  }
 };
 } // anonymous namespace
 TEST_F(SelectionDAGPatternMatchTest, matchContext) {
@@ -410,15 +414,54 @@ TEST_F(SelectionDAGPatternMatchTest, matchContext) {
                                {Vector0, Vector0, Mask0, Scalar0});
   SDValue VPReduceAdd = DAG->getNode(ISD::VP_REDUCE_ADD, DL, Int32VT,
                                      {Scalar0, VPAdd, Mask0, Scalar0});
+  SDValue Add = DAG->getNode(ISD::ADD, DL, VInt32VT, {Vector0, Vector0});
 
   using namespace SDPatternMatch;
   VPMatchContext VPCtx(DAG.get());
   EXPECT_TRUE(sd_context_match(VPAdd, VPCtx, m_Opc(ISD::ADD)));
+  EXPECT_TRUE(
+      sd_context_match(VPAdd, VPCtx, m_Node(ISD::ADD, m_Value(), m_Value())));
+  // VPMatchContext can't match pattern using explicit VP Opcode
+  EXPECT_FALSE(sd_context_match(VPAdd, VPCtx,
+                                m_Node(ISD::VP_ADD, m_Value(), m_Value())));
+  EXPECT_FALSE(sd_context_match(
+      VPAdd, VPCtx,
+      m_Node(ISD::VP_ADD, m_Value(), m_Value(), m_Value(), m_Value())));
+  // Check Binary Op Pattern
+  EXPECT_TRUE(sd_context_match(VPAdd, VPCtx, m_Add(m_Value(), m_Value())));
   // VP_REDUCE_ADD doesn't have a based opcode, so we use a normal
   // sd_match before switching to VPMatchContext when checking VPAdd.
   EXPECT_TRUE(sd_match(VPReduceAdd, m_Node(ISD::VP_REDUCE_ADD, m_Value(),
                                            m_Context(VPCtx, m_Opc(ISD::ADD)),
                                            m_Value(), m_Value())));
+  // non-vector predicated should match too
+  EXPECT_TRUE(sd_context_match(Add, VPCtx, m_Opc(ISD::ADD)));
+  EXPECT_TRUE(
+      sd_context_match(Add, VPCtx, m_Node(ISD::ADD, m_Value(), m_Value())));
+  EXPECT_FALSE(sd_context_match(
+      Add, VPCtx,
+      m_Node(ISD::ADD, m_Value(), m_Value(), m_Value(), m_Value())));
+  EXPECT_TRUE(sd_context_match(Add, VPCtx, m_Add(m_Value(), m_Value())));
+}
+
+TEST_F(SelectionDAGPatternMatchTest, matchVPWithBasicContext) {
+  SDLoc DL;
+  auto BoolVT = EVT::getIntegerVT(Context, 1);
+  auto Int32VT = EVT::getIntegerVT(Context, 32);
+  auto VInt32VT = EVT::getVectorVT(Context, Int32VT, 4);
+  auto MaskVT = EVT::getVectorVT(Context, BoolVT, 4);
+
+  SDValue Vector0 = DAG->getCopyFromReg(DAG->getEntryNode(), DL, 1, VInt32VT);
+  SDValue Mask = DAG->getCopyFromReg(DAG->getEntryNode(), DL, 2, MaskVT);
+  SDValue EL = DAG->getCopyFromReg(DAG->getEntryNode(), DL, 3, Int32VT);
+
+  SDValue VPAdd =
+      DAG->getNode(ISD::VP_ADD, DL, VInt32VT, Vector0, Vector0, Mask, EL);
+
+  using namespace SDPatternMatch;
+  EXPECT_FALSE(sd_match(VPAdd, m_Node(ISD::VP_ADD, m_Value(), m_Value())));
+  EXPECT_TRUE(sd_match(
+      VPAdd, m_Node(ISD::VP_ADD, m_Value(), m_Value(), m_Value(), m_Value())));
 }
 
 TEST_F(SelectionDAGPatternMatchTest, matchAdvancedProperties) {

From 75cb9edf09fdc091e5bc0f3d46a96c2877735a39 Mon Sep 17 00:00:00 2001
From: Victor Perez <victor.perez@codeplay.com>
Date: Fri, 16 Aug 2024 11:18:35 +0200
Subject: [PATCH 141/441] [MLIR][GPU-LLVM] Add GPU to LLVM-SPV address space
 mapping (#102621)

Implement mapping:

- `global`: 1
- `workgroup`: 3
- `private`: 0

Add `addressSpaceToStorageClass`, mapping GPU address spaces to SPIR-V
storage classes to be able to use SPIR-V's
`storageClassToAddressSpace`, mapping SPIR-V storage classes to LLVM
address spaces according to our mapping above *by definition*.

---------

Signed-off-by: Victor Perez <victor.perez@codeplay.com>
---
 .../GPUCommon/AttrToSPIRVConverter.h          | 18 +++++++++++++++
 .../GPUToLLVMSPV/GPUToLLVMSPVPass.h           |  5 ++++
 .../GPUCommon/AttrToSPIRVConverter.cpp        | 23 +++++++++++++++++++
 mlir/lib/Conversion/GPUCommon/CMakeLists.txt  |  1 +
 .../Conversion/GPUToLLVMSPV/GPUToLLVMSPV.cpp  | 22 +++++++++++++++---
 .../GPUToLLVMSPV/gpu-to-llvm-spv.mlir         | 20 ++++++++++++----
 6 files changed, 82 insertions(+), 7 deletions(-)
 create mode 100644 mlir/include/mlir/Conversion/GPUCommon/AttrToSPIRVConverter.h
 create mode 100644 mlir/lib/Conversion/GPUCommon/AttrToSPIRVConverter.cpp

diff --git a/mlir/include/mlir/Conversion/GPUCommon/AttrToSPIRVConverter.h b/mlir/include/mlir/Conversion/GPUCommon/AttrToSPIRVConverter.h
new file mode 100644
index 000000000000000..e1dcedba094c03d
--- /dev/null
+++ b/mlir/include/mlir/Conversion/GPUCommon/AttrToSPIRVConverter.h
@@ -0,0 +1,18 @@
+//===- AttrToSPIRVConverter.h - GPU attributes conversion to SPIR-V - C++ -===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef MLIR_CONVERSION_GPUCOMMON_ATTRTOSPIRVCONVERTER_H_
+#define MLIR_CONVERSION_GPUCOMMON_ATTRTOSPIRVCONVERTER_H_
+
+#include <mlir/Dialect/GPU/IR/GPUDialect.h>
+#include <mlir/Dialect/SPIRV/IR/SPIRVEnums.h>
+
+namespace mlir {
+spirv::StorageClass addressSpaceToStorageClass(gpu::AddressSpace addressSpace);
+} // namespace mlir
+
+#endif // MLIR_CONVERSION_GPUCOMMON_ATTRTOSPIRVCONVERTER_H_
diff --git a/mlir/include/mlir/Conversion/GPUToLLVMSPV/GPUToLLVMSPVPass.h b/mlir/include/mlir/Conversion/GPUToLLVMSPV/GPUToLLVMSPVPass.h
index e156c3093e21be4..20c9f9c394525df 100644
--- a/mlir/include/mlir/Conversion/GPUToLLVMSPV/GPUToLLVMSPVPass.h
+++ b/mlir/include/mlir/Conversion/GPUToLLVMSPV/GPUToLLVMSPVPass.h
@@ -16,12 +16,17 @@ class DialectRegistry;
 class LLVMTypeConverter;
 class RewritePatternSet;
 class Pass;
+class TypeConverter;
 
 #define GEN_PASS_DECL_CONVERTGPUOPSTOLLVMSPVOPS
 #include "mlir/Conversion/Passes.h.inc"
 
 void populateGpuToLLVMSPVConversionPatterns(LLVMTypeConverter &converter,
                                             RewritePatternSet &patterns);
+
+/// Populates memory space attribute conversion rules for lowering
+/// gpu.address_space to integer values.
+void populateGpuMemorySpaceAttributeConversions(TypeConverter &typeConverter);
 } // namespace mlir
 
 #endif // MLIR_CONVERSION_GPUTOLLVMSPV_GPUTOLLVMSPVPASS_H_
diff --git a/mlir/lib/Conversion/GPUCommon/AttrToSPIRVConverter.cpp b/mlir/lib/Conversion/GPUCommon/AttrToSPIRVConverter.cpp
new file mode 100644
index 000000000000000..fcabad32974acb7
--- /dev/null
+++ b/mlir/lib/Conversion/GPUCommon/AttrToSPIRVConverter.cpp
@@ -0,0 +1,23 @@
+//===- AttrToSPIRVConverter.cpp - GPU attributes conversion to SPIR-V - C++===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <mlir/Conversion/GPUCommon/AttrToSPIRVConverter.h>
+
+namespace mlir {
+spirv::StorageClass addressSpaceToStorageClass(gpu::AddressSpace addressSpace) {
+  switch (addressSpace) {
+  case gpu::AddressSpace::Global:
+    return spirv::StorageClass::CrossWorkgroup;
+  case gpu::AddressSpace::Workgroup:
+    return spirv::StorageClass::Workgroup;
+  case gpu::AddressSpace::Private:
+    return spirv::StorageClass::Private;
+  }
+  llvm_unreachable("Unhandled storage class");
+}
+} // namespace mlir
diff --git a/mlir/lib/Conversion/GPUCommon/CMakeLists.txt b/mlir/lib/Conversion/GPUCommon/CMakeLists.txt
index 55bd168b83a0ad4..ce914c0ea3dd8aa 100644
--- a/mlir/lib/Conversion/GPUCommon/CMakeLists.txt
+++ b/mlir/lib/Conversion/GPUCommon/CMakeLists.txt
@@ -15,6 +15,7 @@ if (MLIR_ENABLE_ROCM_CONVERSIONS)
 endif()
 
 add_mlir_conversion_library(MLIRGPUToGPURuntimeTransforms
+  AttrToSPIRVConverter.cpp
   GPUToLLVMConversion.cpp
   GPUOpsLowering.cpp
 
diff --git a/mlir/lib/Conversion/GPUToLLVMSPV/GPUToLLVMSPV.cpp b/mlir/lib/Conversion/GPUToLLVMSPV/GPUToLLVMSPV.cpp
index 36e4a6a38a68e47..ced4236402923ad 100644
--- a/mlir/lib/Conversion/GPUToLLVMSPV/GPUToLLVMSPV.cpp
+++ b/mlir/lib/Conversion/GPUToLLVMSPV/GPUToLLVMSPV.cpp
@@ -9,6 +9,8 @@
 #include "mlir/Conversion/GPUToLLVMSPV/GPUToLLVMSPVPass.h"
 
 #include "../GPUCommon/GPUOpsLowering.h"
+#include "mlir/Conversion/GPUCommon/AttrToSPIRVConverter.h"
+#include "mlir/Conversion/GPUCommon/GPUCommonPass.h"
 #include "mlir/Conversion/LLVMCommon/ConversionTarget.h"
 #include "mlir/Conversion/LLVMCommon/LoweringOptions.h"
 #include "mlir/Conversion/LLVMCommon/Pattern.h"
@@ -328,6 +330,7 @@ struct GPUToLLVMSPVConversionPass final
                         gpu::ReturnOp, gpu::ShuffleOp, gpu::ThreadIdOp>();
 
     populateGpuToLLVMSPVConversionPatterns(converter, patterns);
+    populateGpuMemorySpaceAttributeConversions(converter);
 
     if (failed(applyPartialConversion(getOperation(), target,
                                       std::move(patterns))))
@@ -341,6 +344,15 @@ struct GPUToLLVMSPVConversionPass final
 //===----------------------------------------------------------------------===//
 
 namespace mlir {
+namespace {
+static unsigned
+gpuAddressSpaceToOCLAddressSpace(gpu::AddressSpace addressSpace) {
+  constexpr spirv::ClientAPI clientAPI = spirv::ClientAPI::OpenCL;
+  return storageClassToAddressSpace(clientAPI,
+                                    addressSpaceToStorageClass(addressSpace));
+}
+} // namespace
+
 void populateGpuToLLVMSPVConversionPatterns(LLVMTypeConverter &typeConverter,
                                             RewritePatternSet &patterns) {
   patterns.add<GPUBarrierConversion, GPUReturnOpLowering, GPUShuffleConversion,
@@ -349,12 +361,11 @@ void populateGpuToLLVMSPVConversionPatterns(LLVMTypeConverter &typeConverter,
                LaunchConfigOpConversion<gpu::BlockDimOp>,
                LaunchConfigOpConversion<gpu::ThreadIdOp>,
                LaunchConfigOpConversion<gpu::GlobalIdOp>>(typeConverter);
-  constexpr spirv::ClientAPI clientAPI = spirv::ClientAPI::OpenCL;
   MLIRContext *context = &typeConverter.getContext();
   unsigned privateAddressSpace =
-      storageClassToAddressSpace(clientAPI, spirv::StorageClass::Function);
+      gpuAddressSpaceToOCLAddressSpace(gpu::AddressSpace::Private);
   unsigned localAddressSpace =
-      storageClassToAddressSpace(clientAPI, spirv::StorageClass::Workgroup);
+      gpuAddressSpaceToOCLAddressSpace(gpu::AddressSpace::Workgroup);
   OperationName llvmFuncOpName(LLVM::LLVMFuncOp::getOperationName(), context);
   StringAttr kernelBlockSizeAttributeName =
       LLVM::LLVMFuncOp::getReqdWorkGroupSizeAttrName(llvmFuncOpName);
@@ -366,4 +377,9 @@ void populateGpuToLLVMSPVConversionPatterns(LLVMTypeConverter &typeConverter,
           LLVM::CConv::SPIR_KERNEL, LLVM::CConv::SPIR_FUNC,
           /*encodeWorkgroupAttributionsAsArguments=*/true});
 }
+
+void populateGpuMemorySpaceAttributeConversions(TypeConverter &typeConverter) {
+  populateGpuMemorySpaceAttributeConversions(typeConverter,
+                                             gpuAddressSpaceToOCLAddressSpace);
+}
 } // namespace mlir
diff --git a/mlir/test/Conversion/GPUToLLVMSPV/gpu-to-llvm-spv.mlir b/mlir/test/Conversion/GPUToLLVMSPV/gpu-to-llvm-spv.mlir
index 8e133288b832b68..ec4f4a304d50734 100644
--- a/mlir/test/Conversion/GPUToLLVMSPV/gpu-to-llvm-spv.mlir
+++ b/mlir/test/Conversion/GPUToLLVMSPV/gpu-to-llvm-spv.mlir
@@ -444,7 +444,7 @@ gpu.module @kernels {
 // CHECK:           %[[VAL_17:.*]] = llvm.insertvalue %[[VAL_15]], %[[VAL_16]][0]
 // CHECK:           llvm.insertvalue %[[VAL_15]], %[[VAL_17]][1]
   gpu.func @kernel_with_private_attributions()
-      private(%arg2: memref<32xf32>, %arg3: memref<16xi16>)
+      private(%arg2: memref<32xf32, #gpu.address_space<private>>, %arg3: memref<16xi16, #gpu.address_space<private>>)
       kernel {
     gpu.return
   }
@@ -471,7 +471,7 @@ gpu.module @kernels {
 // CHECK:           %[[VAL_42:.*]] = llvm.insertvalue %[[VAL_30]], %[[VAL_41]][0]
 // CHECK:           llvm.insertvalue %[[VAL_30]], %[[VAL_42]][1]
   gpu.func @kernel_with_workgoup_attributions()
-      workgroup(%arg2: memref<32xf32, 3>, %arg3: memref<16xi16, 3>)
+      workgroup(%arg2: memref<32xf32, #gpu.address_space<workgroup>>, %arg3: memref<16xi16, #gpu.address_space<workgroup>>)
       kernel {
     gpu.return
   }
@@ -491,8 +491,8 @@ gpu.module @kernels {
 // CHECK-64:        %[[VAL_92:.*]] = llvm.alloca %[[VAL_91]] x i64 : (i64) -> !llvm.ptr
 // CHECK-32:        %[[VAL_92:.*]] = llvm.alloca %[[VAL_91]] x i32 : (i64) -> !llvm.ptr
   gpu.func @kernel_with_both_attributions()
-      workgroup(%arg4: memref<8xf32, 3>, %arg5: memref<16xindex, 3>)
-      private(%arg6: memref<32xi32>, %arg7: memref<32xindex>)
+      workgroup(%arg4: memref<8xf32, #gpu.address_space<workgroup>>, %arg5: memref<16xindex, #gpu.address_space<workgroup>>)
+      private(%arg6: memref<32xi32, #gpu.address_space<private>>, %arg7: memref<32xindex, #gpu.address_space<private>>)
       kernel {
     gpu.return
   }
@@ -503,3 +503,15 @@ gpu.module @kernels {
     gpu.return
   }
 }
+
+// -----
+
+gpu.module @kernels {
+// CHECK-LABEL:   llvm.func spir_funccc @address_spaces(
+// CHECK-SAME:                                          {{.*}}: !llvm.ptr<1>
+// CHECK-SAME:                                          {{.*}}: !llvm.ptr<3>
+// CHECK-SAME:                                          {{.*}}: !llvm.ptr
+  gpu.func @address_spaces(%arg0: memref<f32, #gpu.address_space<global>>, %arg1: memref<f32, #gpu.address_space<workgroup>>, %arg2: memref<f32, #gpu.address_space<private>>) {
+    gpu.return
+  }
+}

From 61abc15a9f94b081cced18277de8ae571e4d853d Mon Sep 17 00:00:00 2001
From: Vlad Serebrennikov <serebrennikov.vladislav@gmail.com>
Date: Fri, 16 Aug 2024 12:18:42 +0300
Subject: [PATCH 142/441] [clang][NFC] Update `cxx_dr_status.html`

---
 clang/www/cxx_dr_status.html | 26 +++++++++++++++++++++++++-
 1 file changed, 25 insertions(+), 1 deletion(-)

diff --git a/clang/www/cxx_dr_status.html b/clang/www/cxx_dr_status.html
index 34ca6f76bb2ce62..c901e1ce0e15b32 100755
--- a/clang/www/cxx_dr_status.html
+++ b/clang/www/cxx_dr_status.html
@@ -7302,7 +7302,7 @@ <h2 id="cxxdr">C++ defect report implementation status</h2>
   <tr class="open" id="1248">
     <td><a href="https://cplusplus.github.io/CWG/issues/1248.html">1248</a></td>
     <td>open</td>
-    <td>Updating Annex C to C99</td>
+    <td>Updating Annex C to C99 and C23</td>
     <td align="center">Not resolved</td>
   </tr>
   <tr id="1249">
@@ -17325,6 +17325,30 @@ <h2 id="cxxdr">C++ defect report implementation status</h2>
     <td>open</td>
     <td>Consideration of constraints for address of overloaded function</td>
     <td align="center">Not resolved</td>
+  </tr>
+  <tr class="open" id="2919">
+    <td><a href="https://cplusplus.github.io/CWG/issues/2919.html">2919</a></td>
+    <td>open</td>
+    <td>Conversion function candidates for initialization of const lvalue reference</td>
+    <td align="center">Not resolved</td>
+  </tr>
+  <tr class="open" id="2920">
+    <td><a href="https://cplusplus.github.io/CWG/issues/2920.html">2920</a></td>
+    <td>open</td>
+    <td>The <TT>template</TT> keyword for base classes</td>
+    <td align="center">Not resolved</td>
+  </tr>
+  <tr class="open" id="2921">
+    <td><a href="https://cplusplus.github.io/CWG/issues/2921.html">2921</a></td>
+    <td>open</td>
+    <td>Exporting redeclarations of entities not attached to a named module</td>
+    <td align="center">Not resolved</td>
+  </tr>
+  <tr class="open" id="2922">
+    <td><a href="https://cplusplus.github.io/CWG/issues/2922.html">2922</a></td>
+    <td>open</td>
+    <td>constexpr placement-new is too permissive</td>
+    <td align="center">Not resolved</td>
   </tr></table>
 
 </div>

From dc3b029dc744730e69abac987c38b2d08b465fba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timm=20B=C3=A4der?= <tbaeder@redhat.com>
Date: Fri, 16 Aug 2024 11:24:47 +0200
Subject: [PATCH 143/441] [clang][Interp][NFC] Remove Function::Loc

Unused.
---
 clang/lib/AST/Interp/Function.cpp | 7 +++----
 clang/lib/AST/Interp/Function.h   | 5 -----
 2 files changed, 3 insertions(+), 9 deletions(-)

diff --git a/clang/lib/AST/Interp/Function.cpp b/clang/lib/AST/Interp/Function.cpp
index 00f5a1fced531a9..fc3345cbe123f88 100644
--- a/clang/lib/AST/Interp/Function.cpp
+++ b/clang/lib/AST/Interp/Function.cpp
@@ -21,10 +21,9 @@ Function::Function(Program &P, const FunctionDecl *F, unsigned ArgSize,
                    llvm::DenseMap<unsigned, ParamDescriptor> &&Params,
                    llvm::SmallVectorImpl<unsigned> &&ParamOffsets,
                    bool HasThisPointer, bool HasRVO, bool UnevaluatedBuiltin)
-    : P(P), Loc(F->getBeginLoc()), F(F), ArgSize(ArgSize),
-      ParamTypes(std::move(ParamTypes)), Params(std::move(Params)),
-      ParamOffsets(std::move(ParamOffsets)), HasThisPointer(HasThisPointer),
-      HasRVO(HasRVO), Variadic(F->isVariadic()),
+    : P(P), F(F), ArgSize(ArgSize), ParamTypes(std::move(ParamTypes)),
+      Params(std::move(Params)), ParamOffsets(std::move(ParamOffsets)),
+      HasThisPointer(HasThisPointer), HasRVO(HasRVO), Variadic(F->isVariadic()),
       IsUnevaluatedBuiltin(UnevaluatedBuiltin) {}
 
 Function::ParamDescriptor Function::getParamDescriptor(unsigned Offset) const {
diff --git a/clang/lib/AST/Interp/Function.h b/clang/lib/AST/Interp/Function.h
index 92bcd96927912db..625d5af0407a721 100644
--- a/clang/lib/AST/Interp/Function.h
+++ b/clang/lib/AST/Interp/Function.h
@@ -100,9 +100,6 @@ class Function final {
     return F->getQualifiedNameAsString();
   }
 
-  /// Returns the location.
-  SourceLocation getLoc() const { return Loc; }
-
   /// Returns a parameter descriptor.
   ParamDescriptor getParamDescriptor(unsigned Offset) const;
 
@@ -235,8 +232,6 @@ class Function final {
 
   /// Program reference.
   Program &P;
-  /// Location of the executed code.
-  SourceLocation Loc;
   /// Declaration this function was compiled from.
   const FunctionDecl *F;
   /// Local area size: storage + metadata.

From e54f683fc5a896cdc167596c98d01db8763220b9 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Fri, 16 Aug 2024 10:29:21 +0100
Subject: [PATCH 144/441] [X86] Add shuffle tests for #104482

---
 .../CodeGen/X86/vector-shuffle-128-v16.ll     | 44 +++++++++++++++++++
 1 file changed, 44 insertions(+)

diff --git a/llvm/test/CodeGen/X86/vector-shuffle-128-v16.ll b/llvm/test/CodeGen/X86/vector-shuffle-128-v16.ll
index ccaaca6c44f050d..fcc50041a1d829d 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-128-v16.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-128-v16.ll
@@ -2771,3 +2771,47 @@ entry:
   %vzip.i = shufflevector <16 x i8> %lane, <16 x i8> %lane3, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23>
   ret <16 x i8> %vzip.i
 }
+
+define <8 x i16> @PR104482(<16 x i8> %i) {
+; SSE2-LABEL: PR104482:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15]
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,1,0,3]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,3,2,3,4,5,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[3,0,1,2]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [65535,65535,65535,65535,65535,0,65535,0]
+; SSE2-NEXT:    pand %xmm3, %xmm1
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,1,0,3]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,3,2,3,4,5,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,3,1,2]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[2,1,0,3,4,5,6,7]
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    packuswb %xmm0, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: PR104482:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[15],zero,xmm0[14],zero,xmm0[13],zero,xmm0[12],zero,xmm0[11],zero,xmm0[10],zero,xmm0[9],zero,xmm0[8],zero
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: PR104482:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[15],zero,xmm0[14],zero,xmm0[13],zero,xmm0[12],zero,xmm0[11],zero,xmm0[10],zero,xmm0[9],zero,xmm0[8],zero
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: PR104482:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[15],zero,xmm0[14],zero,xmm0[13],zero,xmm0[12],zero,xmm0[11],zero,xmm0[10],zero,xmm0[9],zero,xmm0[8],zero
+; AVX-NEXT:    retq
+  %i7 = shufflevector <16 x i8> %i, <16 x i8> <i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>, <16 x i32> <i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>
+  %1 = bitcast <16 x i8> %i7 to <8 x i16>
+  %i10 = shufflevector <8 x i16> %1, <8 x i16> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
+  %i11 = shufflevector <8 x i16> %i10, <8 x i16> poison, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 4, i32 5, i32 6, i32 7>
+  %i12 = shufflevector <8 x i16> %i11, <8 x i16> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 7, i32 6, i32 5, i32 4>
+  ret <8 x i16> %i12
+}

From 39a0dedcbf69adaf9d4ffeebd9172844b38974c7 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Fri, 16 Aug 2024 10:41:50 +0100
Subject: [PATCH 145/441] [X86] lowerShuffleAsDecomposedShuffleMerge - don't
 lower to unpack+permute if either source is zero.

Fixes #104482
---
 llvm/lib/Target/X86/X86ISelLowering.cpp       | 54 ++++++++++---------
 .../CodeGen/X86/vector-shuffle-128-v16.ll     | 24 +++------
 2 files changed, 36 insertions(+), 42 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 11c9a992cbdee92..db50f132b1349f8 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -11156,7 +11156,7 @@ static bool isSingleElementRepeatedMask(ArrayRef<int> Mask) {
 /// blends. For vXi8/vXi16 shuffles we may use unpack instead of blend.
 static SDValue lowerShuffleAsDecomposedShuffleMerge(
     const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
-    const X86Subtarget &Subtarget, SelectionDAG &DAG) {
+    const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG) {
   int NumElts = Mask.size();
   int NumLanes = VT.getSizeInBits() / 128;
   int NumEltsPerLane = NumElts / NumLanes;
@@ -11164,6 +11164,7 @@ static SDValue lowerShuffleAsDecomposedShuffleMerge(
   // Shuffle the input elements into the desired positions in V1 and V2 and
   // unpack/blend them together.
   bool IsAlternating = true;
+  bool V1Zero = true, V2Zero = true;
   SmallVector<int, 32> V1Mask(NumElts, -1);
   SmallVector<int, 32> V2Mask(NumElts, -1);
   SmallVector<int, 32> FinalMask(NumElts, -1);
@@ -11172,10 +11173,12 @@ static SDValue lowerShuffleAsDecomposedShuffleMerge(
     if (M >= 0 && M < NumElts) {
       V1Mask[i] = M;
       FinalMask[i] = i;
+      V1Zero &= Zeroable[i];
       IsAlternating &= (i & 1) == 0;
     } else if (M >= NumElts) {
       V2Mask[i] = M - NumElts;
       FinalMask[i] = i + NumElts;
+      V2Zero &= Zeroable[i];
       IsAlternating &= (i & 1) == 1;
     }
   }
@@ -11228,7 +11231,7 @@ static SDValue lowerShuffleAsDecomposedShuffleMerge(
     // t5: v16i8 = vector_shuffle<16,0,16,1,16,2,16,3,16,4,16,5,16,6,16,7> t2, t4
     // it is better to process t4 first to create a vector of t4[0], then unpack
     // that vector with t2.
-    if (!isSingleElementRepeatedMask(V1Mask) &&
+    if (!V1Zero && !V2Zero && !isSingleElementRepeatedMask(V1Mask) &&
         !isSingleElementRepeatedMask(V2Mask))
       if (SDValue UnpackPerm =
               lowerShuffleAsUNPCKAndPermute(DL, VT, V1, V2, Mask, DAG))
@@ -12955,7 +12958,7 @@ static SDValue lowerV2I64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
   // a permute. That will be faster than the domain cross.
   if (IsBlendSupported)
     return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v2i64, V1, V2, Mask,
-                                                Subtarget, DAG);
+                                                Zeroable, Subtarget, DAG);
 
   // We implement this with SHUFPD which is pretty lame because it will likely
   // incur 2 cycles of stall for integer vectors on Nehalem and older chips.
@@ -13274,7 +13277,7 @@ static SDValue lowerV4I32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
     // a permute. That will be faster than the domain cross.
     if (IsBlendSupported)
       return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v4i32, V1, V2, Mask,
-                                                  Subtarget, DAG);
+                                                  Zeroable, Subtarget, DAG);
 
     // Try to lower by permuting the inputs into an unpack instruction.
     if (SDValue Unpack = lowerShuffleAsPermuteAndUnpack(DL, MVT::v4i32, V1, V2,
@@ -14065,8 +14068,8 @@ static SDValue lowerV8I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
 
   // We can always bit-blend if we have to so the fallback strategy is to
   // decompose into single-input permutes and blends/unpacks.
-  return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v8i16, V1, V2,
-                                              Mask, Subtarget, DAG);
+  return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v8i16, V1, V2, Mask,
+                                              Zeroable, Subtarget, DAG);
 }
 
 /// Lower 8-lane 16-bit floating point shuffles.
@@ -14444,7 +14447,7 @@ static SDValue lowerV16I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
   // Handle multi-input cases by blending/unpacking single-input shuffles.
   if (NumV2Elements > 0)
     return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v16i8, V1, V2, Mask,
-                                                Subtarget, DAG);
+                                                Zeroable, Subtarget, DAG);
 
   // The fallback path for single-input shuffles widens this into two v8i16
   // vectors with unpacks, shuffles those, and then pulls them back together
@@ -14668,6 +14671,7 @@ static SDValue splitAndLowerShuffle(const SDLoc &DL, MVT VT, SDValue V1,
 /// results.
 static SDValue lowerShuffleAsSplitOrBlend(const SDLoc &DL, MVT VT, SDValue V1,
                                           SDValue V2, ArrayRef<int> Mask,
+                                          const APInt &Zeroable,
                                           const X86Subtarget &Subtarget,
                                           SelectionDAG &DAG) {
   assert(!V2.isUndef() && "This routine must not be used to lower single-input "
@@ -14694,8 +14698,8 @@ static SDValue lowerShuffleAsSplitOrBlend(const SDLoc &DL, MVT VT, SDValue V1,
     return true;
   };
   if (DoBothBroadcast())
-    return lowerShuffleAsDecomposedShuffleMerge(DL, VT, V1, V2, Mask, Subtarget,
-                                                DAG);
+    return lowerShuffleAsDecomposedShuffleMerge(DL, VT, V1, V2, Mask, Zeroable,
+                                                Subtarget, DAG);
 
   // If the inputs all stem from a single 128-bit lane of each input, then we
   // split them rather than blending because the split will decompose to
@@ -14714,8 +14718,8 @@ static SDValue lowerShuffleAsSplitOrBlend(const SDLoc &DL, MVT VT, SDValue V1,
 
   // Otherwise, just fall back to decomposed shuffles and a blend/unpack. This
   // requires that the decomposed single-input shuffles don't end up here.
-  return lowerShuffleAsDecomposedShuffleMerge(DL, VT, V1, V2, Mask, Subtarget,
-                                              DAG);
+  return lowerShuffleAsDecomposedShuffleMerge(DL, VT, V1, V2, Mask, Zeroable,
+                                              Subtarget, DAG);
 }
 
 // Lower as SHUFPD(VPERM2F128(V1, V2), VPERM2F128(V1, V2)).
@@ -15907,7 +15911,7 @@ static SDValue lowerV4F64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
   // blend the result.
   if (V1IsInPlace || V2IsInPlace)
     return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v4f64, V1, V2, Mask,
-                                                Subtarget, DAG);
+                                                Zeroable, Subtarget, DAG);
 
   // Try to create an in-lane repeating shuffle mask and then shuffle the
   // results into the target lanes.
@@ -15934,10 +15938,10 @@ static SDValue lowerV4F64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
   // can fully permute the elements.
   if (Subtarget.hasAVX2())
     return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v4f64, V1, V2, Mask,
-                                                Subtarget, DAG);
+                                                Zeroable, Subtarget, DAG);
 
   // Otherwise fall back on generic lowering.
-  return lowerShuffleAsSplitOrBlend(DL, MVT::v4f64, V1, V2, Mask,
+  return lowerShuffleAsSplitOrBlend(DL, MVT::v4f64, V1, V2, Mask, Zeroable,
                                     Subtarget, DAG);
 }
 
@@ -16027,7 +16031,7 @@ static SDValue lowerV4I64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
   // blend the result.
   if (V1IsInPlace || V2IsInPlace)
     return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v4i64, V1, V2, Mask,
-                                                Subtarget, DAG);
+                                                Zeroable, Subtarget, DAG);
 
   // Try to create an in-lane repeating shuffle mask and then shuffle the
   // results into the target lanes.
@@ -16051,7 +16055,7 @@ static SDValue lowerV4I64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
 
   // Otherwise fall back on generic blend lowering.
   return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v4i64, V1, V2, Mask,
-                                              Subtarget, DAG);
+                                              Zeroable, Subtarget, DAG);
 }
 
 /// Handle lowering of 8-lane 32-bit floating point shuffles.
@@ -16162,17 +16166,17 @@ static SDValue lowerV8F32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
   // since after split we get a more efficient code using vpunpcklwd and
   // vpunpckhwd instrs than vblend.
   if (!Subtarget.hasAVX512() && isUnpackWdShuffleMask(Mask, MVT::v8f32, DAG))
-    return lowerShuffleAsSplitOrBlend(DL, MVT::v8f32, V1, V2, Mask, Subtarget,
-                                      DAG);
+    return lowerShuffleAsSplitOrBlend(DL, MVT::v8f32, V1, V2, Mask, Zeroable,
+                                      Subtarget, DAG);
 
   // If we have AVX2 then we always want to lower with a blend because at v8 we
   // can fully permute the elements.
   if (Subtarget.hasAVX2())
     return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v8f32, V1, V2, Mask,
-                                                Subtarget, DAG);
+                                                Zeroable, Subtarget, DAG);
 
   // Otherwise fall back on generic lowering.
-  return lowerShuffleAsSplitOrBlend(DL, MVT::v8f32, V1, V2, Mask,
+  return lowerShuffleAsSplitOrBlend(DL, MVT::v8f32, V1, V2, Mask, Zeroable,
                                     Subtarget, DAG);
 }
 
@@ -16210,8 +16214,8 @@ static SDValue lowerV8I32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
   // vpunpcklwd and vpunpckhwd instrs.
   if (isUnpackWdShuffleMask(Mask, MVT::v8i32, DAG) && !V2.isUndef() &&
       !Subtarget.hasAVX512())
-    return lowerShuffleAsSplitOrBlend(DL, MVT::v8i32, V1, V2, Mask, Subtarget,
-                                      DAG);
+    return lowerShuffleAsSplitOrBlend(DL, MVT::v8i32, V1, V2, Mask, Zeroable,
+                                      Subtarget, DAG);
 
   if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v8i32, V1, V2, Mask,
                                           Zeroable, Subtarget, DAG))
@@ -16315,7 +16319,7 @@ static SDValue lowerV8I32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
 
   // Otherwise fall back on generic blend lowering.
   return lowerShuffleAsDecomposedShuffleMerge(DL, MVT::v8i32, V1, V2, Mask,
-                                              Subtarget, DAG);
+                                              Zeroable, Subtarget, DAG);
 }
 
 /// Handle lowering of 16-lane 16-bit integer shuffles.
@@ -16437,7 +16441,7 @@ static SDValue lowerV16I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
       return V;
 
   // Otherwise fall back on generic lowering.
-  return lowerShuffleAsSplitOrBlend(DL, MVT::v16i16, V1, V2, Mask,
+  return lowerShuffleAsSplitOrBlend(DL, MVT::v16i16, V1, V2, Mask, Zeroable,
                                     Subtarget, DAG);
 }
 
@@ -16558,7 +16562,7 @@ static SDValue lowerV32I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
       return V;
 
   // Otherwise fall back on generic lowering.
-  return lowerShuffleAsSplitOrBlend(DL, MVT::v32i8, V1, V2, Mask,
+  return lowerShuffleAsSplitOrBlend(DL, MVT::v32i8, V1, V2, Mask, Zeroable,
                                     Subtarget, DAG);
 }
 
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-128-v16.ll b/llvm/test/CodeGen/X86/vector-shuffle-128-v16.ll
index fcc50041a1d829d..24c05ba9c52f6f1 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-128-v16.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-128-v16.ll
@@ -2775,23 +2775,13 @@ entry:
 define <8 x i16> @PR104482(<16 x i8> %i) {
 ; SSE2-LABEL: PR104482:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15]
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,1,0,3]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,3,2,3,4,5,6,7]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[3,0,1,2]
-; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [65535,65535,65535,65535,65535,0,65535,0]
-; SSE2-NEXT:    pand %xmm3, %xmm1
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,1,0,3]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[0,3,2,3,4,5,6,7]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,3,1,2]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[2,1,0,3,4,5,6,7]
-; SSE2-NEXT:    pand %xmm3, %xmm0
-; SSE2-NEXT:    packuswb %xmm0, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[3,2,1,0,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,6,5,4]
+; SSE2-NEXT:    packuswb %xmm0, %xmm0
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: PR104482:

From 899a3df02427086f365e1308fd11ca8ab981e9df Mon Sep 17 00:00:00 2001
From: Dmitry Vasilyev <dvassiliev@accesssoftek.com>
Date: Fri, 16 Aug 2024 13:46:45 +0400
Subject: [PATCH 146/441] [lldb][NFC] Moved FindSchemeByProtocol() from
 Acceptor to Socket (#104439)

This is the prerequisite for #104238.
---
 lldb/include/lldb/Host/Socket.h     |  4 +++
 lldb/source/Host/common/Socket.cpp  | 32 +++++++++++++++++++++++
 lldb/tools/lldb-server/Acceptor.cpp | 39 +++--------------------------
 3 files changed, 39 insertions(+), 36 deletions(-)

diff --git a/lldb/include/lldb/Host/Socket.h b/lldb/include/lldb/Host/Socket.h
index 4dad88e78faa112..573c881f727d8f9 100644
--- a/lldb/include/lldb/Host/Socket.h
+++ b/lldb/include/lldb/Host/Socket.h
@@ -60,6 +60,10 @@ class Socket : public IOObject {
 
   ~Socket() override;
 
+  static const char *FindSchemeByProtocol(const SocketProtocol protocol);
+  static bool FindProtocolByScheme(const char *scheme,
+                                   SocketProtocol &protocol);
+
   static llvm::Error Initialize();
   static void Terminate();
 
diff --git a/lldb/source/Host/common/Socket.cpp b/lldb/source/Host/common/Socket.cpp
index f9911cf136cbd09..7364a12280cfdd0 100644
--- a/lldb/source/Host/common/Socket.cpp
+++ b/lldb/source/Host/common/Socket.cpp
@@ -70,6 +70,38 @@ static bool IsInterrupted() {
 #endif
 }
 
+struct SocketScheme {
+  const char *m_scheme;
+  const Socket::SocketProtocol m_protocol;
+};
+
+static SocketScheme socket_schemes[] = {
+    {"tcp", Socket::ProtocolTcp},
+    {"udp", Socket::ProtocolUdp},
+    {"unix", Socket::ProtocolUnixDomain},
+    {"unix-abstract", Socket::ProtocolUnixAbstract},
+};
+
+const char *
+Socket::FindSchemeByProtocol(const Socket::SocketProtocol protocol) {
+  for (auto s : socket_schemes) {
+    if (s.m_protocol == protocol)
+      return s.m_scheme;
+  }
+  return nullptr;
+}
+
+bool Socket::FindProtocolByScheme(const char *scheme,
+                                  Socket::SocketProtocol &protocol) {
+  for (auto s : socket_schemes) {
+    if (!strcmp(s.m_scheme, scheme)) {
+      protocol = s.m_protocol;
+      return true;
+    }
+  }
+  return false;
+}
+
 Socket::Socket(SocketProtocol protocol, bool should_close,
                bool child_processes_inherit)
     : IOObject(eFDTypeSocket), m_protocol(protocol),
diff --git a/lldb/tools/lldb-server/Acceptor.cpp b/lldb/tools/lldb-server/Acceptor.cpp
index 2037f1e0f62b9a7..cf9a55c6c07da56 100644
--- a/lldb/tools/lldb-server/Acceptor.cpp
+++ b/lldb/tools/lldb-server/Acceptor.cpp
@@ -22,40 +22,6 @@ using namespace lldb_private;
 using namespace lldb_private::lldb_server;
 using namespace llvm;
 
-namespace {
-
-struct SocketScheme {
-  const char *m_scheme;
-  const Socket::SocketProtocol m_protocol;
-};
-
-SocketScheme socket_schemes[] = {
-    {"tcp", Socket::ProtocolTcp},
-    {"udp", Socket::ProtocolUdp},
-    {"unix", Socket::ProtocolUnixDomain},
-    {"unix-abstract", Socket::ProtocolUnixAbstract},
-};
-
-bool FindProtocolByScheme(const char *scheme,
-                          Socket::SocketProtocol &protocol) {
-  for (auto s : socket_schemes) {
-    if (!strcmp(s.m_scheme, scheme)) {
-      protocol = s.m_protocol;
-      return true;
-    }
-  }
-  return false;
-}
-
-const char *FindSchemeByProtocol(const Socket::SocketProtocol protocol) {
-  for (auto s : socket_schemes) {
-    if (s.m_protocol == protocol)
-      return s.m_scheme;
-  }
-  return nullptr;
-}
-}
-
 Status Acceptor::Listen(int backlog) {
   return m_listener_socket_up->Listen(StringRef(m_name), backlog);
 }
@@ -74,7 +40,7 @@ Socket::SocketProtocol Acceptor::GetSocketProtocol() const {
 }
 
 const char *Acceptor::GetSocketScheme() const {
-  return FindSchemeByProtocol(GetSocketProtocol());
+  return Socket::FindSchemeByProtocol(GetSocketProtocol());
 }
 
 std::string Acceptor::GetLocalSocketId() const { return m_local_socket_id(); }
@@ -87,7 +53,8 @@ std::unique_ptr<Acceptor> Acceptor::Create(StringRef name,
   Socket::SocketProtocol socket_protocol = Socket::ProtocolUnixDomain;
   // Try to match socket name as URL - e.g., tcp://localhost:5555
   if (std::optional<URI> res = URI::Parse(name)) {
-    if (!FindProtocolByScheme(res->scheme.str().c_str(), socket_protocol))
+    if (!Socket::FindProtocolByScheme(res->scheme.str().c_str(),
+                                      socket_protocol))
       error.SetErrorStringWithFormat("Unknown protocol scheme \"%s\"",
                                      res->scheme.str().c_str());
     else

From cc78639453d91250dddaded0e084e5b81fefff3c Mon Sep 17 00:00:00 2001
From: lancesix <98881381+lancesix@users.noreply.github.com>
Date: Fri, 16 Aug 2024 11:22:19 +0100
Subject: [PATCH 147/441] [AMDGPU][NFC] AMDGPUUsage.rst: document corefile
 format (#104419)

This patch adds a description of the core file format used for AMDGPU.

Reference implementation for creating and loading AMDGPU core dump is
available in
[ROCgdb-6.2](https://github.com/ROCm/ROCgdb/tree/rocm-6.2.x/gdb)
---
 llvm/docs/AMDGPUUsage.rst | 113 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 113 insertions(+)

diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst
index 117fc2cf6bbbc89..97fca32d4ece66b 100644
--- a/llvm/docs/AMDGPUUsage.rst
+++ b/llvm/docs/AMDGPUUsage.rst
@@ -2378,6 +2378,9 @@ are deprecated and should not be used.
      ======== ============================== ======================================
      "AMDGPU" ``NT_AMDGPU_METADATA``         Metadata in Message Pack [MsgPack]_
                                              binary format.
+     "AMDGPU" ``NT_AMDGPU_KFD_CORE_STATE``   Snapshot of runtime, agent and queues
+                                             state for use in core dump.  See
+                                             :ref:`amdgpu_corefile_note`.
      ======== ============================== ======================================
 
 ..
@@ -2390,6 +2393,7 @@ are deprecated and should not be used.
      ============================== =====
      *reserved*                     0-31
      ``NT_AMDGPU_METADATA``         32
+     ``NT_AMDGPU_KFD_CORE_STATE``   33
      ============================== =====
 
 ``NT_AMDGPU_METADATA``
@@ -15024,6 +15028,115 @@ instructions are handled as follows:
                                      trap handler installed.
      =============== =============== ===========================================
 
+Core file format
+================
+
+This section describes the format of core files supporting AMDGPU. Core dumps
+for an AMDGPU program can come in 2 flavors: split or unified core files.
+
+The split layout consists of one host core file containing the information to
+rebuild the image of the host process and one AMDGPU core file that contains
+the information for the AMDGPU agents used in the process.  The AMDGPU core
+file consists of:
+
+* A note describing the state of the AMDGPU agents, AMDGPU queues, and AMDGPU
+  runtime for the process (see :ref:`amdgpu_corefile_note`).
+* A list of load segments containing an image of the AMDGPU agents' memory (see
+  :ref:`amdgpu_corefile_memory`).
+
+The unified core file is the union of all the information contained in
+the two files of the split layout (all notes and load segments).  It contains
+all the information required to reconstruct the image of the process across all
+the agents.
+
+Core file header
+----------------
+
+An AMDGPU core file is an ``ELF64`` core file.  The content of the header
+differs in unified core file layout and AMDGPU core file layout.
+
+Split files
+~~~~~~~~~~~
+
+In the split files layout, the AMDGPU core file is an ``ELF64`` file with the
+header configured as described in :ref:`amdgpu-corefile-headers-table`:
+
+  .. table:: AMDGPU corefile headers
+     :name: amdgpu-corefile-headers-table
+
+     ========================== ===================================
+     Field                      Value
+     ========================== ===================================
+     ``e_ident[EI_CLASS]``      ``ELFCLASS64`` (``0x2``)
+     ``e_ident[EI_DATA]``       ``ELFDATA2LSB`` (``0x1``)
+     ``e_ident[EI_OSABI]``      ``ELFOSABI_AMDGPU_HSA`` (``0x40``)
+     ``e_type``                 ``ET_CORE``(``0x4``)
+     ``e_ident[EI_ABIVERSION]`` ``ELFABIVERSION_AMDGPU_HSA_5``
+     ``e_machine``              ``EM_AMDGPU`` (``0xe0``)
+     ========================== ===================================
+
+Unified file
+~~~~~~~~~~~~
+
+In the unified core file mode, the ``ELF64`` headers are set to describe
+the host architecture and process.
+
+.. _amdgpu_corefile_note:
+
+Core file notes
+---------------
+
+An AMDGPU core file must contain one snapshot note in a ``PT_NOTE`` segment.
+When using a split core file layout, this note is in the AMDGPU file.
+
+The note record vendor field is "``AMDGPU``" and the record type is
+"``NT_AMDGPU_KFD_CORE_STATE``" (see :ref:`amdgpu-note-records-v3-onwards`)
+
+The content of the note is defined in table
+:ref:`amdgpu-core-snapshot-note-layout-table-v1`:
+
+  .. table:: AMDGPU snapshot note format V1
+     :name: amdgpu-core-snapshot-note-layout-table-v1
+
+     ================================ ======================================= ======================= ============== ===========================
+     Field                            Type                                    Size (bytes)            Byte alignment Comment
+     ================================ ======================================= ======================= ============== ===========================
+     ``version_major``                ``uint32``                              4                       4              ``KFD_IOCTL_MAJOR_VERSION``
+     ``version_minor``                ``uint32``                              4                       4              ``KFD_IOCTL_MINOR_VERSION``
+     ``runtime_info_size``            ``uint64``                              8                       8              Must be a multiple of 8
+     ``n_agents``                     ``uint32``                              4                       8
+     ``agent_info_entry_size``        ``uint32``                              4                       4              Must be a multiple of 8
+     ``n_queues``                     ``uint32``                              4                       8
+     ``queue_info_entry_size``        ``uint32``                              4                       4              Must be a multiple of 8
+     ``runtime_info``                 ``kfd_runtime_info``                    ``runtime_info_size``   8
+     ``agents_info``                  ``kfd_dbg_device_info_entry[n_agents]`` ``n_agents *            8
+                                                                              agent_info_entry_size``
+     ``queues_info``                  ``kfd_queue_snapshot_entry[n_queues]``  ``n_queues *
+                                                                              queue_info_entry_size`` 8
+     ================================ ======================================= ======================= ============== ===========================
+
+The definition of all the ``kfd_*`` types comes from the
+``include/uapi/linux/kfd_ioctl.h`` header file from the KFD repository. It is
+usually installed in ``/usr/include/linux/kfd_ioctl.h``. The version of the
+``kfd_ioctl.h`` file used must define values for
+``KFD_IOCTL_MAJOR_VERSION`` and ``KFD_IOCTL_MINOR_VERSION`` matching
+the values of ``kfd_version_major`` and ``kfd_version_major`` from the
+note.
+
+.. _amdgpu_corefile_memory:
+
+Memory segments
+---------------
+
+An AMDGPU core file must contain an image of the AMDGPU agents' memory in load
+segments (of type ``PT_LOAD``).  Those segments must correspond to the memory
+regions where the content of the agent memory is mapped into the host process
+by the ROCr runtime (note that those memory mappings are usually not readable
+by the process itself).
+
+When using the split core file layout, those segments must be included in the
+AMDGPU core file.
+
 Source Languages
 ================
 

From 7d77fb2971e36233f69634380ae55421e5fd70a1 Mon Sep 17 00:00:00 2001
From: Haojian Wu <hokein.wu@gmail.com>
Date: Fri, 16 Aug 2024 12:22:20 +0200
Subject: [PATCH 148/441] [bazel] Port for
 75cb9edf09fdc091e5bc0f3d46a96c2877735a39

---
 utils/bazel/llvm-project-overlay/mlir/BUILD.bazel | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
index 7273429d4f344fa..e697db26d763c97 100644
--- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
@@ -6052,9 +6052,11 @@ cc_library(
     srcs = [
         "lib/Conversion/GPUCommon/GPUOpsLowering.cpp",
         "lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp",
+        "lib/Conversion/GPUCommon/AttrToSPIRVConverter.cpp",
     ],
     hdrs = [
         "include/mlir/Conversion/GPUCommon/GPUCommonPass.h",
+        "include/mlir/Conversion/GPUCommon/AttrToSPIRVConverter.h",
         "lib/Conversion/GPUCommon/GPUOpsLowering.h",
     ],
     includes = ["include"],
@@ -6074,6 +6076,7 @@ cc_library(
         ":LLVMCommonConversion",
         ":LLVMDialect",
         ":MemRefDialect",
+        ":SPIRVDialect",
         ":MemRefToLLVM",
         ":Support",
         ":VectorToLLVM",

From 65390f9d6f0aa5f7bc5125d73337eb658e162d0a Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Fri, 16 Aug 2024 12:36:56 +0200
Subject: [PATCH 149/441] [SimplifyCFG] Add test for #104567 (NFC)

---
 .../SimplifyCFG/sink-and-convert-switch.ll    | 61 +++++++++++++++++++
 1 file changed, 61 insertions(+)
 create mode 100644 llvm/test/Transforms/SimplifyCFG/sink-and-convert-switch.ll

diff --git a/llvm/test/Transforms/SimplifyCFG/sink-and-convert-switch.ll b/llvm/test/Transforms/SimplifyCFG/sink-and-convert-switch.ll
new file mode 100644
index 000000000000000..4944554f4b49ec1
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/sink-and-convert-switch.ll
@@ -0,0 +1,61 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -passes='simplifycfg<sink-common-insts;switch-to-lookup>' < %s | FileCheck %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+
+define void @pr104567(i8 %x, ptr %f) {
+; CHECK-LABEL: define void @pr104567(
+; CHECK-SAME: i8 [[X:%.*]], ptr [[F:%.*]]) {
+; CHECK-NEXT:  [[START:.*:]]
+; CHECK-NEXT:    [[Y:%.*]] = alloca [1 x i8], align 1
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 1, ptr nonnull [[Y]])
+; CHECK-NEXT:    switch i8 [[X]], label %[[DEFAULT_UNREACHABLE:.*]] [
+; CHECK-NEXT:      i8 0, label %[[BB4:.*]]
+; CHECK-NEXT:      i8 1, label %[[BB3:.*]]
+; CHECK-NEXT:      i8 2, label %[[BB2:.*]]
+; CHECK-NEXT:    ]
+; CHECK:       [[DEFAULT_UNREACHABLE]]:
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    store i8 4, ptr [[Y]], align 1
+; CHECK-NEXT:    br label %[[BB5:.*]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    store i8 5, ptr [[Y]], align 1
+; CHECK-NEXT:    br label %[[BB5]]
+; CHECK:       [[BB2]]:
+; CHECK-NEXT:    store i8 6, ptr [[Y]], align 1
+; CHECK-NEXT:    br label %[[BB5]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void [[F]](ptr [[Y]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 1, ptr nonnull [[Y]])
+; CHECK-NEXT:    ret void
+;
+start:
+  %y = alloca [1 x i8], align 1
+  call void @llvm.lifetime.start.p0(i64 1, ptr nonnull %y)
+  switch i8 %x, label %default.unreachable [
+  i8 0, label %bb4
+  i8 1, label %bb3
+  i8 2, label %bb2
+  ]
+
+default.unreachable:
+  unreachable
+
+bb4:
+  store i8 4, ptr %y, align 1
+  br label %bb5
+
+bb3:
+  store i8 5, ptr %y, align 1
+  br label %bb5
+
+bb2:
+  store i8 6, ptr %y, align 1
+  br label %bb5
+
+bb5:
+  call void %f(ptr %y)
+  call void @llvm.lifetime.end.p0(i64 1, ptr nonnull %y)
+  ret void
+}

From 6a84af704f57defd919a4ec2e34b70a48d548719 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Fri, 16 Aug 2024 12:52:57 +0200
Subject: [PATCH 150/441] [LAA] Use computeConstantDifference() (#103725)

Use computeConstantDifference() instead of casting getMinusSCEV() to
SCEVConstant. This can be much faster in some cases, because
computeConstantDifference() computes the result without creating new
SCEV expressions.

This improves LTO/ThinLTO compile-time for lencod by more than 10%.

I've verified that computeConstantDifference() does not produce worse
results than the previous code for anything in llvm-test-suite. This
required raising the iteration cutoff to 6. I ended up increasing it to
8 just to be on the safe side (for code outside llvm-test-suite), and
because this doesn't materially affect compile-time anyway (we'll almost
always bail out earlier).
---
 llvm/lib/Analysis/LoopAccessAnalysis.cpp | 14 ++++++--------
 llvm/lib/Analysis/ScalarEvolution.cpp    |  2 +-
 2 files changed, 7 insertions(+), 9 deletions(-)

diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index d67fd7985f6a505..872bc52b82cca7d 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -412,12 +412,10 @@ bool RuntimePointerChecking::needsChecking(
 /// Return nullptr in case we couldn't find an answer.
 static const SCEV *getMinFromExprs(const SCEV *I, const SCEV *J,
                                    ScalarEvolution *SE) {
-  const SCEV *Diff = SE->getMinusSCEV(J, I);
-  const SCEVConstant *C = dyn_cast<const SCEVConstant>(Diff);
-
-  if (!C)
+  std::optional<APInt> Diff = SE->computeConstantDifference(J, I);
+  if (!Diff)
     return nullptr;
-  return C->getValue()->isNegative() ? J : I;
+  return Diff->isNegative() ? J : I;
 }
 
 bool RuntimeCheckingPtrGroup::addPointer(
@@ -1599,11 +1597,11 @@ std::optional<int> llvm::getPointersDiff(Type *ElemTyA, Value *PtrA,
     // Otherwise compute the distance with SCEV between the base pointers.
     const SCEV *PtrSCEVA = SE.getSCEV(PtrA);
     const SCEV *PtrSCEVB = SE.getSCEV(PtrB);
-    const auto *Diff =
-        dyn_cast<SCEVConstant>(SE.getMinusSCEV(PtrSCEVB, PtrSCEVA));
+    std::optional<APInt> Diff =
+        SE.computeConstantDifference(PtrSCEVB, PtrSCEVA);
     if (!Diff)
       return std::nullopt;
-    Val = Diff->getAPInt().getSExtValue();
+    Val = Diff->getSExtValue();
   }
   int Size = DL.getTypeStoreSize(ElemTyA);
   int Dist = Val / Size;
diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp
index 487844f000ac695..a19358dee8ef491 100644
--- a/llvm/lib/Analysis/ScalarEvolution.cpp
+++ b/llvm/lib/Analysis/ScalarEvolution.cpp
@@ -11956,7 +11956,7 @@ ScalarEvolution::computeConstantDifference(const SCEV *More, const SCEV *Less) {
   APInt DiffMul(BW, 1);
   // Try various simplifications to reduce the difference to a constant. Limit
   // the number of allowed simplifications to keep compile-time low.
-  for (unsigned I = 0; I < 5; ++I) {
+  for (unsigned I = 0; I < 8; ++I) {
     if (More == Less)
       return Diff;
 

From 1db674b83d2bb165d038846cbd0b88120a67535e Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Fri, 16 Aug 2024 15:04:31 +0400
Subject: [PATCH 151/441] InferAddressSpaces: Convert test to generated checks

Also use named values
---
 .../AMDGPU/builtin-assumed-addrspace.ll       | 151 ++++++++++++------
 1 file changed, 98 insertions(+), 53 deletions(-)

diff --git a/llvm/test/Transforms/InferAddressSpaces/AMDGPU/builtin-assumed-addrspace.ll b/llvm/test/Transforms/InferAddressSpaces/AMDGPU/builtin-assumed-addrspace.ll
index 14614ae3dd6cc12..e0c80c03895417d 100644
--- a/llvm/test/Transforms/InferAddressSpaces/AMDGPU/builtin-assumed-addrspace.ll
+++ b/llvm/test/Transforms/InferAddressSpaces/AMDGPU/builtin-assumed-addrspace.ll
@@ -1,84 +1,129 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -passes=infer-address-spaces -o - %s | FileCheck %s
 
-; CHECK-LABEL: @f0
-; CHECK: addrspacecast ptr {{%.*}} to ptr addrspace(3)
-; CHECK: getelementptr inbounds float, ptr addrspace(3)
-; CHECK: load float, ptr addrspace(3)
 define float @f0(ptr %p) {
+; CHECK-LABEL: define float @f0(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[IS_SHARED:%.*]] = call i1 @llvm.amdgcn.is.shared(ptr [[P]])
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[IS_SHARED]])
+; CHECK-NEXT:    [[WORKITEM_ID_X:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT:    [[IDXPROM:%.*]] = zext i32 [[WORKITEM_ID_X]] to i64
+; CHECK-NEXT:    [[TMP0:%.*]] = addrspacecast ptr [[P]] to ptr addrspace(3)
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr addrspace(3) [[TMP0]], i64 [[IDXPROM]]
+; CHECK-NEXT:    [[LOAD:%.*]] = load float, ptr addrspace(3) [[ARRAYIDX]], align 4
+; CHECK-NEXT:    ret float [[LOAD]]
+;
 entry:
-  %0 = call i1 @llvm.amdgcn.is.shared(ptr %p)
-  tail call void @llvm.assume(i1 %0)
-  %1 = tail call i32 @llvm.amdgcn.workitem.id.x()
-  %idxprom = zext i32 %1 to i64
+  %is.shared = call i1 @llvm.amdgcn.is.shared(ptr %p)
+  tail call void @llvm.assume(i1 %is.shared)
+  %workitem.id.x = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %idxprom = zext i32 %workitem.id.x to i64
   %arrayidx = getelementptr inbounds float, ptr %p, i64 %idxprom
-  %2 = load float, ptr %arrayidx, align 4
-  ret float %2
+  %load = load float, ptr %arrayidx, align 4
+  ret float %load
 }
 
-; CHECK-LABEL: @f1
-; CHECK: addrspacecast ptr {{%.*}} to ptr addrspace(5)
-; CHECK: getelementptr inbounds float, ptr addrspace(5)
-; CHECK: load float, ptr addrspace(5)
 define float @f1(ptr %p) {
+; CHECK-LABEL: define float @f1(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[P]])
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[IS_PRIVATE]])
+; CHECK-NEXT:    [[WORKITEM_ID_X:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT:    [[IDXPROM:%.*]] = zext i32 [[WORKITEM_ID_X]] to i64
+; CHECK-NEXT:    [[TMP0:%.*]] = addrspacecast ptr [[P]] to ptr addrspace(5)
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr addrspace(5) [[TMP0]], i64 [[IDXPROM]]
+; CHECK-NEXT:    [[LOAD:%.*]] = load float, ptr addrspace(5) [[ARRAYIDX]], align 4
+; CHECK-NEXT:    ret float [[LOAD]]
+;
 entry:
-  %0 = call i1 @llvm.amdgcn.is.private(ptr %p)
-  tail call void @llvm.assume(i1 %0)
-  %1 = tail call i32 @llvm.amdgcn.workitem.id.x()
-  %idxprom = zext i32 %1 to i64
+  %is.private = call i1 @llvm.amdgcn.is.private(ptr %p)
+  tail call void @llvm.assume(i1 %is.private)
+  %workitem.id.x = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %idxprom = zext i32 %workitem.id.x to i64
   %arrayidx = getelementptr inbounds float, ptr %p, i64 %idxprom
-  %2 = load float, ptr %arrayidx, align 4
-  ret float %2
+  %load = load float, ptr %arrayidx, align 4
+  ret float %load
 }
 
-; CHECK-LABEL: @f2
-; CHECK: addrspacecast ptr {{%.*}} to ptr addrspace(1)
-; CHECK: getelementptr inbounds float, ptr addrspace(1)
-; CHECK: load float, ptr addrspace(1)
 define float @f2(ptr %p) {
+; CHECK-LABEL: define float @f2(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[P]])
+; CHECK-NEXT:    [[NOT_PRIVATE:%.*]] = xor i1 [[IS_PRIVATE]], true
+; CHECK-NEXT:    [[IS_SHARED:%.*]] = call i1 @llvm.amdgcn.is.shared(ptr [[P]])
+; CHECK-NEXT:    [[NOT_SHARED:%.*]] = xor i1 [[IS_SHARED]], true
+; CHECK-NEXT:    [[NOT_PRIVATE_AND_NOT_SHARED:%.*]] = and i1 [[NOT_PRIVATE]], [[NOT_SHARED]]
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[NOT_PRIVATE_AND_NOT_SHARED]])
+; CHECK-NEXT:    [[WORKITEM_ID_X:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT:    [[IDXPROM:%.*]] = zext i32 [[WORKITEM_ID_X]] to i64
+; CHECK-NEXT:    [[TMP0:%.*]] = addrspacecast ptr [[P]] to ptr addrspace(1)
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[TMP0]], i64 [[IDXPROM]]
+; CHECK-NEXT:    [[LOAD:%.*]] = load float, ptr addrspace(1) [[ARRAYIDX]], align 4
+; CHECK-NEXT:    ret float [[LOAD]]
+;
 entry:
-  %0 = call i1 @llvm.amdgcn.is.private(ptr %p)
-  %1 = xor i1 %0, -1
-  %2 = call i1 @llvm.amdgcn.is.shared(ptr %p)
-  %3 = xor i1 %2, -1
-  %4 = and i1 %1, %3
-  tail call void @llvm.assume(i1 %4)
-  %5 = tail call i32 @llvm.amdgcn.workitem.id.x()
-  %idxprom = zext i32 %5 to i64
+  %is.private = call i1 @llvm.amdgcn.is.private(ptr %p)
+  %not.private = xor i1 %is.private, true
+  %is.shared = call i1 @llvm.amdgcn.is.shared(ptr %p)
+  %not.shared = xor i1 %is.shared, true
+  %not.private.and.not.shared = and i1 %not.private, %not.shared
+  tail call void @llvm.assume(i1 %not.private.and.not.shared)
+  %workitem.id.x = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %idxprom = zext i32 %workitem.id.x to i64
   %arrayidx = getelementptr inbounds float, ptr %p, i64 %idxprom
-  %6 = load float, ptr %arrayidx, align 4
-  ret float %6
+  %load = load float, ptr %arrayidx, align 4
+  ret float %load
 }
 
-; CHECK-LABEL: @g0
-; CHECK: if.then:
-; CHECK: addrspacecast ptr {{%.*}} to ptr addrspace(3)
-; CHECK: getelementptr inbounds float, ptr addrspace(3)
-; CHECK: load float, ptr addrspace(3)
-; CHECK: if.end:
-; CHECK: getelementptr inbounds float, ptr
-; CHECK: load float, ptr
 define float @g0(i32 %c, ptr %p) {
+; CHECK-LABEL: define float @g0(
+; CHECK-SAME: i32 [[C:%.*]], ptr [[P:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[TOBOOL_NOT:%.*]] = icmp eq i32 [[C]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL_NOT]], label %[[IF_END:.*]], label %[[IF_THEN:.*]]
+; CHECK:       [[IF_THEN]]:
+; CHECK-NEXT:    [[IS_SHARED:%.*]] = call i1 @llvm.amdgcn.is.shared(ptr [[P]])
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[IS_SHARED]])
+; CHECK-NEXT:    [[WORKITEM_ID_X_0:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT:    [[IDXPROM:%.*]] = zext i32 [[WORKITEM_ID_X_0]] to i64
+; CHECK-NEXT:    [[TMP0:%.*]] = addrspacecast ptr [[P]] to ptr addrspace(3)
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr addrspace(3) [[TMP0]], i64 [[IDXPROM]]
+; CHECK-NEXT:    [[LOAD0:%.*]] = load float, ptr addrspace(3) [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[ADD:%.*]] = fadd float [[LOAD0]], 0.000000e+00
+; CHECK-NEXT:    br label %[[IF_END]]
+; CHECK:       [[IF_END]]:
+; CHECK-NEXT:    [[S:%.*]] = phi float [ [[ADD]], %[[IF_THEN]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; CHECK-NEXT:    [[WORKITEM_ID_X_1:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.y()
+; CHECK-NEXT:    [[IDXPROM2:%.*]] = zext i32 [[WORKITEM_ID_X_1]] to i64
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[P]], i64 [[IDXPROM2]]
+; CHECK-NEXT:    [[LOAD1:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[ADD2:%.*]] = fadd float [[S]], [[LOAD1]]
+; CHECK-NEXT:    ret float [[ADD2]]
+;
 entry:
   %tobool.not = icmp eq i32 %c, 0
   br i1 %tobool.not, label %if.end, label %if.then
 
 if.then:
-  %0 = call i1 @llvm.amdgcn.is.shared(ptr %p)
-  tail call void @llvm.assume(i1 %0)
-  %1 = tail call i32 @llvm.amdgcn.workitem.id.x()
-  %idxprom = zext i32 %1 to i64
+  %is.shared = call i1 @llvm.amdgcn.is.shared(ptr %p)
+  tail call void @llvm.assume(i1 %is.shared)
+  %workitem.id.x.0 = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %idxprom = zext i32 %workitem.id.x.0 to i64
   %arrayidx = getelementptr inbounds float, ptr %p, i64 %idxprom
-  %2 = load float, ptr %arrayidx, align 4
-  %add = fadd float %2, 0.
+  %load0 = load float, ptr %arrayidx, align 4
+  %add = fadd float %load0, 0.0
   br label %if.end
 
 if.end:
-  %s = phi float [ %add, %if.then ], [ 0., %entry ]
-  %3 = tail call i32 @llvm.amdgcn.workitem.id.y()
-  %idxprom2 = zext i32 %3 to i64
+  %s = phi float [ %add, %if.then ], [ 0.0, %entry ]
+  %workitem.id.x.1 = tail call i32 @llvm.amdgcn.workitem.id.y()
+  %idxprom2 = zext i32 %workitem.id.x.1 to i64
   %arrayidx2 = getelementptr inbounds float, ptr %p, i64 %idxprom2
-  %4 = load float, ptr %arrayidx2, align 4
-  %add2 = fadd float %s, %4
+  %load1 = load float, ptr %arrayidx2, align 4
+  %add2 = fadd float %s, %load1
   ret float %add2
 }
 

From a426ffdee1ca7814f2684b6104c48f5669815aad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?kadir=20=C3=A7etinkaya?= <kadircet@google.com>
Date: Fri, 16 Aug 2024 13:17:58 +0200
Subject: [PATCH 152/441] [include-cleaner] Add handling for new/delete
 expressions (#104033)

---
 clang-tools-extra/include-cleaner/lib/WalkAST.cpp      | 10 +++++++++-
 .../include-cleaner/unittests/WalkASTTest.cpp          |  7 +++++++
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/clang-tools-extra/include-cleaner/lib/WalkAST.cpp b/clang-tools-extra/include-cleaner/lib/WalkAST.cpp
index b15d428326ac12d..a5ac3760a3be2aa 100644
--- a/clang-tools-extra/include-cleaner/lib/WalkAST.cpp
+++ b/clang-tools-extra/include-cleaner/lib/WalkAST.cpp
@@ -23,7 +23,6 @@
 #include "clang/Basic/IdentifierTable.h"
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Basic/Specifiers.h"
-#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/STLFunctionalExtras.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/Casting.h"
@@ -350,6 +349,15 @@ class ASTWalker : public RecursiveASTVisitor<ASTWalker> {
            RefType::Implicit);
     return true;
   }
+
+  bool VisitCXXNewExpr(CXXNewExpr *E) {
+    report(E->getExprLoc(), E->getOperatorNew());
+    return true;
+  }
+  bool VisitCXXDeleteExpr(CXXDeleteExpr *E) {
+    report(E->getExprLoc(), E->getOperatorDelete());
+    return true;
+  }
 };
 
 } // namespace
diff --git a/clang-tools-extra/include-cleaner/unittests/WalkASTTest.cpp b/clang-tools-extra/include-cleaner/unittests/WalkASTTest.cpp
index 282abce3246ca3d..79371f5978fc334 100644
--- a/clang-tools-extra/include-cleaner/unittests/WalkASTTest.cpp
+++ b/clang-tools-extra/include-cleaner/unittests/WalkASTTest.cpp
@@ -555,5 +555,12 @@ TEST(WalkAST, FriendDecl) {
   testWalk("void $explicit^foo();", "struct Bar { friend void ^foo(); };");
   testWalk("struct $explicit^Foo {};", "struct Bar { friend struct ^Foo; };");
 }
+
+TEST(WalkAST, OperatorNewDelete) {
+  testWalk("void* $explicit^operator new(unsigned long, void*);",
+           "struct Bar { void foo() { Bar b; ^new (&b) Bar; } };");
+  testWalk("struct A { static void $explicit^operator delete(void*); };",
+           "void foo() { A a; ^delete &a; }");
+}
 } // namespace
 } // namespace clang::include_cleaner

From a84790e5fca0429683b24f7bb52d2c4d947dc011 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Fri, 16 Aug 2024 12:23:39 +0100
Subject: [PATCH 153/441] [X86] SimplifyDemandedVectorEltsForTargetNode -
 reduce width of X86 conversions nodes when upper elements are not demanded.
 (#102882)

Fixes #83402
---
 llvm/lib/Target/X86/X86ISelLowering.cpp       | 20 +++++++++++++++++++
 .../CodeGen/X86/vector-half-conversions.ll    | 19 ++++++++----------
 2 files changed, 28 insertions(+), 11 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index db50f132b1349f8..c6e1764edffc3a7 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -42524,6 +42524,26 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode(
       SDValue Insert =
           insertSubVector(UndefVec, ExtOp, 0, TLO.DAG, DL, ExtSizeInBits);
       return TLO.CombineTo(Op, Insert);
+    }
+      // Conversions.
+      // TODO: Add more CVT opcodes when we have test coverage.
+    case X86ISD::CVTTP2SI:
+    case X86ISD::CVTTP2UI:
+    case X86ISD::CVTPH2PS: {
+      SDLoc DL(Op);
+      unsigned Scale = SizeInBits / ExtSizeInBits;
+      SDValue SrcOp = Op.getOperand(0);
+      MVT SrcVT = SrcOp.getSimpleValueType();
+      unsigned SrcExtSize =
+          std::max<unsigned>(SrcVT.getSizeInBits() / Scale, 128);
+      MVT ExtVT = MVT::getVectorVT(VT.getSimpleVT().getScalarType(),
+                                   ExtSizeInBits / VT.getScalarSizeInBits());
+      SDValue ExtOp = TLO.DAG.getNode(
+          Opc, DL, ExtVT, extractSubVector(SrcOp, 0, TLO.DAG, DL, SrcExtSize));
+      SDValue UndefVec = TLO.DAG.getUNDEF(VT);
+      SDValue Insert =
+          insertSubVector(UndefVec, ExtOp, 0, TLO.DAG, DL, ExtSizeInBits);
+      return TLO.CombineTo(Op, Insert);
     }
       // Zero upper elements.
     case X86ISD::VZEXT_MOVL:
diff --git a/llvm/test/CodeGen/X86/vector-half-conversions.ll b/llvm/test/CodeGen/X86/vector-half-conversions.ll
index e87814ebb1dbe48..ef0f3f3e816dfa3 100644
--- a/llvm/test/CodeGen/X86/vector-half-conversions.ll
+++ b/llvm/test/CodeGen/X86/vector-half-conversions.ll
@@ -4990,6 +4990,7 @@ define <4 x i32> @fptosi_2f16_to_4i32(<2 x half> %a) nounwind {
   ret <4 x i32> %ext
 }
 
+; PR83402
 define <4 x i32> @fptosi_4f16_to_4i32(<4 x half> %a) nounwind {
 ; AVX-LABEL: fptosi_4f16_to_4i32:
 ; AVX:       # %bb.0:
@@ -5024,16 +5025,14 @@ define <4 x i32> @fptosi_4f16_to_4i32(<4 x half> %a) nounwind {
 ;
 ; F16C-LABEL: fptosi_4f16_to_4i32:
 ; F16C:       # %bb.0:
-; F16C-NEXT:    vcvtph2ps %xmm0, %ymm0
+; F16C-NEXT:    vcvtph2ps %xmm0, %xmm0
 ; F16C-NEXT:    vcvttps2dq %xmm0, %xmm0
-; F16C-NEXT:    vzeroupper
 ; F16C-NEXT:    retq
 ;
 ; AVX512-LABEL: fptosi_4f16_to_4i32:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vcvtph2ps %xmm0, %ymm0
+; AVX512-NEXT:    vcvtph2ps %xmm0, %xmm0
 ; AVX512-NEXT:    vcvttps2dq %xmm0, %xmm0
-; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
   %cvt = fptosi <4 x half> %a to <4 x i32>
   ret <4 x i32> %cvt
@@ -5213,13 +5212,12 @@ define <4 x i32> @fptoui_4f16_to_4i32(<4 x half> %a) nounwind {
 ;
 ; F16C-LABEL: fptoui_4f16_to_4i32:
 ; F16C:       # %bb.0:
-; F16C-NEXT:    vcvtph2ps %xmm0, %ymm0
-; F16C-NEXT:    vcvttps2dq %ymm0, %ymm1
-; F16C-NEXT:    vsubps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
-; F16C-NEXT:    vcvttps2dq %ymm0, %ymm0
+; F16C-NEXT:    vcvtph2ps %xmm0, %xmm0
+; F16C-NEXT:    vcvttps2dq %xmm0, %xmm1
+; F16C-NEXT:    vsubps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; F16C-NEXT:    vcvttps2dq %xmm0, %xmm0
 ; F16C-NEXT:    vorps %xmm0, %xmm1, %xmm0
 ; F16C-NEXT:    vblendvps %xmm1, %xmm0, %xmm1, %xmm0
-; F16C-NEXT:    vzeroupper
 ; F16C-NEXT:    retq
 ;
 ; AVX512F-LABEL: fptoui_4f16_to_4i32:
@@ -5232,9 +5230,8 @@ define <4 x i32> @fptoui_4f16_to_4i32(<4 x half> %a) nounwind {
 ;
 ; AVX512-FASTLANE-LABEL: fptoui_4f16_to_4i32:
 ; AVX512-FASTLANE:       # %bb.0:
-; AVX512-FASTLANE-NEXT:    vcvtph2ps %xmm0, %ymm0
+; AVX512-FASTLANE-NEXT:    vcvtph2ps %xmm0, %xmm0
 ; AVX512-FASTLANE-NEXT:    vcvttps2udq %xmm0, %xmm0
-; AVX512-FASTLANE-NEXT:    vzeroupper
 ; AVX512-FASTLANE-NEXT:    retq
   %cvt = fptoui <4 x half> %a to <4 x i32>
   ret <4 x i32> %cvt

From d867988cd9b0f8ea31e7dd9f995e341c9d5d8157 Mon Sep 17 00:00:00 2001
From: Sergei Barannikov <barannikov88@gmail.com>
Date: Fri, 16 Aug 2024 14:51:35 +0300
Subject: [PATCH 154/441] [DataLayout] Refactor parsing of "ni" specification
 (#104546)

Split off of #104545 to reduce patch size.
This introduces `parseAddrSpace` function, intended as a replacement for
`getAddrSpace`, which doesn't check for trailing characters after the
address space number. `getAddrSpace` will be removed after switching all
uses to `parseAddrSpace`.

Pull Request: https://github.com/llvm/llvm-project/pull/104546
---
 llvm/lib/IR/DataLayout.cpp           | 52 +++++++++++++++++++---------
 llvm/unittests/IR/DataLayoutTest.cpp | 41 ++++++++++++++++++++++
 2 files changed, 77 insertions(+), 16 deletions(-)

diff --git a/llvm/lib/IR/DataLayout.cpp b/llvm/lib/IR/DataLayout.cpp
index 24c29458abfa819..7043fa410192a35 100644
--- a/llvm/lib/IR/DataLayout.cpp
+++ b/llvm/lib/IR/DataLayout.cpp
@@ -272,6 +272,22 @@ static Error reportError(const Twine &Message) {
   return createStringError(inconvertibleErrorCode(), Message);
 }
 
+static Error createSpecFormatError(Twine Format) {
+  return createStringError("malformed specification, must be of the form \"" +
+                           Format + "\"");
+}
+
+/// Attempts to parse an address space component of a specification.
+static Error parseAddrSpace(StringRef Str, unsigned &AddrSpace) {
+  if (Str.empty())
+    return createStringError("address space component cannot be empty");
+
+  if (!to_integer(Str, AddrSpace, 10) || !isUInt<24>(AddrSpace))
+    return createStringError("address space must be a 24-bit integer");
+
+  return Error::success();
+}
+
 /// Checked version of split, to ensure mandatory subparts.
 static Error split(StringRef Str, char Separator,
                    std::pair<StringRef, StringRef> &Split) {
@@ -313,6 +329,26 @@ static Error getAddrSpace(StringRef R, unsigned &AddrSpace) {
 }
 
 Error DataLayout::parseSpecification(StringRef Spec) {
+  // The "ni" specifier is the only two-character specifier. Handle it first.
+  if (Spec.starts_with("ni")) {
+    // ni:<address space>[:<address space>]...
+    StringRef Rest = Spec.drop_front(2);
+
+    // Drop the first ':', then split the rest of the string the usual way.
+    if (!Rest.consume_front(":"))
+      return createSpecFormatError("ni:<address space>[:<address space>]...");
+
+    for (StringRef Str : split(Rest, ':')) {
+      unsigned AddrSpace;
+      if (Error Err = parseAddrSpace(Str, AddrSpace))
+        return Err;
+      if (AddrSpace == 0)
+        return createStringError("address space 0 cannot be non-integral");
+      NonIntegralAddressSpaces.push_back(AddrSpace);
+    }
+    return Error::success();
+  }
+
   // Split at ':'.
   std::pair<StringRef, StringRef> Split;
   if (Error Err = ::split(Spec, ':', Split))
@@ -322,22 +358,6 @@ Error DataLayout::parseSpecification(StringRef Spec) {
   StringRef &Tok = Split.first;   // Current token.
   StringRef &Rest = Split.second; // The rest of the string.
 
-  if (Tok == "ni") {
-    do {
-      if (Error Err = ::split(Rest, ':', Split))
-        return Err;
-      Rest = Split.second;
-      unsigned AS;
-      if (Error Err = getInt(Split.first, AS))
-        return Err;
-      if (AS == 0)
-        return reportError("Address space 0 can never be non-integral");
-      NonIntegralAddressSpaces.push_back(AS);
-    } while (!Rest.empty());
-
-    return Error::success();
-  }
-
   char SpecifierChar = Tok.front();
   Tok = Tok.substr(1);
 
diff --git a/llvm/unittests/IR/DataLayoutTest.cpp b/llvm/unittests/IR/DataLayoutTest.cpp
index f5c930ebdbb9c46..a40ac5b83a83e40 100644
--- a/llvm/unittests/IR/DataLayoutTest.cpp
+++ b/llvm/unittests/IR/DataLayoutTest.cpp
@@ -134,6 +134,47 @@ TEST(DataLayout, LayoutStringFormat) {
         FailedWithMessage("empty specification is not allowed"));
 }
 
+TEST(DataLayout, ParseNonIntegralAddrSpace) {
+  for (StringRef Str : {"ni:1", "ni:16777215", "ni:1:16777215"})
+    EXPECT_THAT_EXPECTED(DataLayout::parse(Str), Succeeded());
+
+  for (StringRef Str : {"ni", "ni42", "nix"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(Str),
+        FailedWithMessage("malformed specification, must be of the form "
+                          "\"ni:<address space>[:<address space>]...\""));
+
+  for (StringRef Str : {"ni:", "ni::42", "ni:42:"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(Str),
+        FailedWithMessage("address space component cannot be empty"));
+
+  for (StringRef Str : {"ni:x", "ni:42:0x1", "ni:16777216", "ni:42:16777216"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(Str),
+        FailedWithMessage("address space must be a 24-bit integer"));
+
+  for (StringRef Str : {"ni:0", "ni:42:0"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(Str),
+        FailedWithMessage("address space 0 cannot be non-integral"));
+}
+
+TEST(DataLayout, IsNonIntegralAddressSpace) {
+  DataLayout Default;
+  EXPECT_THAT(Default.getNonIntegralAddressSpaces(), ::testing::SizeIs(0));
+  EXPECT_FALSE(Default.isNonIntegralAddressSpace(0));
+  EXPECT_FALSE(Default.isNonIntegralAddressSpace(1));
+
+  DataLayout Custom = cantFail(DataLayout::parse("ni:2:16777215"));
+  EXPECT_THAT(Custom.getNonIntegralAddressSpaces(),
+              ::testing::ElementsAreArray({2U, 16777215U}));
+  EXPECT_FALSE(Custom.isNonIntegralAddressSpace(0));
+  EXPECT_FALSE(Custom.isNonIntegralAddressSpace(1));
+  EXPECT_TRUE(Custom.isNonIntegralAddressSpace(2));
+  EXPECT_TRUE(Custom.isNonIntegralAddressSpace(16777215));
+}
+
 TEST(DataLayoutTest, CopyAssignmentInvalidatesStructLayout) {
   DataLayout DL1 = cantFail(DataLayout::parse("p:32:32"));
   DataLayout DL2 = cantFail(DataLayout::parse("p:64:64"));

From 8aa9d6206ce55bdaaf422839c351fbd63f033b89 Mon Sep 17 00:00:00 2001
From: Alan Wu <XrXr@users.noreply.github.com>
Date: Fri, 16 Aug 2024 08:16:17 -0400
Subject: [PATCH 155/441] [compiler-rt] Stop using x86 builtin on AArch64 with
 GCC (#93890)

Previously, building `multc3.c` on A64 with GCC 7 or up but 9 and lower
will attempt to reference `__builtin_copysignq`, an [x86-specific
intrinsic][1]:

```
$ gcc -c multc3.c
In file included from fp_lib.h:24,
                 from multc3.c:14:
multc3.c: In function '__multc3':
int_math.h:71:32: warning: implicit declaration of function '__builtin_copysignq'; did you mean '__builtin_copysign'? [-Wimplicit-function-declaration]
 #define crt_copysignf128(x, y) __builtin_copysignq((x), (y))
                                ^~~~~~~~~~~~~~~~~~~
```

This is because `__has_builtin` is from GCC 10, and defined to 0 at the
top of int_math.h for affected GCC versions, so the fallback definition
is used. But `__builtin_copysignq` is unavailable on A64.

Use version detection to find `__builtin_copysignf128` instead. It's
available since GCC 7 and [available][2] on both x86 and A64, given this
macro is only used when `CRT_HAS_IEEE_TF`.

---

I realize this is fixing a problem for an out-of-tree build
configuration, but help would be greatly appreciated. Rust
[builds](https://github.com/rust-lang/compiler-builtins) `multc3.c` with
GCC 8 and this mis-selection is causing [build
issues](https://github.com/rust-lang/rust/issues/125619) way downstream.

ref: d2ce3e9621411f3391def327f89e3a650918989f

[1]: https://gcc.gnu.org/onlinedocs/gcc/x86-Built-in-Functions.html
[2]: https://gcc.gnu.org/gcc-7/changes.html
---
 compiler-rt/lib/builtins/int_math.h | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/compiler-rt/lib/builtins/int_math.h b/compiler-rt/lib/builtins/int_math.h
index 74d3e311db5e710..08bfe922ffa13b4 100644
--- a/compiler-rt/lib/builtins/int_math.h
+++ b/compiler-rt/lib/builtins/int_math.h
@@ -65,9 +65,12 @@
 #define crt_copysign(x, y) __builtin_copysign((x), (y))
 #define crt_copysignf(x, y) __builtin_copysignf((x), (y))
 #define crt_copysignl(x, y) __builtin_copysignl((x), (y))
-#if __has_builtin(__builtin_copysignf128)
+// We define __has_builtin to always return 0 for GCC versions below 10,
+// but __builtin_copysignf128 is available since version 7.
+#if __has_builtin(__builtin_copysignf128) ||                                   \
+    (defined(__GNUC__) && __GNUC__ >= 7)
 #define crt_copysignf128(x, y) __builtin_copysignf128((x), (y))
-#elif __has_builtin(__builtin_copysignq) || (defined(__GNUC__) && __GNUC__ >= 7)
+#elif __has_builtin(__builtin_copysignq)
 #define crt_copysignf128(x, y) __builtin_copysignq((x), (y))
 #endif
 #endif
@@ -80,9 +83,11 @@
 #define crt_fabs(x) __builtin_fabs((x))
 #define crt_fabsf(x) __builtin_fabsf((x))
 #define crt_fabsl(x) __builtin_fabsl((x))
-#if __has_builtin(__builtin_fabsf128)
+// We define __has_builtin to always return 0 for GCC versions below 10,
+// but __builtin_fabsf128 is available since version 7.
+#if __has_builtin(__builtin_fabsf128) || (defined(__GNUC__) && __GNUC__ >= 7)
 #define crt_fabsf128(x) __builtin_fabsf128((x))
-#elif __has_builtin(__builtin_fabsq) || (defined(__GNUC__) && __GNUC__ >= 7)
+#elif __has_builtin(__builtin_fabsq)
 #define crt_fabsf128(x) __builtin_fabsq((x))
 #endif
 #endif

From 9f430de4b2577fb0a0169b6409a6267db927c08d Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell@arm.com>
Date: Fri, 16 Aug 2024 12:26:00 +0000
Subject: [PATCH 156/441] [mlir][ArmSME][docs] Fix broken link (NFC)

---
 mlir/docs/Dialects/ArmSME.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/mlir/docs/Dialects/ArmSME.md b/mlir/docs/Dialects/ArmSME.md
index ce0a76ed60f19cf..d38a3e581f48d13 100644
--- a/mlir/docs/Dialects/ArmSME.md
+++ b/mlir/docs/Dialects/ArmSME.md
@@ -12,14 +12,14 @@ operation to Arm SME
 integration tests for reference:
 
 * [Linalg/CPU/ArmSME/matmul.mlir](https://github.com/llvm/llvm-project/blob/main/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/matmul.mlir)
-* [Vector/CPU/ArmSME/test-outerproduct-f64.mlir](https://github.com/llvm/llvm-project/blob/main/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/test-outerproduct-f64.mlir)
+* [Vector/CPU/ArmSME/outerproduct-f64.mlir](https://github.com/llvm/llvm-project/blob/main/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/outerproduct-f64.mlir)
 
 In order to run ArmSME integration tests, include these flags in the CMake
 invocation when configuring LLVM and MLIR:
 ```bash
   -DMLIR_INCLUDE_INTEGRATION_TESTS=On
   -DMLIR_RUN_ARM_SME_TESTS=On
-  -DARM_EMULATOR_EXECUTABLE=<path-to-emulator> 
+  -DARM_EMULATOR_EXECUTABLE=<path-to-emulator>
 ```
 
 These tests are run "post-commit" by the

From 7417b4cd138f7384bba83f7eb62a890303f82600 Mon Sep 17 00:00:00 2001
From: Vlad Serebrennikov <serebrennikov.vladislav@gmail.com>
Date: Fri, 16 Aug 2024 16:35:34 +0400
Subject: [PATCH 157/441] [clang] Add a new test for CWG2091 (#104573)

This patch adds a test from
https://github.com/llvm/llvm-project/issues/42233, and updates CWG2091
status from `2.7` to `10`.
---
 clang/test/CXX/drs/cwg20xx.cpp | 25 ++++++++++++++++++++++++-
 clang/www/cxx_dr_status.html   |  2 +-
 2 files changed, 25 insertions(+), 2 deletions(-)

diff --git a/clang/test/CXX/drs/cwg20xx.cpp b/clang/test/CXX/drs/cwg20xx.cpp
index cd2f26360d40215..b2dc5a9865e382c 100644
--- a/clang/test/CXX/drs/cwg20xx.cpp
+++ b/clang/test/CXX/drs/cwg20xx.cpp
@@ -401,11 +401,34 @@ namespace cwg2083 { // cwg2083: partial
 #endif
 }
 
-namespace cwg2091 { // cwg2091: 2.7
+namespace cwg2091 { // cwg2091: 10
 template<int &> struct X;
 template<int &N> void f(X<N>&);
 int n;
 void g(X<n> &x) { f(x); }
+
+namespace GH42233 {
+enum E { I };
+
+class AA { };
+E EV[1] = {I};
+
+template<class ENUM, const ENUM* const VALUES>
+struct S
+{
+  template< class E, const E* const V>
+    friend AA& operator<<( AA& os, const S<E,V>& e );
+};
+
+int f()
+{
+  S< E, EV > x;
+
+  AA a;
+  a << x;
+  return 0;
+}
+} // namespace GH42233
 } // namespace cwg2091 
 
 namespace cwg2094 { // cwg2094: 5
diff --git a/clang/www/cxx_dr_status.html b/clang/www/cxx_dr_status.html
index c901e1ce0e15b32..236574312338600 100755
--- a/clang/www/cxx_dr_status.html
+++ b/clang/www/cxx_dr_status.html
@@ -12361,7 +12361,7 @@ <h2 id="cxxdr">C++ defect report implementation status</h2>
     <td><a href="https://cplusplus.github.io/CWG/issues/2091.html">2091</a></td>
     <td>CD4</td>
     <td>Deducing reference non-type template arguments</td>
-    <td class="full" align="center">Clang 2.7</td>
+    <td class="full" align="center">Clang 10</td>
   </tr>
   <tr id="2092">
     <td><a href="https://cplusplus.github.io/CWG/issues/2092.html">2092</a></td>

From 1cdf1e693425ae6830883d2fc0eaaa601a8250d8 Mon Sep 17 00:00:00 2001
From: Timm Baeder <tbaeder@redhat.com>
Date: Fri, 16 Aug 2024 14:37:45 +0200
Subject: [PATCH 158/441] [clang][Interp] Fix classifying enum types (#104582)

We used to always return PT_IntAP(s) for them.
---
 clang/lib/AST/Interp/Context.cpp | 3 +++
 clang/test/AST/Interp/enums.cpp  | 5 +++++
 2 files changed, 8 insertions(+)

diff --git a/clang/lib/AST/Interp/Context.cpp b/clang/lib/AST/Interp/Context.cpp
index 92ac28137fdb45d..e9c1fd1b8dc9f96 100644
--- a/clang/lib/AST/Interp/Context.cpp
+++ b/clang/lib/AST/Interp/Context.cpp
@@ -135,6 +135,9 @@ std::optional<PrimType> Context::classify(QualType T) const {
   if (T->isAnyComplexType() || T->isVectorType())
     return std::nullopt;
 
+  if (const auto *ET = T->getAs<EnumType>())
+    return classify(ET->getDecl()->getIntegerType());
+
   if (T->isSignedIntegerOrEnumerationType()) {
     switch (Ctx.getIntWidth(T)) {
     case 64:
diff --git a/clang/test/AST/Interp/enums.cpp b/clang/test/AST/Interp/enums.cpp
index c4db7875451677e..459cf3a0c83d540 100644
--- a/clang/test/AST/Interp/enums.cpp
+++ b/clang/test/AST/Interp/enums.cpp
@@ -48,3 +48,8 @@ constexpr EC getB() {
 
 
 static_assert(getB() == EC::B, "");
+
+namespace B {
+  enum E : bool { Zero, One };
+  static_assert((int)(E)2 == 1, "");
+} // namespace B

From 7e23a23d5e4fa367842546af9f92dcdef869ef9a Mon Sep 17 00:00:00 2001
From: Volodymyr Vasylkun <vvmposeydon@gmail.com>
Date: Fri, 16 Aug 2024 13:38:13 +0100
Subject: [PATCH 159/441] [InstCombine] Fold an unsigned icmp of ucmp/scmp with
 a constant to an icmp of the original arguments (#104471)

Proofs: https://alive2.llvm.org/ce/z/9mv8HU
---
 .../InstCombine/InstCombineCompares.cpp       | 10 ++++++++
 llvm/test/Transforms/InstCombine/scmp.ll      | 25 +++++++++++++++++++
 llvm/test/Transforms/InstCombine/ucmp.ll      | 24 ++++++++++++++++++
 3 files changed, 59 insertions(+)

diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
index 10488ecb747a484..34c9e0fde4f4285 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
@@ -4018,6 +4018,16 @@ foldICmpOfCmpIntrinsicWithConstant(ICmpInst::Predicate Pred, IntrinsicInst *I,
       NewPredicate = ICmpInst::ICMP_ULE;
     break;
 
+  case ICmpInst::ICMP_ULT:
+    if (C.ugt(1))
+      NewPredicate = ICmpInst::ICMP_UGE;
+    break;
+
+  case ICmpInst::ICMP_UGT:
+    if (!C.isZero() && !C.isAllOnes())
+      NewPredicate = ICmpInst::ICMP_ULT;
+    break;
+
   default:
     break;
   }
diff --git a/llvm/test/Transforms/InstCombine/scmp.ll b/llvm/test/Transforms/InstCombine/scmp.ll
index 2523872562cad98..460ea96b16fcfa9 100644
--- a/llvm/test/Transforms/InstCombine/scmp.ll
+++ b/llvm/test/Transforms/InstCombine/scmp.ll
@@ -157,6 +157,31 @@ define i1 @scmp_sle_neg_1(i32 %x, i32 %y) {
   ret i1 %2
 }
 
+; scmp(x, y) u< C => x s>= y when C u> 1 and C != -1
+define i1 @scmp_ult_positive_const_gt_than_1_lt_than_umax(i32 %x, i32 %y) {
+; CHECK-LABEL: define i1 @scmp_ult_positive_const_gt_than_1_lt_than_umax(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sge i32 [[X]], [[Y]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %1 = call i8 @llvm.scmp(i32 %x, i32 %y)
+  %2 = icmp ult i8 %1, 4
+  ret i1 %2
+}
+
+; scmp(x, y) s> C => x s< y when C != 0 and C != -1
+define i1 @ucmp_ugt_const_not_0_or_neg1(i32 %x, i32 %y) {
+; CHECK-LABEL: define i1 @ucmp_ugt_const_not_0_or_neg1(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i32 [[X]], [[Y]]
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %1 = call i8 @llvm.scmp(i32 %x, i32 %y)
+  %2 = icmp ugt i8 %1, 12
+  ret i1 %2
+}
+
+
 ; ========== Fold -scmp(x, y) => scmp(y, x) ==========
 define i8 @scmp_negated(i32 %x, i32 %y) {
 ; CHECK-LABEL: define i8 @scmp_negated(
diff --git a/llvm/test/Transforms/InstCombine/ucmp.ll b/llvm/test/Transforms/InstCombine/ucmp.ll
index 7210455094baacb..8690e23ea2e0e53 100644
--- a/llvm/test/Transforms/InstCombine/ucmp.ll
+++ b/llvm/test/Transforms/InstCombine/ucmp.ll
@@ -157,6 +157,30 @@ define i1 @ucmp_sle_neg_1(i32 %x, i32 %y) {
   ret i1 %2
 }
 
+; ucmp(x, y) u< C => x u>= y when C u> 1 and C != -1
+define i1 @ucmp_ult_positive_const_gt_than_1_lt_than_umax(i32 %x, i32 %y) {
+; CHECK-LABEL: define i1 @ucmp_ult_positive_const_gt_than_1_lt_than_umax(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp uge i32 [[X]], [[Y]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %1 = call i8 @llvm.ucmp(i32 %x, i32 %y)
+  %2 = icmp ult i8 %1, 4
+  ret i1 %2
+}
+
+; ucmp(x, y) u> C => x u< y when C != 0 and C != -1
+define i1 @ucmp_ugt_const_not_0_or_neg1(i32 %x, i32 %y) {
+; CHECK-LABEL: define i1 @ucmp_ugt_const_not_0_or_neg1(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i32 [[X]], [[Y]]
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %1 = call i8 @llvm.ucmp(i32 %x, i32 %y)
+  %2 = icmp ugt i8 %1, 12
+  ret i1 %2
+}
+
 ; ========== Fold -ucmp(x, y) => ucmp(y, x) ==========
 define i8 @ucmp_negated(i32 %x, i32 %y) {
 ; CHECK-LABEL: define i8 @ucmp_negated(

From e8e88873ab528eb9a44804e6e68953bb058eceb7 Mon Sep 17 00:00:00 2001
From: wr7 <d-wr7@outlook.com>
Date: Fri, 16 Aug 2024 07:44:25 -0500
Subject: [PATCH 160/441] [llvm-c] Add non-cstring versions of
 LLVMGetNamedFunction and LLVMGetNamedGlobal (#103396)

Add `LLVMGetNamedFunctionWithLength` and `LLVMGetNamedGlobalWithLength`

As far as i know, it isn't currently possible to use
`LLVMGetNamedFunction` and `LLVMGetNamedGlobal` with non-null-terminated
strings.

These new functions are more convenient for C programs that use
non-null-terminated strings or for languages like Rust that primarily
use non-null-terminated strings.
---
 llvm/docs/ReleaseNotes.rst |  5 +++++
 llvm/include/llvm-c/Core.h | 12 ++++++++++++
 llvm/lib/IR/Core.cpp       | 10 ++++++++++
 3 files changed, 27 insertions(+)

diff --git a/llvm/docs/ReleaseNotes.rst b/llvm/docs/ReleaseNotes.rst
index 67175e13a8e9ffa..42c0ad976089f7b 100644
--- a/llvm/docs/ReleaseNotes.rst
+++ b/llvm/docs/ReleaseNotes.rst
@@ -151,6 +151,11 @@ Changes to the C API
   * ``LLVMX86MMXTypeInContext``
   * ``LLVMX86MMXType``
 
+ * The following functions are added to further support non-null-terminated strings:
+
+  * ``LLVMGetNamedFunctionWithLength``
+  * ``LLVMGetNamedGlobalWithLength``
+
 Changes to the CodeGen infrastructure
 -------------------------------------
 
diff --git a/llvm/include/llvm-c/Core.h b/llvm/include/llvm-c/Core.h
index 4bde82d816e35cf..7d2e7c955207613 100644
--- a/llvm/include/llvm-c/Core.h
+++ b/llvm/include/llvm-c/Core.h
@@ -1179,6 +1179,16 @@ LLVMValueRef LLVMAddFunction(LLVMModuleRef M, const char *Name,
  */
 LLVMValueRef LLVMGetNamedFunction(LLVMModuleRef M, const char *Name);
 
+/**
+ * Obtain a Function value from a Module by its name.
+ *
+ * The returned value corresponds to a llvm::Function value.
+ *
+ * @see llvm::Module::getFunction()
+ */
+LLVMValueRef LLVMGetNamedFunctionWithLength(LLVMModuleRef M, const char *Name,
+                                            size_t Length);
+
 /**
  * Obtain an iterator to the first Function in a Module.
  *
@@ -2633,6 +2643,8 @@ LLVMValueRef LLVMAddGlobalInAddressSpace(LLVMModuleRef M, LLVMTypeRef Ty,
                                          const char *Name,
                                          unsigned AddressSpace);
 LLVMValueRef LLVMGetNamedGlobal(LLVMModuleRef M, const char *Name);
+LLVMValueRef LLVMGetNamedGlobalWithLength(LLVMModuleRef M, const char *Name,
+                                          size_t Length);
 LLVMValueRef LLVMGetFirstGlobal(LLVMModuleRef M);
 LLVMValueRef LLVMGetLastGlobal(LLVMModuleRef M);
 LLVMValueRef LLVMGetNextGlobal(LLVMValueRef GlobalVar);
diff --git a/llvm/lib/IR/Core.cpp b/llvm/lib/IR/Core.cpp
index 17c0bf72ef05dc3..dcad76ee8491dd8 100644
--- a/llvm/lib/IR/Core.cpp
+++ b/llvm/lib/IR/Core.cpp
@@ -2203,6 +2203,11 @@ LLVMValueRef LLVMGetNamedGlobal(LLVMModuleRef M, const char *Name) {
   return wrap(unwrap(M)->getNamedGlobal(Name));
 }
 
+LLVMValueRef LLVMGetNamedGlobalWithLength(LLVMModuleRef M, const char *Name,
+                                          size_t Length) {
+  return wrap(unwrap(M)->getNamedGlobal(StringRef(Name, Length)));
+}
+
 LLVMValueRef LLVMGetFirstGlobal(LLVMModuleRef M) {
   Module *Mod = unwrap(M);
   Module::global_iterator I = Mod->global_begin();
@@ -2381,6 +2386,11 @@ LLVMValueRef LLVMGetNamedFunction(LLVMModuleRef M, const char *Name) {
   return wrap(unwrap(M)->getFunction(Name));
 }
 
+LLVMValueRef LLVMGetNamedFunctionWithLength(LLVMModuleRef M, const char *Name,
+                                            size_t Length) {
+  return wrap(unwrap(M)->getFunction(StringRef(Name, Length)));
+}
+
 LLVMValueRef LLVMGetFirstFunction(LLVMModuleRef M) {
   Module *Mod = unwrap(M);
   Module::iterator I = Mod->begin();

From ce128d8fe8298e87799b5302e08bd0c6ea8ccce4 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Fri, 16 Aug 2024 15:00:08 +0200
Subject: [PATCH 161/441] [InstCombine] Add i128 test for select of lshr/ashr
 transform (NFC)

---
 llvm/test/Transforms/InstCombine/ashr-lshr.ll | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/llvm/test/Transforms/InstCombine/ashr-lshr.ll b/llvm/test/Transforms/InstCombine/ashr-lshr.ll
index c2a4f35412670be..4a3f836e95c3c35 100644
--- a/llvm/test/Transforms/InstCombine/ashr-lshr.ll
+++ b/llvm/test/Transforms/InstCombine/ashr-lshr.ll
@@ -61,6 +61,21 @@ define i32 @ashr_lshr2(i32 %x, i32 %y) {
   ret i32 %ret
 }
 
+define i128 @ashr_lshr2_i128(i128 %x, i128 %y) {
+; CHECK-LABEL: @ashr_lshr2_i128(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i128 [[X:%.*]], 5
+; CHECK-NEXT:    [[L:%.*]] = lshr i128 [[X]], [[Y:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = ashr exact i128 [[X]], [[Y]]
+; CHECK-NEXT:    [[RET:%.*]] = select i1 [[CMP]], i128 [[L]], i128 [[R]]
+; CHECK-NEXT:    ret i128 [[RET]]
+;
+  %cmp = icmp sgt i128 %x, 5
+  %l = lshr i128 %x, %y
+  %r = ashr exact i128 %x, %y
+  %ret = select i1 %cmp, i128 %l, i128 %r
+  ret i128 %ret
+}
+
 define <2 x i32> @ashr_lshr_splat_vec(<2 x i32> %x, <2 x i32> %y) {
 ; CHECK-LABEL: @ashr_lshr_splat_vec(
 ; CHECK-NEXT:    [[CMP12:%.*]] = ashr <2 x i32> [[X:%.*]], [[Y:%.*]]

From 5d28678277195e0b136198323ef4afa4561f8796 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Fri, 16 Aug 2024 15:01:16 +0200
Subject: [PATCH 162/441] [InstCombine] Fix incorrect zero ext in select of
 lshr/ashr fold

The -1 constant should be sign extended, not zero extended.

Split out from https://github.com/llvm/llvm-project/pull/80309.
---
 llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp | 8 ++++----
 llvm/test/Transforms/InstCombine/ashr-lshr.ll         | 7 ++-----
 2 files changed, 6 insertions(+), 9 deletions(-)

diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
index b86f27168303a1e..b8b77bec1b900d6 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
@@ -679,11 +679,11 @@ static Value *foldSelectICmpLshrAshr(const ICmpInst *IC, Value *TrueVal,
   Value *X, *Y;
   unsigned Bitwidth = CmpRHS->getType()->getScalarSizeInBits();
   if ((Pred != ICmpInst::ICMP_SGT ||
-       !match(CmpRHS,
-              m_SpecificInt_ICMP(ICmpInst::ICMP_SGE, APInt(Bitwidth, -1)))) &&
+       !match(CmpRHS, m_SpecificInt_ICMP(ICmpInst::ICMP_SGE,
+                                         APInt::getAllOnes(Bitwidth)))) &&
       (Pred != ICmpInst::ICMP_SLT ||
-       !match(CmpRHS,
-              m_SpecificInt_ICMP(ICmpInst::ICMP_SGE, APInt(Bitwidth, 0)))))
+       !match(CmpRHS, m_SpecificInt_ICMP(ICmpInst::ICMP_SGE,
+                                         APInt::getZero(Bitwidth)))))
     return nullptr;
 
   // Canonicalize so that ashr is in FalseVal.
diff --git a/llvm/test/Transforms/InstCombine/ashr-lshr.ll b/llvm/test/Transforms/InstCombine/ashr-lshr.ll
index 4a3f836e95c3c35..a81cd47b1cd4b64 100644
--- a/llvm/test/Transforms/InstCombine/ashr-lshr.ll
+++ b/llvm/test/Transforms/InstCombine/ashr-lshr.ll
@@ -63,11 +63,8 @@ define i32 @ashr_lshr2(i32 %x, i32 %y) {
 
 define i128 @ashr_lshr2_i128(i128 %x, i128 %y) {
 ; CHECK-LABEL: @ashr_lshr2_i128(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i128 [[X:%.*]], 5
-; CHECK-NEXT:    [[L:%.*]] = lshr i128 [[X]], [[Y:%.*]]
-; CHECK-NEXT:    [[R:%.*]] = ashr exact i128 [[X]], [[Y]]
-; CHECK-NEXT:    [[RET:%.*]] = select i1 [[CMP]], i128 [[L]], i128 [[R]]
-; CHECK-NEXT:    ret i128 [[RET]]
+; CHECK-NEXT:    [[CMP1:%.*]] = ashr i128 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i128 [[CMP1]]
 ;
   %cmp = icmp sgt i128 %x, 5
   %l = lshr i128 %x, %y

From ff906403dd8c9791eaa3168faeb546a5266ceac0 Mon Sep 17 00:00:00 2001
From: Lukacma <Marian.Lukac@arm.com>
Date: Fri, 16 Aug 2024 14:05:07 +0100
Subject: [PATCH 163/441] Add FPMR register and update dependencies of FP8
 instructions (#102910)

Currently FP8 instructions are missing dependecies on system registers,
namely FPMR and FPCR. This patch fixes this issue.
---
 llvm/lib/Target/AArch64/AArch64InstrInfo.td   |  8 +--
 .../Target/AArch64/AArch64RegisterInfo.cpp    |  1 +
 .../lib/Target/AArch64/AArch64RegisterInfo.td |  3 ++
 .../lib/Target/AArch64/AArch64SVEInstrInfo.td |  9 ++--
 llvm/lib/Target/AArch64/SMEInstrFormats.td    |  9 +++-
 llvm/lib/Target/AArch64/SVEInstrFormats.td    | 51 +++++++++++++++++--
 llvm/test/CodeGen/AArch64/preserve.ll         |  6 +--
 7 files changed, 69 insertions(+), 18 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index 403453a4ee0a92c..a9324af5beb784d 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -10148,7 +10148,7 @@ let Predicates = [HasD128] in {
 // 2023 Architecture Extensions:
 //===----------------------------===//
 
-let Predicates = [HasFP8] in {
+let Uses = [FPMR, FPCR], Predicates = [HasFP8] in {
   defm F1CVTL  : SIMDMixedTwoVectorFP8<0b00, "f1cvtl">;
   defm F2CVTL  : SIMDMixedTwoVectorFP8<0b01, "f2cvtl">;
   defm BF1CVTL : SIMDMixedTwoVectorFP8<0b10, "bf1cvtl">;
@@ -10173,7 +10173,7 @@ let Predicates = [HasNEON, HasFAMINMAX] in {
  }
 } // End let Predicates = [HasNEON, HasFAMINMAX]
 
-let Predicates = [HasFP8FMA] in {
+let Uses = [FPMR, FPCR], Predicates = [HasFP8FMA] in {
  defm FMLALBlane : SIMDThreeSameVectorMLAIndex<0b0, "fmlalb">;
  defm FMLALTlane : SIMDThreeSameVectorMLAIndex<0b1, "fmlalt">;
  defm FMLALLBBlane : SIMDThreeSameVectorMLALIndex<0b0, 0b00, "fmlallbb">;
@@ -10189,12 +10189,12 @@ let Predicates = [HasFP8FMA] in {
  defm FMLALLTT : SIMDThreeSameVectorMLAL<0b1, 0b01, "fmlalltt">;
 } // End let Predicates = [HasFP8FMA]
 
-let Predicates = [HasFP8DOT2] in {
+let Uses = [FPMR, FPCR], Predicates = [HasFP8DOT2] in {
  defm FDOTlane : SIMDThreeSameVectorFP8DOT2Index<"fdot">;
  defm FDOT : SIMDThreeSameVectorDOT2<"fdot">;
 } // End let Predicates = [HasFP8DOT2]
 
-let Predicates = [HasFP8DOT4] in {
+let Uses = [FPMR, FPCR], Predicates = [HasFP8DOT4] in {
  defm FDOTlane : SIMDThreeSameVectorFP8DOT4Index<"fdot">;
  defm FDOT : SIMDThreeSameVectorDOT4<"fdot">;
 } // End let Predicates = [HasFP8DOT4]
diff --git a/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp b/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
index 435cc18cdea6249..2deffbd282569c1 100644
--- a/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
@@ -479,6 +479,7 @@ AArch64RegisterInfo::getStrictlyReservedRegs(const MachineFunction &MF) const {
   }
 
   markSuperRegs(Reserved, AArch64::FPCR);
+  markSuperRegs(Reserved, AArch64::FPMR);
   markSuperRegs(Reserved, AArch64::FPSR);
 
   if (MF.getFunction().getCallingConv() == CallingConv::GRAAL) {
diff --git a/llvm/lib/Target/AArch64/AArch64RegisterInfo.td b/llvm/lib/Target/AArch64/AArch64RegisterInfo.td
index 4dc33e6168cbdaf..f754c32e1176d1d 100644
--- a/llvm/lib/Target/AArch64/AArch64RegisterInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64RegisterInfo.td
@@ -147,6 +147,9 @@ def VG : AArch64Reg<0, "vg">, DwarfRegNum<[46]>;
 // Floating-point control register
 def FPCR : AArch64Reg<0, "fpcr">;
 
+// Floating-point Mode Register
+def FPMR : AArch64Reg<0, "fpmr">;
+
 // Floating-point status register.
 def FPSR : AArch64Reg<0, "fpsr">;
 
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index d9a70b5ef02fcb1..3f4651ea9c2b68f 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -4244,19 +4244,18 @@ def FMLALLTT_ZZZ : sve2_fp8_mla<0b011, ZPR32, "fmlalltt">;
 
 let Predicates = [HasSSVE_FP8DOT2] in {
 // FP8 Widening Dot-Product - Indexed Group
-defm FDOT_ZZZI_BtoH : sve2_fp8_dot_indexed<"fdot">;
+defm FDOT_ZZZI_BtoH : sve2_fp8_dot_indexed_h<"fdot">;
 // FP8 Widening Dot-Product - Group
 // TODO: Replace nxv16i8 by nxv16f8
-defm FDOT_ZZZ_BtoH : sve_float_dot<0b0, 0b1, ZPR16, ZPR8, "fdot", nxv16i8, null_frag>;
+defm FDOT_ZZZ_BtoH : sve_fp8_dot<0b0, ZPR16, "fdot">;
 }
 
 // TODO: Replace nxv16i8 by nxv16f8
 let Predicates = [HasSSVE_FP8DOT4] in {
 // FP8 Widening Dot-Product - Indexed Group
-defm FDOT_ZZZI_BtoS : sve_float_dot_indexed<0b1, 0b01, ZPR8, ZPR3b8, "fdot",
-                                            nxv16i8, null_frag>;
+defm FDOT_ZZZI_BtoS : sve2_fp8_dot_indexed_s<"fdot">;
 // FP8 Widening Dot-Product - Group
-defm FDOT_ZZZ_BtoS : sve_float_dot<0b1, 0b1, ZPR32, ZPR8, "fdot", nxv16i8, null_frag>;
+defm FDOT_ZZZ_BtoS : sve_fp8_dot<0b1, ZPR32, "fdot">;
 }
 
 let Predicates = [HasSVE2orSME2, HasLUT] in {
diff --git a/llvm/lib/Target/AArch64/SMEInstrFormats.td b/llvm/lib/Target/AArch64/SMEInstrFormats.td
index 77cf5cb56728b91..a83c132496efadf 100644
--- a/llvm/lib/Target/AArch64/SMEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SMEInstrFormats.td
@@ -2240,7 +2240,9 @@ multiclass sme2_cvt_vg2_single<string mnemonic, bits<5> op, ValueType out_vt,
 
 // SME2 multi-vec FP8 down convert two registers
 multiclass sme2_fp8_cvt_vg2_single<string mnemonic, bit op> {
-  def NAME :  sme2_cvt_vg2_single<mnemonic, {op, 0b1000}, ZPR8, ZZ_h_mul_r>;
+  def NAME :  sme2_cvt_vg2_single<mnemonic, {op, 0b1000}, ZPR8, ZZ_h_mul_r>{
+    let Uses = [FPMR, FPCR];
+  }
 }
 
 class sme2_cvt_unpk_vector_vg2<bits<2>sz, bits<3> op, bit u, RegisterOperand first_ty,
@@ -2273,7 +2275,9 @@ multiclass sme2p1_fp_cvt_vector_vg2_single<string mnemonic, bit l> {
 
 // SME2 multi-vec FP8 up convert two registers
 multiclass sme2p1_fp8_cvt_vector_vg2_single<string mnemonic, bits<2> opc, bit L> {
-  def _NAME : sme2_cvt_unpk_vector_vg2<opc, 0b110, L, ZZ_h_mul_r, ZPR8, mnemonic>;
+  def _NAME : sme2_cvt_unpk_vector_vg2<opc, 0b110, L, ZZ_h_mul_r, ZPR8, mnemonic>{
+    let Uses = [FPMR, FPCR];
+  }
 }
 
 
@@ -2536,6 +2540,7 @@ class sme2_fp8_multi_vec_array_vg4_index<string mnemonic, bit T>
   let Inst{10} = i{1};
   let Inst{3}  = i{0};
   let AsmString = !strconcat(mnemonic, "{\t$ZAda[$Rv, $imm3, vgx4], $Zn, $Zm$i}");
+  let Uses = [FPMR, FPCR];
 }
 
 // SME2 multi-vec ternary indexed two registers 64-bit
diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td
index 482d3a0664d72fb..107bc79f70dbcb5 100644
--- a/llvm/lib/Target/AArch64/SVEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td
@@ -8708,6 +8708,12 @@ multiclass sve_float_dot<bit bf, bit o2, ZPRRegOp dst_ty, ZPRRegOp src_ty,
   def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, InVT, InVT, !cast<Instruction>(NAME)>;
 }
 
+multiclass sve_fp8_dot<bit bf, ZPRRegOp dst_ty, string asm> {
+  def NAME : sve_float_dot<bf, 0b1, dst_ty, ZPR8, asm>{
+    let Uses = [FPMR, FPCR];
+  }
+}
+
 class sve_float_dot_indexed<bit bf, ZPRRegOp dst_ty, ZPRRegOp src1_ty,
                             ZPRRegOp src2_ty, Operand iop_ty, string asm>
 : I<(outs dst_ty:$Zda), (ins dst_ty:$_Zda, src1_ty:$Zn, src2_ty:$Zm, iop_ty:$iop),
@@ -10212,6 +10218,7 @@ class sve2_fp8_cvt_single<bit L, bits<2> opc, string mnemonic,
   let Inst{11-10} = opc;
   let Inst{9-5}   = Zn;
   let Inst{4-0}   = Zd;
+  let Uses = [FPMR, FPCR];
 }
 
 multiclass sve2_fp8_cvt_single<bit L, bits<2> opc, string mnemonic> {
@@ -10231,6 +10238,7 @@ class sve2_fp8_down_cvt_single<bits<2> opc, string mnemonic,
   let Inst{9-6} = Zn;
   let Inst{5} = 0b0;
   let Inst{4-0} = Zd;
+  let Uses = [FPMR, FPCR];
 }
 
 multiclass sve2_fp8_down_cvt_single<bits<2> opc, string mnemonic, RegisterOperand src> {
@@ -10259,6 +10267,7 @@ class sve2_fp8_mla_long_by_indexed_elem<bit T, string mnemonic>
   let Constraints = "$Zda = $_Zda";
   let DestructiveInstType = DestructiveOther;
   let ElementSize         = ZPR16.ElementSize;
+  let Uses = [FPMR, FPCR];
 }
 
 // FP8 Widening Multiply-Add (Long)/(Long Long) Group
@@ -10282,6 +10291,7 @@ class sve2_fp8_mla<bits<3>opc, ZPRRegOp dst_ty, string mnemonic>
   let Constraints = "$Zda = $_Zda";
   let DestructiveInstType = DestructiveOther;
   let ElementSize         = dst_ty.ElementSize;
+  let Uses = [FPMR, FPCR];
 }
 
 // FP8 Widening Multiply-Add Long Long - Indexed Group
@@ -10306,15 +10316,48 @@ class sve2_fp8_mla_long_long_by_indexed_elem<bits<2> TT, string mnemonic>
   let Constraints = "$Zda = $_Zda";
   let DestructiveInstType = DestructiveOther;
   let ElementSize         = ZPR32.ElementSize;
+  let Uses = [FPMR, FPCR];
+}
+
+class sve_fp8_dot_indexed<bits<4> opc, ZPRRegOp dst_ty, Operand iop_ty, string mnemonic>
+: I<(outs dst_ty:$Zda), (ins dst_ty:$_Zda, ZPR8:$Zn, ZPR3b8:$Zm, iop_ty:$iop),
+    mnemonic, "\t$Zda, $Zn, $Zm$iop", "", []>, Sched<[]> {
+  bits<5> Zda;
+  bits<5> Zn;
+  bits<3> Zm;
+  let Inst{31-23} = 0b011001000;
+  let Inst{22}    = opc{3};
+  let Inst{21}    = 0b1;
+  let Inst{20-19} = opc{2-1};
+  let Inst{18-16} = Zm;
+  let Inst{15-12} = 0b0100;
+  let Inst{11}    = opc{0};
+  let Inst{10}    = 0b1;
+  let Inst{9-5}   = Zn;
+  let Inst{4-0}   = Zda;
+
+  let Uses = [FPMR, FPCR];
+  let Constraints = "$Zda = $_Zda";
+  let DestructiveInstType = DestructiveOther;
+  let hasSideEffects = 0;
+  let mayRaiseFPException = 1;
 }
 
 // FP8 Widening Dot-Product - Indexed Group
-multiclass sve2_fp8_dot_indexed<string mnemonic>{
-  def NAME : sve_float_dot_indexed<0b0, ZPR16, ZPR8, ZPR3b8, VectorIndexH, mnemonic> {
+multiclass sve2_fp8_dot_indexed_h<string asm>{
+  def NAME : sve_fp8_dot_indexed<{0, ?, ?, ?}, ZPR16, VectorIndexH, asm> {
     bits<3> iop;
+
     let Inst{20-19} = iop{2-1};
-    let Inst{11} = iop{0};
-    let Inst{10} = 0b1;
+    let Inst{11}    = iop{0};
+  }
+}
+
+multiclass sve2_fp8_dot_indexed_s<string asm>{
+  def NAME : sve_fp8_dot_indexed<{1, ?, ?, 0}, ZPR32, VectorIndexS32b, asm> {
+    bits<2> iop;
+
+    let Inst{20-19} = iop{1-0};
   }
 }
 
diff --git a/llvm/test/CodeGen/AArch64/preserve.ll b/llvm/test/CodeGen/AArch64/preserve.ll
index a8acdc1df976068..5839bb5440a9b05 100644
--- a/llvm/test/CodeGen/AArch64/preserve.ll
+++ b/llvm/test/CodeGen/AArch64/preserve.ll
@@ -4,19 +4,19 @@
 target triple = "aarch64-unknown-unknown"
 declare void @bar1()
 define preserve_mostcc void @baz() #0 {
-; CHECK: baz Clobbered Registers: $ffr $fpcr $fpsr $nzcv $sp $vg $wsp $za $b0 $b1 $b2 $b3 $b4 $b5 $b6 $b7 $b16 $b17 $b18 $b19 $b20 $b21 $b22 $b23 $b24 $b25 $b26 $b27 $b28 $b29 $b30 $b31 $d0 $d1 $d2 $d3 $d4 $d5 $d6 $d7 $d16 $d17 $d18 $d19 $d20 $d21 $d22 $d23 $d24 $d25 $d26 $d27 $d28 $d29 $d30 $d31 $h0 $h1 $h2 $h3 $h4 $h5 $h6 $h7 $h16 $h17 $h18 $h19 $h20 $h21 $h22 $h23 $h24 $h25 $h26 $h27 $h28 $h29 $h30 $h31 $p0 $p1 $p2 $p3 $p4 $p5 $p6 $p7 $p8 $p9 $p10 $p11 $p12 $p13 $p14 $p15 $pn0 $pn1 $pn2 $pn3 $pn4 $pn5 $pn6 $pn7 $pn8 $pn9 $pn10 $pn11 $pn12 $pn13 $pn14 $pn15 $q0 $q1 $q2 $q3 $q4 $q5 $q6 $q7 $q8 $q9 $q10 $q11 $q12 $q13 $q14 $q15 $q16 $q17 $q18 $q19 $q20 $q21 $q22 $q23 $q24 $q25 $q26 $q27 $q28 $q29 $q30 $q31 $s0 $s1 $s2 $s3 $s4 $s5 $s6 $s7 $s16 $s17 $s18 $s19 $s20 $s21 $s22 $s23 $s24 $s25 $s26 $s27 $s28 $s29 $s30 $s31 $w0 $w1 $w2 $w3 $w4 $w5 $w6 $w7 $w8 $w16 $w17 $w18 $x0 $x1 $x2 $x3 $x4 $x5 $x6 $x7 $x8 $x16 $x17 $x18 $z0 $z1 $z2 $z3 $z4 $z5 $z6 $z7 $z8 $z9 $z10 $z11 $z12 $z13 $z14 $z15 $z16 $z17 $z18 $z19 $z20 $z21 $z22 $z23 $z24 $z25 $z26 $z27 $z28 $z29 $z30 $z31 $zab0 $zad0 $zad1 $zad2 $zad3 $zad4 $zad5 $zad6 $zad7 $zah0 $zah1 $zaq0 $zaq1 $zaq2 $zaq3 $zaq4 $zaq5 $zaq6 $zaq7 $zaq8 $zaq9 $zaq10 $zaq11 $zaq12 $zaq13 $zaq14 $zaq15 $zas0 $zas1 $zas2 $zas3 $zt0 $d0_d1 $d1_d2 $d2_d3 $d3_d4 $d4_d5 $d5_d6 $d6_d7 $d7_d8 $d15_d16 $d16_d17 $d17_d18 $d18_d19 $d19_d20 $d20_d21 $d21_d22 $d22_d23 $d23_d24 $d24_d25 $d25_d26 $d26_d27 $d27_d28 $d28_d29 $d29_d30 $d30_d31 $d31_d0 $d0_d1_d2_d3 $d1_d2_d3_d4 $d2_d3_d4_d5 $d3_d4_d5_d6 $d4_d5_d6_d7 $d5_d6_d7_d8 $d6_d7_d8_d9 $d7_d8_d9_d10 $d13_d14_d15_d16 $d14_d15_d16_d17 $d15_d16_d17_d18 $d16_d17_d18_d19 $d17_d18_d19_d20 $d18_d19_d20_d21 $d19_d20_d21_d22 $d20_d21_d22_d23 $d21_d22_d23_d24 $d22_d23_d24_d25 $d23_d24_d25_d26 $d24_d25_d26_d27 $d25_d26_d27_d28 $d26_d27_d28_d29 $d27_d28_d29_d30 $d28_d29_d30_d31 $d29_d30_d31_d0 $d30_d31_d0_d1 $d31_d0_d1_d2 $d0_d1_d2 $d1_d2_d3 $d2_d3_d4 $d3_d4_d5 $d4_d5_d6 $d5_d6_d7 $d6_d7_d8 $d7_d8_d9 $d14_d15_d16 $d15_d16_d17 $d16_d17_d18 $d17_d18_d19 $d18_d19_d20 $d19_d20_d21 $d20_d21_d22 $d21_d22_d23 $d22_d23_d24 $d23_d24_d25 $d24_d25_d26 $d25_d26_d27 $d26_d27_d28 $d27_d28_d29 $d28_d29_d30 $d29_d30_d31 $d30_d31_d0 $d31_d0_d1 $p0_p1 $p1_p2 $p2_p3 $p3_p4 $p4_p5 $p5_p6 $p6_p7 $p7_p8 $p8_p9 $p9_p10 $p10_p11 $p11_p12 $p12_p13 $p13_p14 $p14_p15 $p15_p0 $q0_q1 $q1_q2 $q2_q3 $q3_q4 $q4_q5 $q5_q6 $q6_q7 $q7_q8 $q8_q9 $q9_q10 $q10_q11 $q11_q12 $q12_q13 $q13_q14 $q14_q15 $q15_q16 $q16_q17 $q17_q18 $q18_q19 $q19_q20 $q20_q21 $q21_q22 $q22_q23 $q23_q24 $q24_q25 $q25_q26 $q26_q27 $q27_q28 $q28_q29 $q29_q30 $q30_q31 $q31_q0 $q0_q1_q2_q3 $q1_q2_q3_q4 $q2_q3_q4_q5 $q3_q4_q5_q6 $q4_q5_q6_q7 $q5_q6_q7_q8 $q6_q7_q8_q9 $q7_q8_q9_q10 $q8_q9_q10_q11 $q9_q10_q11_q12 $q10_q11_q12_q13 $q11_q12_q13_q14 $q12_q13_q14_q15 $q13_q14_q15_q16 $q14_q15_q16_q17 $q15_q16_q17_q18 $q16_q17_q18_q19 $q17_q18_q19_q20 $q18_q19_q20_q21 $q19_q20_q21_q22 $q20_q21_q22_q23 $q21_q22_q23_q24 $q22_q23_q24_q25 $q23_q24_q25_q26 $q24_q25_q26_q27 $q25_q26_q27_q28 $q26_q27_q28_q29 $q27_q28_q29_q30 $q28_q29_q30_q31 $q29_q30_q31_q0 $q30_q31_q0_q1 $q31_q0_q1_q2 $q0_q1_q2 $q1_q2_q3 $q2_q3_q4 $q3_q4_q5 $q4_q5_q6 $q5_q6_q7 $q6_q7_q8 $q7_q8_q9 $q8_q9_q10 $q9_q10_q11 $q10_q11_q12 $q11_q12_q13 $q12_q13_q14 $q13_q14_q15 $q14_q15_q16 $q15_q16_q17 $q16_q17_q18 $q17_q18_q19 $q18_q19_q20 $q19_q20_q21 $q20_q21_q22 $q21_q22_q23 $q22_q23_q24 $q23_q24_q25 $q24_q25_q26 $q25_q26_q27 $q26_q27_q28 $q27_q28_q29 $q28_q29_q30 $q29_q30_q31 $q30_q31_q0 $q31_q0_q1 $x0_x1_x2_x3_x4_x5_x6_x7 $x2_x3_x4_x5_x6_x7_x8_x9 $x4_x5_x6_x7_x8_x9_x10_x11 $x6_x7_x8_x9_x10_x11_x12_x13 $x8_x9_x10_x11_x12_x13_x14_x15 $x10_x11_x12_x13_x14_x15_x16_x17 $x12_x13_x14_x15_x16_x17_x18_x19 $x14_x15_x16_x17_x18_x19_x20_x21 $x16_x17_x18_x19_x20_x21_x22_x23 $x18_x19_x20_x21_x22_x23_x24_x25 $w30_wzr $w0_w1 $w2_w3 $w4_w5 $w6_w7 $w8_w9 $w10_w11 $w12_w13 $w14_w15 $w16_w17 $w18_w19 $lr_xzr $x0_x1 $x2_x3 $x4_x5 $x6_x7 $x8_x9 $x10_x11 $x12_x13 $x14_x15 $x16_x17 $x18_x19 $z0_z1 $z1_z2 $z2_z3 $z3_z4 $z4_z5 $z5_z6 $z6_z7 $z7_z8 $z8_z9 $z9_z10 $z10_z11 $z11_z12 $z12_z13 $z13_z14 $z14_z15 $z15_z16 $z16_z17 $z17_z18 $z18_z19 $z19_z20 $z20_z21 $z21_z22 $z22_z23 $z23_z24 $z24_z25 $z25_z26 $z26_z27 $z27_z28 $z28_z29 $z29_z30 $z30_z31 $z31_z0 $z0_z1_z2_z3 $z1_z2_z3_z4 $z2_z3_z4_z5 $z3_z4_z5_z6 $z4_z5_z6_z7 $z5_z6_z7_z8 $z6_z7_z8_z9 $z7_z8_z9_z10 $z8_z9_z10_z11 $z9_z10_z11_z12 $z10_z11_z12_z13 $z11_z12_z13_z14 $z12_z13_z14_z15 $z13_z14_z15_z16 $z14_z15_z16_z17 $z15_z16_z17_z18 $z16_z17_z18_z19 $z17_z18_z19_z20 $z18_z19_z20_z21 $z19_z20_z21_z22 $z20_z21_z22_z23 $z21_z22_z23_z24 $z22_z23_z24_z25 $z23_z24_z25_z26 $z24_z25_z26_z27 $z25_z26_z27_z28 $z26_z27_z28_z29 $z27_z28_z29_z30 $z28_z29_z30_z31 $z29_z30_z31_z0 $z30_z31_z0_z1 $z31_z0_z1_z2 $z0_z1_z2 $z1_z2_z3 $z2_z3_z4 $z3_z4_z5 $z4_z5_z6 $z5_z6_z7 $z6_z7_z8 $z7_z8_z9 $z8_z9_z10 $z9_z10_z11 $z10_z11_z12 $z11_z12_z13 $z12_z13_z14 $z13_z14_z15 $z14_z15_z16 $z15_z16_z17 $z16_z17_z18 $z17_z18_z19 $z18_z19_z20 $z19_z20_z21 $z20_z21_z22 $z21_z22_z23 $z22_z23_z24 $z23_z24_z25 $z24_z25_z26 $z25_z26_z27 $z26_z27_z28 $z27_z28_z29 $z28_z29_z30 $z29_z30_z31 $z30_z31_z0 $z31_z0_z1 $z16_z24 $z17_z25 $z18_z26 $z19_z27 $z20_z28 $z21_z29 $z22_z30 $z23_z31 $z0_z8 $z1_z9 $z2_z10 $z3_z11 $z4_z12 $z5_z13 $z6_z14 $z7_z15 $z16_z20_z24_z28 $z17_z21_z25_z29 $z18_z22_z26_z30 $z19_z23_z27_z31 $z0_z4_z8_z12 $z1_z5_z9_z13 $z2_z6_z10_z14 $z3_z7_z11_z15
+; CHECK: baz Clobbered Registers: $ffr $fpcr $fpmr $fpsr $nzcv $sp $vg $wsp $za $b0 $b1 $b2 $b3 $b4 $b5 $b6 $b7 $b16 $b17 $b18 $b19 $b20 $b21 $b22 $b23 $b24 $b25 $b26 $b27 $b28 $b29 $b30 $b31 $d0 $d1 $d2 $d3 $d4 $d5 $d6 $d7 $d16 $d17 $d18 $d19 $d20 $d21 $d22 $d23 $d24 $d25 $d26 $d27 $d28 $d29 $d30 $d31 $h0 $h1 $h2 $h3 $h4 $h5 $h6 $h7 $h16 $h17 $h18 $h19 $h20 $h21 $h22 $h23 $h24 $h25 $h26 $h27 $h28 $h29 $h30 $h31 $p0 $p1 $p2 $p3 $p4 $p5 $p6 $p7 $p8 $p9 $p10 $p11 $p12 $p13 $p14 $p15 $pn0 $pn1 $pn2 $pn3 $pn4 $pn5 $pn6 $pn7 $pn8 $pn9 $pn10 $pn11 $pn12 $pn13 $pn14 $pn15 $q0 $q1 $q2 $q3 $q4 $q5 $q6 $q7 $q8 $q9 $q10 $q11 $q12 $q13 $q14 $q15 $q16 $q17 $q18 $q19 $q20 $q21 $q22 $q23 $q24 $q25 $q26 $q27 $q28 $q29 $q30 $q31 $s0 $s1 $s2 $s3 $s4 $s5 $s6 $s7 $s16 $s17 $s18 $s19 $s20 $s21 $s22 $s23 $s24 $s25 $s26 $s27 $s28 $s29 $s30 $s31 $w0 $w1 $w2 $w3 $w4 $w5 $w6 $w7 $w8 $w16 $w17 $w18 $x0 $x1 $x2 $x3 $x4 $x5 $x6 $x7 $x8 $x16 $x17 $x18 $z0 $z1 $z2 $z3 $z4 $z5 $z6 $z7 $z8 $z9 $z10 $z11 $z12 $z13 $z14 $z15 $z16 $z17 $z18 $z19 $z20 $z21 $z22 $z23 $z24 $z25 $z26 $z27 $z28 $z29 $z30 $z31 $zab0 $zad0 $zad1 $zad2 $zad3 $zad4 $zad5 $zad6 $zad7 $zah0 $zah1 $zaq0 $zaq1 $zaq2 $zaq3 $zaq4 $zaq5 $zaq6 $zaq7 $zaq8 $zaq9 $zaq10 $zaq11 $zaq12 $zaq13 $zaq14 $zaq15 $zas0 $zas1 $zas2 $zas3 $zt0 $d0_d1 $d1_d2 $d2_d3 $d3_d4 $d4_d5 $d5_d6 $d6_d7 $d7_d8 $d15_d16 $d16_d17 $d17_d18 $d18_d19 $d19_d20 $d20_d21 $d21_d22 $d22_d23 $d23_d24 $d24_d25 $d25_d26 $d26_d27 $d27_d28 $d28_d29 $d29_d30 $d30_d31 $d31_d0 $d0_d1_d2_d3 $d1_d2_d3_d4 $d2_d3_d4_d5 $d3_d4_d5_d6 $d4_d5_d6_d7 $d5_d6_d7_d8 $d6_d7_d8_d9 $d7_d8_d9_d10 $d13_d14_d15_d16 $d14_d15_d16_d17 $d15_d16_d17_d18 $d16_d17_d18_d19 $d17_d18_d19_d20 $d18_d19_d20_d21 $d19_d20_d21_d22 $d20_d21_d22_d23 $d21_d22_d23_d24 $d22_d23_d24_d25 $d23_d24_d25_d26 $d24_d25_d26_d27 $d25_d26_d27_d28 $d26_d27_d28_d29 $d27_d28_d29_d30 $d28_d29_d30_d31 $d29_d30_d31_d0 $d30_d31_d0_d1 $d31_d0_d1_d2 $d0_d1_d2 $d1_d2_d3 $d2_d3_d4 $d3_d4_d5 $d4_d5_d6 $d5_d6_d7 $d6_d7_d8 $d7_d8_d9 $d14_d15_d16 $d15_d16_d17 $d16_d17_d18 $d17_d18_d19 $d18_d19_d20 $d19_d20_d21 $d20_d21_d22 $d21_d22_d23 $d22_d23_d24 $d23_d24_d25 $d24_d25_d26 $d25_d26_d27 $d26_d27_d28 $d27_d28_d29 $d28_d29_d30 $d29_d30_d31 $d30_d31_d0 $d31_d0_d1 $p0_p1 $p1_p2 $p2_p3 $p3_p4 $p4_p5 $p5_p6 $p6_p7 $p7_p8 $p8_p9 $p9_p10 $p10_p11 $p11_p12 $p12_p13 $p13_p14 $p14_p15 $p15_p0 $q0_q1 $q1_q2 $q2_q3 $q3_q4 $q4_q5 $q5_q6 $q6_q7 $q7_q8 $q8_q9 $q9_q10 $q10_q11 $q11_q12 $q12_q13 $q13_q14 $q14_q15 $q15_q16 $q16_q17 $q17_q18 $q18_q19 $q19_q20 $q20_q21 $q21_q22 $q22_q23 $q23_q24 $q24_q25 $q25_q26 $q26_q27 $q27_q28 $q28_q29 $q29_q30 $q30_q31 $q31_q0 $q0_q1_q2_q3 $q1_q2_q3_q4 $q2_q3_q4_q5 $q3_q4_q5_q6 $q4_q5_q6_q7 $q5_q6_q7_q8 $q6_q7_q8_q9 $q7_q8_q9_q10 $q8_q9_q10_q11 $q9_q10_q11_q12 $q10_q11_q12_q13 $q11_q12_q13_q14 $q12_q13_q14_q15 $q13_q14_q15_q16 $q14_q15_q16_q17 $q15_q16_q17_q18 $q16_q17_q18_q19 $q17_q18_q19_q20 $q18_q19_q20_q21 $q19_q20_q21_q22 $q20_q21_q22_q23 $q21_q22_q23_q24 $q22_q23_q24_q25 $q23_q24_q25_q26 $q24_q25_q26_q27 $q25_q26_q27_q28 $q26_q27_q28_q29 $q27_q28_q29_q30 $q28_q29_q30_q31 $q29_q30_q31_q0 $q30_q31_q0_q1 $q31_q0_q1_q2 $q0_q1_q2 $q1_q2_q3 $q2_q3_q4 $q3_q4_q5 $q4_q5_q6 $q5_q6_q7 $q6_q7_q8 $q7_q8_q9 $q8_q9_q10 $q9_q10_q11 $q10_q11_q12 $q11_q12_q13 $q12_q13_q14 $q13_q14_q15 $q14_q15_q16 $q15_q16_q17 $q16_q17_q18 $q17_q18_q19 $q18_q19_q20 $q19_q20_q21 $q20_q21_q22 $q21_q22_q23 $q22_q23_q24 $q23_q24_q25 $q24_q25_q26 $q25_q26_q27 $q26_q27_q28 $q27_q28_q29 $q28_q29_q30 $q29_q30_q31 $q30_q31_q0 $q31_q0_q1 $x0_x1_x2_x3_x4_x5_x6_x7 $x2_x3_x4_x5_x6_x7_x8_x9 $x4_x5_x6_x7_x8_x9_x10_x11 $x6_x7_x8_x9_x10_x11_x12_x13 $x8_x9_x10_x11_x12_x13_x14_x15 $x10_x11_x12_x13_x14_x15_x16_x17 $x12_x13_x14_x15_x16_x17_x18_x19 $x14_x15_x16_x17_x18_x19_x20_x21 $x16_x17_x18_x19_x20_x21_x22_x23 $x18_x19_x20_x21_x22_x23_x24_x25 $w30_wzr $w0_w1 $w2_w3 $w4_w5 $w6_w7 $w8_w9 $w10_w11 $w12_w13 $w14_w15 $w16_w17 $w18_w19 $lr_xzr $x0_x1 $x2_x3 $x4_x5 $x6_x7 $x8_x9 $x10_x11 $x12_x13 $x14_x15 $x16_x17 $x18_x19 $z0_z1 $z1_z2 $z2_z3 $z3_z4 $z4_z5 $z5_z6 $z6_z7 $z7_z8 $z8_z9 $z9_z10 $z10_z11 $z11_z12 $z12_z13 $z13_z14 $z14_z15 $z15_z16 $z16_z17 $z17_z18 $z18_z19 $z19_z20 $z20_z21 $z21_z22 $z22_z23 $z23_z24 $z24_z25 $z25_z26 $z26_z27 $z27_z28 $z28_z29 $z29_z30 $z30_z31 $z31_z0 $z0_z1_z2_z3 $z1_z2_z3_z4 $z2_z3_z4_z5 $z3_z4_z5_z6 $z4_z5_z6_z7 $z5_z6_z7_z8 $z6_z7_z8_z9 $z7_z8_z9_z10 $z8_z9_z10_z11 $z9_z10_z11_z12 $z10_z11_z12_z13 $z11_z12_z13_z14 $z12_z13_z14_z15 $z13_z14_z15_z16 $z14_z15_z16_z17 $z15_z16_z17_z18 $z16_z17_z18_z19 $z17_z18_z19_z20 $z18_z19_z20_z21 $z19_z20_z21_z22 $z20_z21_z22_z23 $z21_z22_z23_z24 $z22_z23_z24_z25 $z23_z24_z25_z26 $z24_z25_z26_z27 $z25_z26_z27_z28 $z26_z27_z28_z29 $z27_z28_z29_z30 $z28_z29_z30_z31 $z29_z30_z31_z0 $z30_z31_z0_z1 $z31_z0_z1_z2 $z0_z1_z2 $z1_z2_z3 $z2_z3_z4 $z3_z4_z5 $z4_z5_z6 $z5_z6_z7 $z6_z7_z8 $z7_z8_z9 $z8_z9_z10 $z9_z10_z11 $z10_z11_z12 $z11_z12_z13 $z12_z13_z14 $z13_z14_z15 $z14_z15_z16 $z15_z16_z17 $z16_z17_z18 $z17_z18_z19 $z18_z19_z20 $z19_z20_z21 $z20_z21_z22 $z21_z22_z23 $z22_z23_z24 $z23_z24_z25 $z24_z25_z26 $z25_z26_z27 $z26_z27_z28 $z27_z28_z29 $z28_z29_z30 $z29_z30_z31 $z30_z31_z0 $z31_z0_z1 $z16_z24 $z17_z25 $z18_z26 $z19_z27 $z20_z28 $z21_z29 $z22_z30 $z23_z31 $z0_z8 $z1_z9 $z2_z10 $z3_z11 $z4_z12 $z5_z13 $z6_z14 $z7_z15 $z16_z20_z24_z28 $z17_z21_z25_z29 $z18_z22_z26_z30 $z19_z23_z27_z31 $z0_z4_z8_z12 $z1_z5_z9_z13 $z2_z6_z10_z14 $z3_z7_z11_z15
   call void @bar1()
   call void @bar2()
   ret void
 }
 define preserve_allcc void @foo() #0 {
-; CHECK: foo Clobbered Registers: $ffr $fpcr $fpsr $nzcv $sp $vg $wsp $za $b0 $b1 $b2 $b3 $b4 $b5 $b6 $b7 $d0 $d1 $d2 $d3 $d4 $d5 $d6 $d7 $h0 $h1 $h2 $h3 $h4 $h5 $h6 $h7 $p0 $p1 $p2 $p3 $p4 $p5 $p6 $p7 $p8 $p9 $p10 $p11 $p12 $p13 $p14 $p15 $pn0 $pn1 $pn2 $pn3 $pn4 $pn5 $pn6 $pn7 $pn8 $pn9 $pn10 $pn11 $pn12 $pn13 $pn14 $pn15 $q0 $q1 $q2 $q3 $q4 $q5 $q6 $q7 $s0 $s1 $s2 $s3 $s4 $s5 $s6 $s7 $w0 $w1 $w2 $w3 $w4 $w5 $w6 $w7 $w8 $w16 $w17 $w18 $x0 $x1 $x2 $x3 $x4 $x5 $x6 $x7 $x8 $x16 $x17 $x18 $z0 $z1 $z2 $z3 $z4 $z5 $z6 $z7 $z8 $z9 $z10 $z11 $z12 $z13 $z14 $z15 $z16 $z17 $z18 $z19 $z20 $z21 $z22 $z23 $z24 $z25 $z26 $z27 $z28 $z29 $z30 $z31 $zab0 $zad0 $zad1 $zad2 $zad3 $zad4 $zad5 $zad6 $zad7 $zah0 $zah1 $zaq0 $zaq1 $zaq2 $zaq3 $zaq4 $zaq5 $zaq6 $zaq7 $zaq8 $zaq9 $zaq10 $zaq11 $zaq12 $zaq13 $zaq14 $zaq15 $zas0 $zas1 $zas2 $zas3 $zt0 $d0_d1 $d1_d2 $d2_d3 $d3_d4 $d4_d5 $d5_d6 $d6_d7 $d7_d8 $d15_d16 $d16_d17 $d17_d18 $d18_d19 $d19_d20 $d20_d21 $d21_d22 $d22_d23 $d23_d24 $d24_d25 $d25_d26 $d26_d27 $d27_d28 $d28_d29 $d29_d30 $d30_d31 $d31_d0 $d0_d1_d2_d3 $d1_d2_d3_d4 $d2_d3_d4_d5 $d3_d4_d5_d6 $d4_d5_d6_d7 $d5_d6_d7_d8 $d6_d7_d8_d9 $d7_d8_d9_d10 $d13_d14_d15_d16 $d14_d15_d16_d17 $d15_d16_d17_d18 $d16_d17_d18_d19 $d17_d18_d19_d20 $d18_d19_d20_d21 $d19_d20_d21_d22 $d20_d21_d22_d23 $d21_d22_d23_d24 $d22_d23_d24_d25 $d23_d24_d25_d26 $d24_d25_d26_d27 $d25_d26_d27_d28 $d26_d27_d28_d29 $d27_d28_d29_d30 $d28_d29_d30_d31 $d29_d30_d31_d0 $d30_d31_d0_d1 $d31_d0_d1_d2 $d0_d1_d2 $d1_d2_d3 $d2_d3_d4 $d3_d4_d5 $d4_d5_d6 $d5_d6_d7 $d6_d7_d8 $d7_d8_d9 $d14_d15_d16 $d15_d16_d17 $d16_d17_d18 $d17_d18_d19 $d18_d19_d20 $d19_d20_d21 $d20_d21_d22 $d21_d22_d23 $d22_d23_d24 $d23_d24_d25 $d24_d25_d26 $d25_d26_d27 $d26_d27_d28 $d27_d28_d29 $d28_d29_d30 $d29_d30_d31 $d30_d31_d0 $d31_d0_d1 $p0_p1 $p1_p2 $p2_p3 $p3_p4 $p4_p5 $p5_p6 $p6_p7 $p7_p8 $p8_p9 $p9_p10 $p10_p11 $p11_p12 $p12_p13 $p13_p14 $p14_p15 $p15_p0 $q0_q1 $q1_q2 $q2_q3 $q3_q4 $q4_q5 $q5_q6 $q6_q7 $q7_q8 $q8_q9 $q9_q10 $q10_q11 $q11_q12 $q12_q13 $q13_q14 $q14_q15 $q15_q16 $q16_q17 $q17_q18 $q18_q19 $q19_q20 $q20_q21 $q21_q22 $q22_q23 $q23_q24 $q24_q25 $q25_q26 $q26_q27 $q27_q28 $q28_q29 $q29_q30 $q30_q31 $q31_q0 $q0_q1_q2_q3 $q1_q2_q3_q4 $q2_q3_q4_q5 $q3_q4_q5_q6 $q4_q5_q6_q7 $q5_q6_q7_q8 $q6_q7_q8_q9 $q7_q8_q9_q10 $q8_q9_q10_q11 $q9_q10_q11_q12 $q10_q11_q12_q13 $q11_q12_q13_q14 $q12_q13_q14_q15 $q13_q14_q15_q16 $q14_q15_q16_q17 $q15_q16_q17_q18 $q16_q17_q18_q19 $q17_q18_q19_q20 $q18_q19_q20_q21 $q19_q20_q21_q22 $q20_q21_q22_q23 $q21_q22_q23_q24 $q22_q23_q24_q25 $q23_q24_q25_q26 $q24_q25_q26_q27 $q25_q26_q27_q28 $q26_q27_q28_q29 $q27_q28_q29_q30 $q28_q29_q30_q31 $q29_q30_q31_q0 $q30_q31_q0_q1 $q31_q0_q1_q2 $q0_q1_q2 $q1_q2_q3 $q2_q3_q4 $q3_q4_q5 $q4_q5_q6 $q5_q6_q7 $q6_q7_q8 $q7_q8_q9 $q8_q9_q10 $q9_q10_q11 $q10_q11_q12 $q11_q12_q13 $q12_q13_q14 $q13_q14_q15 $q14_q15_q16 $q15_q16_q17 $q16_q17_q18 $q17_q18_q19 $q18_q19_q20 $q19_q20_q21 $q20_q21_q22 $q21_q22_q23 $q22_q23_q24 $q23_q24_q25 $q24_q25_q26 $q25_q26_q27 $q26_q27_q28 $q27_q28_q29 $q28_q29_q30 $q29_q30_q31 $q30_q31_q0 $q31_q0_q1 $x0_x1_x2_x3_x4_x5_x6_x7 $x2_x3_x4_x5_x6_x7_x8_x9 $x4_x5_x6_x7_x8_x9_x10_x11 $x6_x7_x8_x9_x10_x11_x12_x13 $x8_x9_x10_x11_x12_x13_x14_x15 $x10_x11_x12_x13_x14_x15_x16_x17 $x12_x13_x14_x15_x16_x17_x18_x19 $x14_x15_x16_x17_x18_x19_x20_x21 $x16_x17_x18_x19_x20_x21_x22_x23 $x18_x19_x20_x21_x22_x23_x24_x25 $w30_wzr $w0_w1 $w2_w3 $w4_w5 $w6_w7 $w8_w9 $w10_w11 $w12_w13 $w14_w15 $w16_w17 $w18_w19 $lr_xzr $x0_x1 $x2_x3 $x4_x5 $x6_x7 $x8_x9 $x10_x11 $x12_x13 $x14_x15 $x16_x17 $x18_x19 $z0_z1 $z1_z2 $z2_z3 $z3_z4 $z4_z5 $z5_z6 $z6_z7 $z7_z8 $z8_z9 $z9_z10 $z10_z11 $z11_z12 $z12_z13 $z13_z14 $z14_z15 $z15_z16 $z16_z17 $z17_z18 $z18_z19 $z19_z20 $z20_z21 $z21_z22 $z22_z23 $z23_z24 $z24_z25 $z25_z26 $z26_z27 $z27_z28 $z28_z29 $z29_z30 $z30_z31 $z31_z0 $z0_z1_z2_z3 $z1_z2_z3_z4 $z2_z3_z4_z5 $z3_z4_z5_z6 $z4_z5_z6_z7 $z5_z6_z7_z8 $z6_z7_z8_z9 $z7_z8_z9_z10 $z8_z9_z10_z11 $z9_z10_z11_z12 $z10_z11_z12_z13 $z11_z12_z13_z14 $z12_z13_z14_z15 $z13_z14_z15_z16 $z14_z15_z16_z17 $z15_z16_z17_z18 $z16_z17_z18_z19 $z17_z18_z19_z20 $z18_z19_z20_z21 $z19_z20_z21_z22 $z20_z21_z22_z23 $z21_z22_z23_z24 $z22_z23_z24_z25 $z23_z24_z25_z26 $z24_z25_z26_z27 $z25_z26_z27_z28 $z26_z27_z28_z29 $z27_z28_z29_z30 $z28_z29_z30_z31 $z29_z30_z31_z0 $z30_z31_z0_z1 $z31_z0_z1_z2 $z0_z1_z2 $z1_z2_z3 $z2_z3_z4 $z3_z4_z5 $z4_z5_z6 $z5_z6_z7 $z6_z7_z8 $z7_z8_z9 $z8_z9_z10 $z9_z10_z11 $z10_z11_z12 $z11_z12_z13 $z12_z13_z14 $z13_z14_z15 $z14_z15_z16 $z15_z16_z17 $z16_z17_z18 $z17_z18_z19 $z18_z19_z20 $z19_z20_z21 $z20_z21_z22 $z21_z22_z23 $z22_z23_z24 $z23_z24_z25 $z24_z25_z26 $z25_z26_z27 $z26_z27_z28 $z27_z28_z29 $z28_z29_z30 $z29_z30_z31 $z30_z31_z0 $z31_z0_z1 $z16_z24 $z17_z25 $z18_z26 $z19_z27 $z20_z28 $z21_z29 $z22_z30 $z23_z31 $z0_z8 $z1_z9 $z2_z10 $z3_z11 $z4_z12 $z5_z13 $z6_z14 $z7_z15 $z16_z20_z24_z28 $z17_z21_z25_z29 $z18_z22_z26_z30 $z19_z23_z27_z31 $z0_z4_z8_z12 $z1_z5_z9_z13 $z2_z6_z10_z14 $z3_z7_z11_z15
+; CHECK: foo Clobbered Registers: $ffr $fpcr $fpmr $fpsr $nzcv $sp $vg $wsp $za $b0 $b1 $b2 $b3 $b4 $b5 $b6 $b7 $d0 $d1 $d2 $d3 $d4 $d5 $d6 $d7 $h0 $h1 $h2 $h3 $h4 $h5 $h6 $h7 $p0 $p1 $p2 $p3 $p4 $p5 $p6 $p7 $p8 $p9 $p10 $p11 $p12 $p13 $p14 $p15 $pn0 $pn1 $pn2 $pn3 $pn4 $pn5 $pn6 $pn7 $pn8 $pn9 $pn10 $pn11 $pn12 $pn13 $pn14 $pn15 $q0 $q1 $q2 $q3 $q4 $q5 $q6 $q7 $s0 $s1 $s2 $s3 $s4 $s5 $s6 $s7 $w0 $w1 $w2 $w3 $w4 $w5 $w6 $w7 $w8 $w16 $w17 $w18 $x0 $x1 $x2 $x3 $x4 $x5 $x6 $x7 $x8 $x16 $x17 $x18 $z0 $z1 $z2 $z3 $z4 $z5 $z6 $z7 $z8 $z9 $z10 $z11 $z12 $z13 $z14 $z15 $z16 $z17 $z18 $z19 $z20 $z21 $z22 $z23 $z24 $z25 $z26 $z27 $z28 $z29 $z30 $z31 $zab0 $zad0 $zad1 $zad2 $zad3 $zad4 $zad5 $zad6 $zad7 $zah0 $zah1 $zaq0 $zaq1 $zaq2 $zaq3 $zaq4 $zaq5 $zaq6 $zaq7 $zaq8 $zaq9 $zaq10 $zaq11 $zaq12 $zaq13 $zaq14 $zaq15 $zas0 $zas1 $zas2 $zas3 $zt0 $d0_d1 $d1_d2 $d2_d3 $d3_d4 $d4_d5 $d5_d6 $d6_d7 $d7_d8 $d15_d16 $d16_d17 $d17_d18 $d18_d19 $d19_d20 $d20_d21 $d21_d22 $d22_d23 $d23_d24 $d24_d25 $d25_d26 $d26_d27 $d27_d28 $d28_d29 $d29_d30 $d30_d31 $d31_d0 $d0_d1_d2_d3 $d1_d2_d3_d4 $d2_d3_d4_d5 $d3_d4_d5_d6 $d4_d5_d6_d7 $d5_d6_d7_d8 $d6_d7_d8_d9 $d7_d8_d9_d10 $d13_d14_d15_d16 $d14_d15_d16_d17 $d15_d16_d17_d18 $d16_d17_d18_d19 $d17_d18_d19_d20 $d18_d19_d20_d21 $d19_d20_d21_d22 $d20_d21_d22_d23 $d21_d22_d23_d24 $d22_d23_d24_d25 $d23_d24_d25_d26 $d24_d25_d26_d27 $d25_d26_d27_d28 $d26_d27_d28_d29 $d27_d28_d29_d30 $d28_d29_d30_d31 $d29_d30_d31_d0 $d30_d31_d0_d1 $d31_d0_d1_d2 $d0_d1_d2 $d1_d2_d3 $d2_d3_d4 $d3_d4_d5 $d4_d5_d6 $d5_d6_d7 $d6_d7_d8 $d7_d8_d9 $d14_d15_d16 $d15_d16_d17 $d16_d17_d18 $d17_d18_d19 $d18_d19_d20 $d19_d20_d21 $d20_d21_d22 $d21_d22_d23 $d22_d23_d24 $d23_d24_d25 $d24_d25_d26 $d25_d26_d27 $d26_d27_d28 $d27_d28_d29 $d28_d29_d30 $d29_d30_d31 $d30_d31_d0 $d31_d0_d1 $p0_p1 $p1_p2 $p2_p3 $p3_p4 $p4_p5 $p5_p6 $p6_p7 $p7_p8 $p8_p9 $p9_p10 $p10_p11 $p11_p12 $p12_p13 $p13_p14 $p14_p15 $p15_p0 $q0_q1 $q1_q2 $q2_q3 $q3_q4 $q4_q5 $q5_q6 $q6_q7 $q7_q8 $q8_q9 $q9_q10 $q10_q11 $q11_q12 $q12_q13 $q13_q14 $q14_q15 $q15_q16 $q16_q17 $q17_q18 $q18_q19 $q19_q20 $q20_q21 $q21_q22 $q22_q23 $q23_q24 $q24_q25 $q25_q26 $q26_q27 $q27_q28 $q28_q29 $q29_q30 $q30_q31 $q31_q0 $q0_q1_q2_q3 $q1_q2_q3_q4 $q2_q3_q4_q5 $q3_q4_q5_q6 $q4_q5_q6_q7 $q5_q6_q7_q8 $q6_q7_q8_q9 $q7_q8_q9_q10 $q8_q9_q10_q11 $q9_q10_q11_q12 $q10_q11_q12_q13 $q11_q12_q13_q14 $q12_q13_q14_q15 $q13_q14_q15_q16 $q14_q15_q16_q17 $q15_q16_q17_q18 $q16_q17_q18_q19 $q17_q18_q19_q20 $q18_q19_q20_q21 $q19_q20_q21_q22 $q20_q21_q22_q23 $q21_q22_q23_q24 $q22_q23_q24_q25 $q23_q24_q25_q26 $q24_q25_q26_q27 $q25_q26_q27_q28 $q26_q27_q28_q29 $q27_q28_q29_q30 $q28_q29_q30_q31 $q29_q30_q31_q0 $q30_q31_q0_q1 $q31_q0_q1_q2 $q0_q1_q2 $q1_q2_q3 $q2_q3_q4 $q3_q4_q5 $q4_q5_q6 $q5_q6_q7 $q6_q7_q8 $q7_q8_q9 $q8_q9_q10 $q9_q10_q11 $q10_q11_q12 $q11_q12_q13 $q12_q13_q14 $q13_q14_q15 $q14_q15_q16 $q15_q16_q17 $q16_q17_q18 $q17_q18_q19 $q18_q19_q20 $q19_q20_q21 $q20_q21_q22 $q21_q22_q23 $q22_q23_q24 $q23_q24_q25 $q24_q25_q26 $q25_q26_q27 $q26_q27_q28 $q27_q28_q29 $q28_q29_q30 $q29_q30_q31 $q30_q31_q0 $q31_q0_q1 $x0_x1_x2_x3_x4_x5_x6_x7 $x2_x3_x4_x5_x6_x7_x8_x9 $x4_x5_x6_x7_x8_x9_x10_x11 $x6_x7_x8_x9_x10_x11_x12_x13 $x8_x9_x10_x11_x12_x13_x14_x15 $x10_x11_x12_x13_x14_x15_x16_x17 $x12_x13_x14_x15_x16_x17_x18_x19 $x14_x15_x16_x17_x18_x19_x20_x21 $x16_x17_x18_x19_x20_x21_x22_x23 $x18_x19_x20_x21_x22_x23_x24_x25 $w30_wzr $w0_w1 $w2_w3 $w4_w5 $w6_w7 $w8_w9 $w10_w11 $w12_w13 $w14_w15 $w16_w17 $w18_w19 $lr_xzr $x0_x1 $x2_x3 $x4_x5 $x6_x7 $x8_x9 $x10_x11 $x12_x13 $x14_x15 $x16_x17 $x18_x19 $z0_z1 $z1_z2 $z2_z3 $z3_z4 $z4_z5 $z5_z6 $z6_z7 $z7_z8 $z8_z9 $z9_z10 $z10_z11 $z11_z12 $z12_z13 $z13_z14 $z14_z15 $z15_z16 $z16_z17 $z17_z18 $z18_z19 $z19_z20 $z20_z21 $z21_z22 $z22_z23 $z23_z24 $z24_z25 $z25_z26 $z26_z27 $z27_z28 $z28_z29 $z29_z30 $z30_z31 $z31_z0 $z0_z1_z2_z3 $z1_z2_z3_z4 $z2_z3_z4_z5 $z3_z4_z5_z6 $z4_z5_z6_z7 $z5_z6_z7_z8 $z6_z7_z8_z9 $z7_z8_z9_z10 $z8_z9_z10_z11 $z9_z10_z11_z12 $z10_z11_z12_z13 $z11_z12_z13_z14 $z12_z13_z14_z15 $z13_z14_z15_z16 $z14_z15_z16_z17 $z15_z16_z17_z18 $z16_z17_z18_z19 $z17_z18_z19_z20 $z18_z19_z20_z21 $z19_z20_z21_z22 $z20_z21_z22_z23 $z21_z22_z23_z24 $z22_z23_z24_z25 $z23_z24_z25_z26 $z24_z25_z26_z27 $z25_z26_z27_z28 $z26_z27_z28_z29 $z27_z28_z29_z30 $z28_z29_z30_z31 $z29_z30_z31_z0 $z30_z31_z0_z1 $z31_z0_z1_z2 $z0_z1_z2 $z1_z2_z3 $z2_z3_z4 $z3_z4_z5 $z4_z5_z6 $z5_z6_z7 $z6_z7_z8 $z7_z8_z9 $z8_z9_z10 $z9_z10_z11 $z10_z11_z12 $z11_z12_z13 $z12_z13_z14 $z13_z14_z15 $z14_z15_z16 $z15_z16_z17 $z16_z17_z18 $z17_z18_z19 $z18_z19_z20 $z19_z20_z21 $z20_z21_z22 $z21_z22_z23 $z22_z23_z24 $z23_z24_z25 $z24_z25_z26 $z25_z26_z27 $z26_z27_z28 $z27_z28_z29 $z28_z29_z30 $z29_z30_z31 $z30_z31_z0 $z31_z0_z1 $z16_z24 $z17_z25 $z18_z26 $z19_z27 $z20_z28 $z21_z29 $z22_z30 $z23_z31 $z0_z8 $z1_z9 $z2_z10 $z3_z11 $z4_z12 $z5_z13 $z6_z14 $z7_z15 $z16_z20_z24_z28 $z17_z21_z25_z29 $z18_z22_z26_z30 $z19_z23_z27_z31 $z0_z4_z8_z12 $z1_z5_z9_z13 $z2_z6_z10_z14 $z3_z7_z11_z15
   call void @bar1()
   call void @bar2()
   ret void
 }
 define preserve_nonecc void @qux() #0 {
-;  CHECK: qux Clobbered Registers: $ffr $fpcr $fpsr $nzcv $sp $vg $wsp $za $b0 $b1 $b2 $b3 $b4 $b5 $b6 $b7 $b16 $b17 $b18 $b19 $b20 $b21 $b22 $b23 $b24 $b25 $b26 $b27 $b28 $b29 $b30 $b31 $d0 $d1 $d2 $d3 $d4 $d5 $d6 $d7 $d16 $d17 $d18 $d19 $d20 $d21 $d22 $d23 $d24 $d25 $d26 $d27 $d28 $d29 $d30 $d31 $h0 $h1 $h2 $h3 $h4 $h5 $h6 $h7 $h16 $h17 $h18 $h19 $h20 $h21 $h22 $h23 $h24 $h25 $h26 $h27 $h28 $h29 $h30 $h31 $p0 $p1 $p2 $p3 $p4 $p5 $p6 $p7 $p8 $p9 $p10 $p11 $p12 $p13 $p14 $p15 $pn0 $pn1 $pn2 $pn3 $pn4 $pn5 $pn6 $pn7 $pn8 $pn9 $pn10 $pn11 $pn12 $pn13 $pn14 $pn15 $q0 $q1 $q2 $q3 $q4 $q5 $q6 $q7 $q8 $q9 $q10 $q11 $q12 $q13 $q14 $q15 $q16 $q17 $q18 $q19 $q20 $q21 $q22 $q23 $q24 $q25 $q26 $q27 $q28 $q29 $q30 $q31 $s0 $s1 $s2 $s3 $s4 $s5 $s6 $s7 $s16 $s17 $s18 $s19 $s20 $s21 $s22 $s23 $s24 $s25 $s26 $s27 $s28 $s29 $s30 $s31 $w0 $w1 $w2 $w3 $w4 $w5 $w6 $w7 $w8 $w9 $w10 $w11 $w12 $w13 $w14 $w15 $w16 $w17 $w18 $x0 $x1 $x2 $x3 $x4 $x5 $x6 $x7 $x8 $x9 $x10 $x11 $x12 $x13 $x14 $x15 $x16 $x17 $x18 $z0 $z1 $z2 $z3 $z4 $z5 $z6 $z7 $z8 $z9 $z10 $z11 $z12 $z13 $z14 $z15 $z16 $z17 $z18 $z19 $z20 $z21 $z22 $z23 $z24 $z25 $z26 $z27 $z28 $z29 $z30 $z31 $zab0 $zad0 $zad1 $zad2 $zad3 $zad4 $zad5 $zad6 $zad7 $zah0 $zah1 $zaq0 $zaq1 $zaq2 $zaq3 $zaq4 $zaq5 $zaq6 $zaq7 $zaq8 $zaq9 $zaq10 $zaq11 $zaq12 $zaq13 $zaq14 $zaq15 $zas0 $zas1 $zas2 $zas3 $zt0 $d0_d1 $d1_d2 $d2_d3 $d3_d4 $d4_d5 $d5_d6 $d6_d7 $d7_d8 $d15_d16 $d16_d17 $d17_d18 $d18_d19 $d19_d20 $d20_d21 $d21_d22 $d22_d23 $d23_d24 $d24_d25 $d25_d26 $d26_d27 $d27_d28 $d28_d29 $d29_d30 $d30_d31 $d31_d0 $d0_d1_d2_d3 $d1_d2_d3_d4 $d2_d3_d4_d5 $d3_d4_d5_d6 $d4_d5_d6_d7 $d5_d6_d7_d8 $d6_d7_d8_d9 $d7_d8_d9_d10 $d13_d14_d15_d16 $d14_d15_d16_d17 $d15_d16_d17_d18 $d16_d17_d18_d19 $d17_d18_d19_d20 $d18_d19_d20_d21 $d19_d20_d21_d22 $d20_d21_d22_d23 $d21_d22_d23_d24 $d22_d23_d24_d25 $d23_d24_d25_d26 $d24_d25_d26_d27 $d25_d26_d27_d28 $d26_d27_d28_d29 $d27_d28_d29_d30 $d28_d29_d30_d31 $d29_d30_d31_d0 $d30_d31_d0_d1 $d31_d0_d1_d2 $d0_d1_d2 $d1_d2_d3 $d2_d3_d4 $d3_d4_d5 $d4_d5_d6 $d5_d6_d7 $d6_d7_d8 $d7_d8_d9 $d14_d15_d16 $d15_d16_d17 $d16_d17_d18 $d17_d18_d19 $d18_d19_d20 $d19_d20_d21 $d20_d21_d22 $d21_d22_d23 $d22_d23_d24 $d23_d24_d25 $d24_d25_d26 $d25_d26_d27 $d26_d27_d28 $d27_d28_d29 $d28_d29_d30 $d29_d30_d31 $d30_d31_d0 $d31_d0_d1 $p0_p1 $p1_p2 $p2_p3 $p3_p4 $p4_p5 $p5_p6 $p6_p7 $p7_p8 $p8_p9 $p9_p10 $p10_p11 $p11_p12 $p12_p13 $p13_p14 $p14_p15 $p15_p0 $q0_q1 $q1_q2 $q2_q3 $q3_q4 $q4_q5 $q5_q6 $q6_q7 $q7_q8 $q8_q9 $q9_q10 $q10_q11 $q11_q12 $q12_q13 $q13_q14 $q14_q15 $q15_q16 $q16_q17 $q17_q18 $q18_q19 $q19_q20 $q20_q21 $q21_q22 $q22_q23 $q23_q24 $q24_q25 $q25_q26 $q26_q27 $q27_q28 $q28_q29 $q29_q30 $q30_q31 $q31_q0 $q0_q1_q2_q3 $q1_q2_q3_q4 $q2_q3_q4_q5 $q3_q4_q5_q6 $q4_q5_q6_q7 $q5_q6_q7_q8 $q6_q7_q8_q9 $q7_q8_q9_q10 $q8_q9_q10_q11 $q9_q10_q11_q12 $q10_q11_q12_q13 $q11_q12_q13_q14 $q12_q13_q14_q15 $q13_q14_q15_q16 $q14_q15_q16_q17 $q15_q16_q17_q18 $q16_q17_q18_q19 $q17_q18_q19_q20 $q18_q19_q20_q21 $q19_q20_q21_q22 $q20_q21_q22_q23 $q21_q22_q23_q24 $q22_q23_q24_q25 $q23_q24_q25_q26 $q24_q25_q26_q27 $q25_q26_q27_q28 $q26_q27_q28_q29 $q27_q28_q29_q30 $q28_q29_q30_q31 $q29_q30_q31_q0 $q30_q31_q0_q1 $q31_q0_q1_q2 $q0_q1_q2 $q1_q2_q3 $q2_q3_q4 $q3_q4_q5 $q4_q5_q6 $q5_q6_q7 $q6_q7_q8 $q7_q8_q9 $q8_q9_q10 $q9_q10_q11 $q10_q11_q12 $q11_q12_q13 $q12_q13_q14 $q13_q14_q15 $q14_q15_q16 $q15_q16_q17 $q16_q17_q18 $q17_q18_q19 $q18_q19_q20 $q19_q20_q21 $q20_q21_q22 $q21_q22_q23 $q22_q23_q24 $q23_q24_q25 $q24_q25_q26 $q25_q26_q27 $q26_q27_q28 $q27_q28_q29 $q28_q29_q30 $q29_q30_q31 $q30_q31_q0 $q31_q0_q1 $x0_x1_x2_x3_x4_x5_x6_x7 $x2_x3_x4_x5_x6_x7_x8_x9 $x4_x5_x6_x7_x8_x9_x10_x11 $x6_x7_x8_x9_x10_x11_x12_x13 $x8_x9_x10_x11_x12_x13_x14_x15 $x10_x11_x12_x13_x14_x15_x16_x17 $x12_x13_x14_x15_x16_x17_x18_x19 $x14_x15_x16_x17_x18_x19_x20_x21 $x16_x17_x18_x19_x20_x21_x22_x23 $x18_x19_x20_x21_x22_x23_x24_x25 $w30_wzr $w0_w1 $w2_w3 $w4_w5 $w6_w7 $w8_w9 $w10_w11 $w12_w13 $w14_w15 $w16_w17 $w18_w19 $lr_xzr $x0_x1 $x2_x3 $x4_x5 $x6_x7 $x8_x9 $x10_x11 $x12_x13 $x14_x15 $x16_x17 $x18_x19 $z0_z1 $z1_z2 $z2_z3 $z3_z4 $z4_z5 $z5_z6 $z6_z7 $z7_z8 $z8_z9 $z9_z10 $z10_z11 $z11_z12 $z12_z13 $z13_z14 $z14_z15 $z15_z16 $z16_z17 $z17_z18 $z18_z19 $z19_z20 $z20_z21 $z21_z22 $z22_z23 $z23_z24 $z24_z25 $z25_z26 $z26_z27 $z27_z28 $z28_z29 $z29_z30 $z30_z31 $z31_z0 $z0_z1_z2_z3 $z1_z2_z3_z4 $z2_z3_z4_z5 $z3_z4_z5_z6 $z4_z5_z6_z7 $z5_z6_z7_z8 $z6_z7_z8_z9 $z7_z8_z9_z10 $z8_z9_z10_z11 $z9_z10_z11_z12 $z10_z11_z12_z13 $z11_z12_z13_z14 $z12_z13_z14_z15 $z13_z14_z15_z16 $z14_z15_z16_z17 $z15_z16_z17_z18 $z16_z17_z18_z19 $z17_z18_z19_z20 $z18_z19_z20_z21 $z19_z20_z21_z22 $z20_z21_z22_z23 $z21_z22_z23_z24 $z22_z23_z24_z25 $z23_z24_z25_z26 $z24_z25_z26_z27 $z25_z26_z27_z28 $z26_z27_z28_z29 $z27_z28_z29_z30 $z28_z29_z30_z31 $z29_z30_z31_z0 $z30_z31_z0_z1 $z31_z0_z1_z2 $z0_z1_z2 $z1_z2_z3 $z2_z3_z4 $z3_z4_z5 $z4_z5_z6 $z5_z6_z7 $z6_z7_z8 $z7_z8_z9 $z8_z9_z10 $z9_z10_z11 $z10_z11_z12 $z11_z12_z13 $z12_z13_z14 $z13_z14_z15 $z14_z15_z16 $z15_z16_z17 $z16_z17_z18 $z17_z18_z19 $z18_z19_z20 $z19_z20_z21 $z20_z21_z22 $z21_z22_z23 $z22_z23_z24 $z23_z24_z25 $z24_z25_z26 $z25_z26_z27 $z26_z27_z28 $z27_z28_z29 $z28_z29_z30 $z29_z30_z31 $z30_z31_z0 $z31_z0_z1 $z16_z24 $z17_z25 $z18_z26 $z19_z27 $z20_z28 $z21_z29 $z22_z30 $z23_z31 $z0_z8 $z1_z9 $z2_z10 $z3_z11 $z4_z12 $z5_z13 $z6_z14 $z7_z15 $z16_z20_z24_z28 $z17_z21_z25_z29 $z18_z22_z26_z30 $z19_z23_z27_z31 $z0_z4_z8_z12 $z1_z5_z9_z13 $z2_z6_z10_z14 $z3_z7_z11_z15
+;  CHECK: qux Clobbered Registers: $ffr $fpcr $fpmr $fpsr $nzcv $sp $vg $wsp $za $b0 $b1 $b2 $b3 $b4 $b5 $b6 $b7 $b16 $b17 $b18 $b19 $b20 $b21 $b22 $b23 $b24 $b25 $b26 $b27 $b28 $b29 $b30 $b31 $d0 $d1 $d2 $d3 $d4 $d5 $d6 $d7 $d16 $d17 $d18 $d19 $d20 $d21 $d22 $d23 $d24 $d25 $d26 $d27 $d28 $d29 $d30 $d31 $h0 $h1 $h2 $h3 $h4 $h5 $h6 $h7 $h16 $h17 $h18 $h19 $h20 $h21 $h22 $h23 $h24 $h25 $h26 $h27 $h28 $h29 $h30 $h31 $p0 $p1 $p2 $p3 $p4 $p5 $p6 $p7 $p8 $p9 $p10 $p11 $p12 $p13 $p14 $p15 $pn0 $pn1 $pn2 $pn3 $pn4 $pn5 $pn6 $pn7 $pn8 $pn9 $pn10 $pn11 $pn12 $pn13 $pn14 $pn15 $q0 $q1 $q2 $q3 $q4 $q5 $q6 $q7 $q8 $q9 $q10 $q11 $q12 $q13 $q14 $q15 $q16 $q17 $q18 $q19 $q20 $q21 $q22 $q23 $q24 $q25 $q26 $q27 $q28 $q29 $q30 $q31 $s0 $s1 $s2 $s3 $s4 $s5 $s6 $s7 $s16 $s17 $s18 $s19 $s20 $s21 $s22 $s23 $s24 $s25 $s26 $s27 $s28 $s29 $s30 $s31 $w0 $w1 $w2 $w3 $w4 $w5 $w6 $w7 $w8 $w9 $w10 $w11 $w12 $w13 $w14 $w15 $w16 $w17 $w18 $x0 $x1 $x2 $x3 $x4 $x5 $x6 $x7 $x8 $x9 $x10 $x11 $x12 $x13 $x14 $x15 $x16 $x17 $x18 $z0 $z1 $z2 $z3 $z4 $z5 $z6 $z7 $z8 $z9 $z10 $z11 $z12 $z13 $z14 $z15 $z16 $z17 $z18 $z19 $z20 $z21 $z22 $z23 $z24 $z25 $z26 $z27 $z28 $z29 $z30 $z31 $zab0 $zad0 $zad1 $zad2 $zad3 $zad4 $zad5 $zad6 $zad7 $zah0 $zah1 $zaq0 $zaq1 $zaq2 $zaq3 $zaq4 $zaq5 $zaq6 $zaq7 $zaq8 $zaq9 $zaq10 $zaq11 $zaq12 $zaq13 $zaq14 $zaq15 $zas0 $zas1 $zas2 $zas3 $zt0 $d0_d1 $d1_d2 $d2_d3 $d3_d4 $d4_d5 $d5_d6 $d6_d7 $d7_d8 $d15_d16 $d16_d17 $d17_d18 $d18_d19 $d19_d20 $d20_d21 $d21_d22 $d22_d23 $d23_d24 $d24_d25 $d25_d26 $d26_d27 $d27_d28 $d28_d29 $d29_d30 $d30_d31 $d31_d0 $d0_d1_d2_d3 $d1_d2_d3_d4 $d2_d3_d4_d5 $d3_d4_d5_d6 $d4_d5_d6_d7 $d5_d6_d7_d8 $d6_d7_d8_d9 $d7_d8_d9_d10 $d13_d14_d15_d16 $d14_d15_d16_d17 $d15_d16_d17_d18 $d16_d17_d18_d19 $d17_d18_d19_d20 $d18_d19_d20_d21 $d19_d20_d21_d22 $d20_d21_d22_d23 $d21_d22_d23_d24 $d22_d23_d24_d25 $d23_d24_d25_d26 $d24_d25_d26_d27 $d25_d26_d27_d28 $d26_d27_d28_d29 $d27_d28_d29_d30 $d28_d29_d30_d31 $d29_d30_d31_d0 $d30_d31_d0_d1 $d31_d0_d1_d2 $d0_d1_d2 $d1_d2_d3 $d2_d3_d4 $d3_d4_d5 $d4_d5_d6 $d5_d6_d7 $d6_d7_d8 $d7_d8_d9 $d14_d15_d16 $d15_d16_d17 $d16_d17_d18 $d17_d18_d19 $d18_d19_d20 $d19_d20_d21 $d20_d21_d22 $d21_d22_d23 $d22_d23_d24 $d23_d24_d25 $d24_d25_d26 $d25_d26_d27 $d26_d27_d28 $d27_d28_d29 $d28_d29_d30 $d29_d30_d31 $d30_d31_d0 $d31_d0_d1 $p0_p1 $p1_p2 $p2_p3 $p3_p4 $p4_p5 $p5_p6 $p6_p7 $p7_p8 $p8_p9 $p9_p10 $p10_p11 $p11_p12 $p12_p13 $p13_p14 $p14_p15 $p15_p0 $q0_q1 $q1_q2 $q2_q3 $q3_q4 $q4_q5 $q5_q6 $q6_q7 $q7_q8 $q8_q9 $q9_q10 $q10_q11 $q11_q12 $q12_q13 $q13_q14 $q14_q15 $q15_q16 $q16_q17 $q17_q18 $q18_q19 $q19_q20 $q20_q21 $q21_q22 $q22_q23 $q23_q24 $q24_q25 $q25_q26 $q26_q27 $q27_q28 $q28_q29 $q29_q30 $q30_q31 $q31_q0 $q0_q1_q2_q3 $q1_q2_q3_q4 $q2_q3_q4_q5 $q3_q4_q5_q6 $q4_q5_q6_q7 $q5_q6_q7_q8 $q6_q7_q8_q9 $q7_q8_q9_q10 $q8_q9_q10_q11 $q9_q10_q11_q12 $q10_q11_q12_q13 $q11_q12_q13_q14 $q12_q13_q14_q15 $q13_q14_q15_q16 $q14_q15_q16_q17 $q15_q16_q17_q18 $q16_q17_q18_q19 $q17_q18_q19_q20 $q18_q19_q20_q21 $q19_q20_q21_q22 $q20_q21_q22_q23 $q21_q22_q23_q24 $q22_q23_q24_q25 $q23_q24_q25_q26 $q24_q25_q26_q27 $q25_q26_q27_q28 $q26_q27_q28_q29 $q27_q28_q29_q30 $q28_q29_q30_q31 $q29_q30_q31_q0 $q30_q31_q0_q1 $q31_q0_q1_q2 $q0_q1_q2 $q1_q2_q3 $q2_q3_q4 $q3_q4_q5 $q4_q5_q6 $q5_q6_q7 $q6_q7_q8 $q7_q8_q9 $q8_q9_q10 $q9_q10_q11 $q10_q11_q12 $q11_q12_q13 $q12_q13_q14 $q13_q14_q15 $q14_q15_q16 $q15_q16_q17 $q16_q17_q18 $q17_q18_q19 $q18_q19_q20 $q19_q20_q21 $q20_q21_q22 $q21_q22_q23 $q22_q23_q24 $q23_q24_q25 $q24_q25_q26 $q25_q26_q27 $q26_q27_q28 $q27_q28_q29 $q28_q29_q30 $q29_q30_q31 $q30_q31_q0 $q31_q0_q1 $x0_x1_x2_x3_x4_x5_x6_x7 $x2_x3_x4_x5_x6_x7_x8_x9 $x4_x5_x6_x7_x8_x9_x10_x11 $x6_x7_x8_x9_x10_x11_x12_x13 $x8_x9_x10_x11_x12_x13_x14_x15 $x10_x11_x12_x13_x14_x15_x16_x17 $x12_x13_x14_x15_x16_x17_x18_x19 $x14_x15_x16_x17_x18_x19_x20_x21 $x16_x17_x18_x19_x20_x21_x22_x23 $x18_x19_x20_x21_x22_x23_x24_x25 $w30_wzr $w0_w1 $w2_w3 $w4_w5 $w6_w7 $w8_w9 $w10_w11 $w12_w13 $w14_w15 $w16_w17 $w18_w19 $lr_xzr $x0_x1 $x2_x3 $x4_x5 $x6_x7 $x8_x9 $x10_x11 $x12_x13 $x14_x15 $x16_x17 $x18_x19 $z0_z1 $z1_z2 $z2_z3 $z3_z4 $z4_z5 $z5_z6 $z6_z7 $z7_z8 $z8_z9 $z9_z10 $z10_z11 $z11_z12 $z12_z13 $z13_z14 $z14_z15 $z15_z16 $z16_z17 $z17_z18 $z18_z19 $z19_z20 $z20_z21 $z21_z22 $z22_z23 $z23_z24 $z24_z25 $z25_z26 $z26_z27 $z27_z28 $z28_z29 $z29_z30 $z30_z31 $z31_z0 $z0_z1_z2_z3 $z1_z2_z3_z4 $z2_z3_z4_z5 $z3_z4_z5_z6 $z4_z5_z6_z7 $z5_z6_z7_z8 $z6_z7_z8_z9 $z7_z8_z9_z10 $z8_z9_z10_z11 $z9_z10_z11_z12 $z10_z11_z12_z13 $z11_z12_z13_z14 $z12_z13_z14_z15 $z13_z14_z15_z16 $z14_z15_z16_z17 $z15_z16_z17_z18 $z16_z17_z18_z19 $z17_z18_z19_z20 $z18_z19_z20_z21 $z19_z20_z21_z22 $z20_z21_z22_z23 $z21_z22_z23_z24 $z22_z23_z24_z25 $z23_z24_z25_z26 $z24_z25_z26_z27 $z25_z26_z27_z28 $z26_z27_z28_z29 $z27_z28_z29_z30 $z28_z29_z30_z31 $z29_z30_z31_z0 $z30_z31_z0_z1 $z31_z0_z1_z2 $z0_z1_z2 $z1_z2_z3 $z2_z3_z4 $z3_z4_z5 $z4_z5_z6 $z5_z6_z7 $z6_z7_z8 $z7_z8_z9 $z8_z9_z10 $z9_z10_z11 $z10_z11_z12 $z11_z12_z13 $z12_z13_z14 $z13_z14_z15 $z14_z15_z16 $z15_z16_z17 $z16_z17_z18 $z17_z18_z19 $z18_z19_z20 $z19_z20_z21 $z20_z21_z22 $z21_z22_z23 $z22_z23_z24 $z23_z24_z25 $z24_z25_z26 $z25_z26_z27 $z26_z27_z28 $z27_z28_z29 $z28_z29_z30 $z29_z30_z31 $z30_z31_z0 $z31_z0_z1 $z16_z24 $z17_z25 $z18_z26 $z19_z27 $z20_z28 $z21_z29 $z22_z30 $z23_z31 $z0_z8 $z1_z9 $z2_z10 $z3_z11 $z4_z12 $z5_z13 $z6_z14 $z7_z15 $z16_z20_z24_z28 $z17_z21_z25_z29 $z18_z22_z26_z30 $z19_z23_z27_z31 $z0_z4_z8_z12 $z1_z5_z9_z13 $z2_z6_z10_z14 $z3_z7_z11_z15
 
     call void @bar1()
     call void @bar2()

From 81f8abdca4e134009ca1acf9e85ddd4890e39822 Mon Sep 17 00:00:00 2001
From: Han-Kuan Chen <hankuan.chen@sifive.com>
Date: Fri, 16 Aug 2024 21:09:48 +0800
Subject: [PATCH 164/441] [SLP][REVEC] Fix CreateInsertElement does not use the
 correct result if MinBWs applied. (#104558)

---
 .../Transforms/Vectorize/SLPVectorizer.cpp    |  2 +-
 llvm/test/Transforms/SLPVectorizer/revec.ll   | 28 +++++++++++++++++++
 2 files changed, 29 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 87b4ed599b5bb15..667d0df781aa4cc 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -11977,7 +11977,7 @@ Value *BoUpSLP::gather(ArrayRef<Value *> VL, Value *Root, Type *ScalarTy) {
     if (auto *VecTy = dyn_cast<FixedVectorType>(Scalar->getType())) {
       assert(SLPReVec && "FixedVectorType is not expected.");
       Vec = InsElt = Builder.CreateInsertVector(
-          Vec->getType(), Vec, V,
+          Vec->getType(), Vec, Scalar,
           Builder.getInt64(Pos * VecTy->getNumElements()));
       auto *II = dyn_cast<IntrinsicInst>(InsElt);
       if (!II || II->getIntrinsicID() != Intrinsic::vector_insert)
diff --git a/llvm/test/Transforms/SLPVectorizer/revec.ll b/llvm/test/Transforms/SLPVectorizer/revec.ll
index 59201da1d9ac1a4..ad1a57c71761975 100644
--- a/llvm/test/Transforms/SLPVectorizer/revec.ll
+++ b/llvm/test/Transforms/SLPVectorizer/revec.ll
@@ -336,3 +336,31 @@ entry:
   store <4 x i32> %vmovl.i108, ptr %add.ptr35, align 4
   ret void
 }
+
+define void @test11(<2 x i64> %0, i64 %1, <2 x i64> %2) {
+; CHECK-LABEL: @test11(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x i64> [[TMP0:%.*]], i64 [[TMP1:%.*]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = add <2 x i64> <i64 5, i64 0>, [[TMP2:%.*]]
+; CHECK-NEXT:    [[TMP5:%.*]] = trunc <2 x i64> [[TMP4]] to <2 x i16>
+; CHECK-NEXT:    [[TMP6:%.*]] = call <4 x i16> @llvm.vector.insert.v4i16.v2i16(<4 x i16> poison, <2 x i16> [[TMP5]], i64 0)
+; CHECK-NEXT:    [[TMP7:%.*]] = trunc <2 x i64> [[TMP3]] to <2 x i16>
+; CHECK-NEXT:    [[TMP8:%.*]] = call <4 x i16> @llvm.vector.insert.v4i16.v2i16(<4 x i16> [[TMP6]], <2 x i16> [[TMP7]], i64 2)
+; CHECK-NEXT:    [[TMP9:%.*]] = trunc <4 x i16> [[TMP8]] to <4 x i8>
+; CHECK-NEXT:    [[TMP10:%.*]] = call <4 x i8> @llvm.vector.insert.v4i8.v2i8(<4 x i8> poison, <2 x i8> zeroinitializer, i64 0)
+; CHECK-NEXT:    [[TMP11:%.*]] = call <4 x i8> @llvm.vector.insert.v4i8.v2i8(<4 x i8> [[TMP10]], <2 x i8> zeroinitializer, i64 2)
+; CHECK-NEXT:    [[TMP12:%.*]] = urem <4 x i8> [[TMP9]], [[TMP11]]
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp ne <4 x i8> [[TMP12]], [[TMP11]]
+; CHECK-NEXT:    ret void
+;
+entry:
+  %3 = insertelement <2 x i64> %0, i64 %1, i32 1
+  %4 = add <2 x i64> <i64 5, i64 0>, %2
+  %5 = trunc <2 x i64> %3 to <2 x i8>
+  %6 = trunc <2 x i64> %4 to <2 x i8>
+  %7 = urem <2 x i8> %5, zeroinitializer
+  %8 = urem <2 x i8> %6, zeroinitializer
+  %9 = icmp ne <2 x i8> %7, zeroinitializer
+  %10 = icmp ne <2 x i8> %8, zeroinitializer
+  ret void
+}

From abdc2b17d2dbb49397cbe4023f199f63461c2a97 Mon Sep 17 00:00:00 2001
From: Tobias Stadler <mail@stadler-tobias.de>
Date: Fri, 16 Aug 2024 15:15:29 +0200
Subject: [PATCH 165/441] [AArch64][GlobalISel] Disable fixed-point iteration
 in all Combiners

Disable fixed-point iteration in all AArch64 Combiners after #102163.
See inline comments for justification of test changes.

Pull Request: https://github.com/llvm/llvm-project/pull/102167
---
 .../GISel/AArch64PostLegalizerCombiner.cpp       |  5 +++++
 .../GISel/AArch64PostLegalizerLowering.cpp       |  5 +++++
 .../GISel/AArch64PreLegalizerCombiner.cpp        |  6 ++++++
 .../GlobalISel/combine-logic-of-compare.mir      |  4 ++--
 .../AArch64/GlobalISel/combine-overflow.mir      |  8 ++++----
 .../CodeGen/AArch64/GlobalISel/combine-udiv.ll   |  8 ++++----
 .../AArch64/GlobalISel/fold-global-offsets.mir   |  6 +++---
 ...legalizer-combiner-sameopcode-hands-crash.mir |  2 ++
 ...tlegalizer-lowering-swap-compare-operands.mir | 16 ++++++++++++----
 llvm/test/CodeGen/AArch64/setcc_knownbits.ll     |  3 +--
 10 files changed, 44 insertions(+), 19 deletions(-)

diff --git a/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp b/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp
index f71fe323a6d3581..28d9f4f50f3883a 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp
@@ -566,6 +566,11 @@ bool AArch64PostLegalizerCombiner::runOnMachineFunction(MachineFunction &MF) {
   CombinerInfo CInfo(/*AllowIllegalOps*/ true, /*ShouldLegalizeIllegal*/ false,
                      /*LegalizerInfo*/ nullptr, EnableOpt, F.hasOptSize(),
                      F.hasMinSize());
+  // Disable fixed-point iteration to reduce compile-time
+  CInfo.MaxIterations = 1;
+  CInfo.ObserverLvl = CombinerInfo::ObserverLevel::SinglePass;
+  // Legalizer performs DCE, so a full DCE pass is unnecessary.
+  CInfo.EnableFullDCE = false;
   AArch64PostLegalizerCombinerImpl Impl(MF, CInfo, TPC, *KB, CSEInfo,
                                         RuleConfig, ST, MDT, LI);
   bool Changed = Impl.combineMachineInstrs();
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp b/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp
index afea8bdec1973a1..90ac4bdff4e0e48 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp
@@ -1296,6 +1296,11 @@ bool AArch64PostLegalizerLowering::runOnMachineFunction(MachineFunction &MF) {
   CombinerInfo CInfo(/*AllowIllegalOps*/ true, /*ShouldLegalizeIllegal*/ false,
                      /*LegalizerInfo*/ nullptr, /*OptEnabled=*/true,
                      F.hasOptSize(), F.hasMinSize());
+  // Disable fixed-point iteration to reduce compile-time
+  CInfo.MaxIterations = 1;
+  CInfo.ObserverLvl = CombinerInfo::ObserverLevel::SinglePass;
+  // PostLegalizerCombiner performs DCE, so a full DCE pass is unnecessary.
+  CInfo.EnableFullDCE = false;
   AArch64PostLegalizerLoweringImpl Impl(MF, CInfo, TPC, /*CSEInfo*/ nullptr,
                                         RuleConfig, ST);
   return Impl.combineMachineInstrs();
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64PreLegalizerCombiner.cpp b/llvm/lib/Target/AArch64/GISel/AArch64PreLegalizerCombiner.cpp
index 8a50cb26b2c2fee..6e689d743804ac9 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64PreLegalizerCombiner.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64PreLegalizerCombiner.cpp
@@ -861,6 +861,12 @@ bool AArch64PreLegalizerCombiner::runOnMachineFunction(MachineFunction &MF) {
   CombinerInfo CInfo(/*AllowIllegalOps*/ true, /*ShouldLegalizeIllegal*/ false,
                      /*LegalizerInfo*/ nullptr, EnableOpt, F.hasOptSize(),
                      F.hasMinSize());
+  // Disable fixed-point iteration to reduce compile-time
+  CInfo.MaxIterations = 1;
+  CInfo.ObserverLvl = CombinerInfo::ObserverLevel::SinglePass;
+  // This is the first Combiner, so the input IR might contain dead
+  // instructions.
+  CInfo.EnableFullDCE = true;
   AArch64PreLegalizerCombinerImpl Impl(MF, CInfo, &TPC, *KB, CSEInfo,
                                        RuleConfig, ST, MDT, LI);
   return Impl.combineMachineInstrs();
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-logic-of-compare.mir b/llvm/test/CodeGen/AArch64/GlobalISel/combine-logic-of-compare.mir
index 1eb445c03efcd6f..d5356711585fafb 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/combine-logic-of-compare.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-logic-of-compare.mir
@@ -203,8 +203,8 @@ body:             |
     ; CHECK-LABEL: name: test_icmp_and_icmp_9_2
     ; CHECK: liveins: $x0, $x1
     ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-    ; CHECK-NEXT: $x0 = COPY [[C]](s64)
+    ; CHECK-NEXT: %zext:_(s64) = G_CONSTANT i64 0
+    ; CHECK-NEXT: $x0 = COPY %zext(s64)
     %0:_(s64) = COPY $x0
     %nine:_(s64) = G_CONSTANT i64 9
     %two:_(s64) = G_CONSTANT i64 2
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-overflow.mir b/llvm/test/CodeGen/AArch64/GlobalISel/combine-overflow.mir
index ec66892b98fc7d3..bc4b5ae7c066a6e 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/combine-overflow.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-overflow.mir
@@ -76,9 +76,9 @@ body:             |
     ; CHECK: liveins: $w0, $w1
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $w2
-    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: %o_wide:_(s32) = G_CONSTANT i32 0
     ; CHECK-NEXT: $w0 = COPY [[COPY]](s32)
-    ; CHECK-NEXT: $w1 = COPY [[C]](s32)
+    ; CHECK-NEXT: $w1 = COPY %o_wide(s32)
     ; CHECK-NEXT: RET_ReallyLR implicit $w0
     %0:_(s32) = COPY $w0
     %1:_(s32) = COPY $w1
@@ -101,10 +101,10 @@ body:             |
     ; CHECK: liveins: $w0, $w1
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $w0
-    ; CHECK-NEXT: %const:_(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: %o_wide:_(s32) = G_CONSTANT i32 0
     ; CHECK-NEXT: $w0 = COPY [[COPY]](s32)
     ; CHECK-NEXT: $w1 = COPY [[COPY]](s32)
-    ; CHECK-NEXT: $w2 = COPY %const(s32)
+    ; CHECK-NEXT: $w2 = COPY %o_wide(s32)
     ; CHECK-NEXT: RET_ReallyLR implicit $w0
     %0:_(s32) = COPY $w0
     %const:_(s32) = G_CONSTANT i32 0
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-udiv.ll b/llvm/test/CodeGen/AArch64/GlobalISel/combine-udiv.ll
index d465e0237201be6..b681e3b22311743 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/combine-udiv.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-udiv.ll
@@ -257,10 +257,10 @@ define i32 @udiv_div_by_180(i32 %x)
 ;
 ; GISEL-LABEL: udiv_div_by_180:
 ; GISEL:       // %bb.0:
-; GISEL-NEXT:    uxtb w8, w0
-; GISEL-NEXT:    mov w9, #5826 // =0x16c2
-; GISEL-NEXT:    movk w9, #364, lsl #16
-; GISEL-NEXT:    umull x8, w8, w9
+; GISEL-NEXT:    mov w8, #5826 // =0x16c2
+; GISEL-NEXT:    and w9, w0, #0xff
+; GISEL-NEXT:    movk w8, #364, lsl #16
+; GISEL-NEXT:    umull x8, w9, w8
 ; GISEL-NEXT:    lsr x0, x8, #32
 ; GISEL-NEXT:    // kill: def $w0 killed $w0 killed $x0
 ; GISEL-NEXT:    ret
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/fold-global-offsets.mir b/llvm/test/CodeGen/AArch64/GlobalISel/fold-global-offsets.mir
index 1e9ef108c99cea6..f0fbfecf219fd94 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/fold-global-offsets.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/fold-global-offsets.mir
@@ -151,9 +151,9 @@ body:             |
     ; CHECK-LABEL: name: ptr_add_chain
     ; CHECK: liveins: $x0
     ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @g + 1
-    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
-    ; CHECK-NEXT: %dont_fold_me:_(p0) = G_PTR_ADD [[GV]], [[C]](s64)
+    ; CHECK-NEXT: [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @g + 4
+    ; CHECK-NEXT: %offset:_(s64) = G_CONSTANT i64 1
+    ; CHECK-NEXT: %dont_fold_me:_(p0) = G_PTR_ADD [[GV]], %offset(s64)
     ; CHECK-NEXT: $x0 = COPY %dont_fold_me(p0)
     ; CHECK-NEXT: RET_ReallyLR implicit $x0
     %global:_(p0) = G_GLOBAL_VALUE @g
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-combiner-sameopcode-hands-crash.mir b/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-combiner-sameopcode-hands-crash.mir
index f8510a2bd3e7137..2d286ede38d5dea 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-combiner-sameopcode-hands-crash.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-combiner-sameopcode-hands-crash.mir
@@ -9,6 +9,8 @@ body:             |
   bb.1:
     ; CHECK-LABEL: name: crash_fn
     ; CHECK: [[C:%[0-9]+]]:_(p0) = G_CONSTANT i64 0
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s1) = G_CONSTANT i1 false
     ; CHECK-NEXT: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[C]](p0) :: (load (s16), align 8)
     ; CHECK-NEXT: [[ZEXTLOAD1:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[C]](p0) :: (load (s16))
     ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ZEXTLOAD1]], [[ZEXTLOAD]]
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-lowering-swap-compare-operands.mir b/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-lowering-swap-compare-operands.mir
index ce7450c69050394..b8020bd50a2b0d6 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-lowering-swap-compare-operands.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-lowering-swap-compare-operands.mir
@@ -64,21 +64,28 @@ body:             |
     ; LOWER-NEXT: {{  $}}
     ; LOWER-NEXT: %reg0:_(s64) = COPY $x0
     ; LOWER-NEXT: %cmp_lhs:_(s64) = G_SEXT_INREG %reg0, 8
+    ; LOWER-NEXT: %reg1:_(s64) = COPY $x1
+    ; LOWER-NEXT: %cmp1:_(s32) = G_ICMP intpred(sge), %cmp_lhs(s64), %reg1
     ; LOWER-NEXT: %add:_(s64) = G_ADD %cmp_lhs, %reg0
     ; LOWER-NEXT: %cmp2:_(s32) = G_ICMP intpred(sge), %cmp_lhs(s64), %add
     ; LOWER-NEXT: $w0 = COPY %cmp2(s32)
-    ; LOWER-NEXT: RET_ReallyLR implicit $w0
+    ; LOWER-NEXT: $w1 = COPY %cmp1(s32)
+    ; LOWER-NEXT: RET_ReallyLR implicit $w0, implicit $w1
     ;
     ; SELECT-LABEL: name: dont_swap_more_than_one_use
     ; SELECT: liveins: $x0, $x1
     ; SELECT-NEXT: {{  $}}
     ; SELECT-NEXT: %reg0:gpr64 = COPY $x0
     ; SELECT-NEXT: %cmp_lhs:gpr64 = SBFMXri %reg0, 0, 7
+    ; SELECT-NEXT: %reg1:gpr64 = COPY $x1
+    ; SELECT-NEXT: [[SUBSXrr:%[0-9]+]]:gpr64 = SUBSXrr %cmp_lhs, %reg1, implicit-def $nzcv
+    ; SELECT-NEXT: %cmp1:gpr32 = CSINCWr $wzr, $wzr, 11, implicit $nzcv
     ; SELECT-NEXT: %add:gpr64 = ADDXrr %cmp_lhs, %reg0
-    ; SELECT-NEXT: [[SUBSXrr:%[0-9]+]]:gpr64 = SUBSXrr %cmp_lhs, %add, implicit-def $nzcv
+    ; SELECT-NEXT: [[SUBSXrr1:%[0-9]+]]:gpr64 = SUBSXrr %cmp_lhs, %add, implicit-def $nzcv
     ; SELECT-NEXT: %cmp2:gpr32 = CSINCWr $wzr, $wzr, 11, implicit $nzcv
     ; SELECT-NEXT: $w0 = COPY %cmp2
-    ; SELECT-NEXT: RET_ReallyLR implicit $w0
+    ; SELECT-NEXT: $w1 = COPY %cmp1
+    ; SELECT-NEXT: RET_ReallyLR implicit $w0, implicit $w1
     %reg0:_(s64) = COPY $x0
     %cmp_lhs:_(s64) = G_SEXT_INREG %reg0, 8
     %reg1:_(s64) = COPY $x1
@@ -88,7 +95,8 @@ body:             |
     %cmp2:_(s32) = G_ICMP intpred(sge), %cmp_lhs(s64), %add
 
     $w0 = COPY %cmp2(s32)
-    RET_ReallyLR implicit $w0
+    $w1 = COPY %cmp1(s32)
+    RET_ReallyLR implicit $w0, implicit $w1
 
 ...
 ---
diff --git a/llvm/test/CodeGen/AArch64/setcc_knownbits.ll b/llvm/test/CodeGen/AArch64/setcc_knownbits.ll
index 8be63b04d8ce588..4dd2a896cd81cbc 100644
--- a/llvm/test/CodeGen/AArch64/setcc_knownbits.ll
+++ b/llvm/test/CodeGen/AArch64/setcc_knownbits.ll
@@ -33,8 +33,7 @@ define noundef i1 @logger(i32 noundef %logLevel, ptr %ea, ptr %pll) {
 ; CHECK-GI-NEXT:    b.hi .LBB1_2
 ; CHECK-GI-NEXT:  // %bb.1: // %land.rhs
 ; CHECK-GI-NEXT:    ldr x8, [x1]
-; CHECK-GI-NEXT:    ldrb w8, [x8]
-; CHECK-GI-NEXT:    and w0, w8, #0x1
+; CHECK-GI-NEXT:    ldrb w0, [x8]
 ; CHECK-GI-NEXT:  .LBB1_2: // %land.end
 ; CHECK-GI-NEXT:    ret
 entry:

From aa427b1aae445ed46d9f60c5e2eaac61bdf76be3 Mon Sep 17 00:00:00 2001
From: RichardLuo <CommAdama@outlook.com>
Date: Fri, 16 Aug 2024 21:26:38 +0800
Subject: [PATCH 166/441] [libc++] Fix backslash as root dir breaks
 lexically_relative, lexically_proximate and hash_value on Windows (#99780)

Various functions like hash_value, lexically_proximate and lexically_relative
would incorrectly handle backslashes in the root directory on Windows, causing
behavior that is inconsistent with the equality comparison for a path.
---
 libcxx/src/filesystem/path.cpp                |  5 +-
 .../path.member/path.compare.pass.cpp         | 60 +++++++++++--------
 .../lexically_relative_and_proximate.pass.cpp |  2 +
 3 files changed, 39 insertions(+), 28 deletions(-)

diff --git a/libcxx/src/filesystem/path.cpp b/libcxx/src/filesystem/path.cpp
index b2019521377ede5..58742442bae6b71 100644
--- a/libcxx/src/filesystem/path.cpp
+++ b/libcxx/src/filesystem/path.cpp
@@ -267,7 +267,7 @@ path path::lexically_relative(const path& base) const {
   // Find the first mismatching element
   auto PP     = PathParser::CreateBegin(__pn_);
   auto PPBase = PathParser::CreateBegin(base.__pn_);
-  while (PP && PPBase && PP.State_ == PPBase.State_ && *PP == *PPBase) {
+  while (PP && PPBase && PP.State_ == PPBase.State_ && (*PP == *PPBase || PP.inRootDir())) {
     ++PP;
     ++PPBase;
   }
@@ -368,7 +368,8 @@ size_t hash_value(const path& __p) noexcept {
   size_t hash_value = 0;
   hash<string_view_t> hasher;
   while (PP) {
-    hash_value = __hash_combine(hash_value, hasher(*PP));
+    string_view_t Part = PP.inRootDir() ? PATHSTR("/") : *PP;
+    hash_value         = __hash_combine(hash_value, hasher(Part));
     ++PP;
   }
   return hash_value;
diff --git a/libcxx/test/std/input.output/filesystems/class.path/path.member/path.compare.pass.cpp b/libcxx/test/std/input.output/filesystems/class.path/path.member/path.compare.pass.cpp
index 99f55d3b17f1b09..3bebf38a2a7c34d 100644
--- a/libcxx/test/std/input.output/filesystems/class.path/path.member/path.compare.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/class.path/path.member/path.compare.pass.cpp
@@ -47,13 +47,20 @@ struct PathCompareTest {
   int expect;
 };
 
-#define LONGA "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
-#define LONGB "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"
-#define LONGC "CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC"
-#define LONGD "DDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDD"
-const PathCompareTest CompareTestCases[] =
-{
-    {"", "",  0},
+#define LONGA                                                                                                          \
+  "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" \
+  "AAAAAAAA"
+#define LONGB                                                                                                          \
+  "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB" \
+  "BBBBBBBB"
+#define LONGC                                                                                                          \
+  "CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC" \
+  "CCCCCCCC"
+#define LONGD                                                                                                          \
+  "DDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDD" \
+  "DDDDDDDD"
+const PathCompareTest CompareTestCases[] = {
+    {"", "", 0},
     {"a", "", 1},
     {"", "a", -1},
     {"a/b/c", "a/b/c", 0},
@@ -62,14 +69,19 @@ const PathCompareTest CompareTestCases[] =
     {"a/b", "a/b/c", -1},
     {"a/b/c", "a/b", 1},
     {"a/b/", "a/b/.", -1},
-    {"a/b/", "a/b",    1},
+    {"a/b/", "a/b", 1},
     {"a/b//////", "a/b/////.", -1},
     {"a/.././b", "a///..//.////b", 0},
     {"//foo//bar///baz////", "//foo/bar/baz/", 0}, // duplicate separators
-    {"///foo/bar", "/foo/bar", 0}, // "///" is not a root directory
-    {"/foo/bar/", "/foo/bar", 1}, // trailing separator
+    {"///foo/bar", "/foo/bar", 0},                 // "///" is not a root directory
+    {"/foo/bar/", "/foo/bar", 1},                  // trailing separator
     {"foo", "/foo", -1}, // if !this->has_root_directory() and p.has_root_directory(), a value less than 0.
-    {"/foo", "foo", 1}, //  if this->has_root_directory() and !p.has_root_directory(), a value greater than 0.
+    {"/foo", "foo", 1},  //  if this->has_root_directory() and !p.has_root_directory(), a value greater than 0.
+#ifdef _WIN32
+    {"C:/a", "C:\\a", 0},
+#else
+    {"C:/a", "C:\\a", -1},
+#endif
     {("//" LONGA "////" LONGB "/" LONGC "///" LONGD), ("//" LONGA "/" LONGB "/" LONGC "/" LONGD), 0},
     {(LONGA "/" LONGB "/" LONGC), (LONGA "/" LONGB "/" LONGB), 1}
 
@@ -79,23 +91,19 @@ const PathCompareTest CompareTestCases[] =
 #undef LONGC
 #undef LONGD
 
-static inline int normalize_ret(int ret)
-{
-  return ret < 0 ? -1 : (ret > 0 ? 1 : 0);
-}
+static inline int normalize_ret(int ret) { return ret < 0 ? -1 : (ret > 0 ? 1 : 0); }
 
-void test_compare_basic()
-{
+void test_compare_basic() {
   using namespace fs;
-  for (auto const & TC : CompareTestCases) {
+  for (auto const& TC : CompareTestCases) {
     const path p1(TC.LHS);
     const path p2(TC.RHS);
     std::string RHS(TC.RHS);
     const path::string_type R(RHS.begin(), RHS.end());
     const std::basic_string_view<path::value_type> RV(R);
-    const path::value_type *Ptr = R.c_str();
-    const int E = TC.expect;
-    { // compare(...) functions
+    const path::value_type* Ptr = R.c_str();
+    const int E                 = TC.expect;
+    {                           // compare(...) functions
       DisableAllocationGuard g; // none of these operations should allocate
 
       // check runtime results
@@ -113,7 +121,7 @@ void test_compare_basic()
       // check signatures
       ASSERT_NOEXCEPT(p1.compare(p2));
     }
-    { // comparison operators
+    {                           // comparison operators
       DisableAllocationGuard g; // none of these operations should allocate
 
       // check signatures
@@ -180,14 +188,14 @@ void test_compare_elements() {
 
   auto BuildPath = [](std::vector<std::string> const& Elems) {
     fs::path p;
-    for (auto &E : Elems)
+    for (auto& E : Elems)
       p /= E;
     return p;
   };
 
-  for (auto &TC : TestCases) {
-    fs::path LHS = BuildPath(TC.LHSElements);
-    fs::path RHS = BuildPath(TC.RHSElements);
+  for (auto& TC : TestCases) {
+    fs::path LHS        = BuildPath(TC.LHSElements);
+    fs::path RHS        = BuildPath(TC.RHSElements);
     const int ExpectCmp = CompareElements(TC.LHSElements, TC.RHSElements);
     assert(ExpectCmp == TC.Expect);
     const int GotCmp = normalize_ret(LHS.compare(RHS));
diff --git a/libcxx/test/std/input.output/filesystems/class.path/path.member/path.gen/lexically_relative_and_proximate.pass.cpp b/libcxx/test/std/input.output/filesystems/class.path/path.member/path.gen/lexically_relative_and_proximate.pass.cpp
index 42fa813ada4a5c8..06954fe876e5572 100644
--- a/libcxx/test/std/input.output/filesystems/class.path/path.member/path.gen/lexically_relative_and_proximate.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/class.path/path.member/path.gen/lexically_relative_and_proximate.pass.cpp
@@ -41,9 +41,11 @@ int main(int, char**) {
 #ifdef _WIN32
       {"//net/", "//net", ""},
       {"//net", "//net/", ""},
+      {"C:\\a\\b", "C:/a", "b"},
 #else
       {"//net/", "//net", "."},
       {"//net", "//net/", "."},
+      {"C:\\a\\b", "C:/a", "../../C:\\a\\b"},
 #endif
       {"//base", "a", ""},
       {"a", "a", "."},

From 710a552dedbf1a28cebac78b481cb78138fbc454 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=BCtt?= <schuett@gmail.com>
Date: Fri, 16 Aug 2024 15:38:56 +0200
Subject: [PATCH 167/441] [GlobalIsel] Revisit ext of ext. (#102769)

Credits:
https://reviews.llvm.org/D86516

combine-ext.mir

Notable semantic changes:
InstCombine does not mix zero and sign extend,
  see CastInst::isEliminableCastPair.
New version has legality checks.
Folds sext/zext of anyext -> sext/zext
Support for nneg

Future work:
nneg zext of sext -> sext
sext of nneg zext -> sext
---
 .../llvm/CodeGen/GlobalISel/CombinerHelper.h  |  9 +--
 .../include/llvm/Target/GlobalISel/Combine.td | 36 ++++++---
 .../lib/CodeGen/GlobalISel/CombinerHelper.cpp | 54 -------------
 .../GlobalISel/CombinerHelperCasts.cpp        | 64 +++++++++++++++
 .../AArch64/GlobalISel/combine-ext.mir        | 80 +++++++++++++++++--
 5 files changed, 166 insertions(+), 77 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
index c76a54ed26ed562..32effc536eb35df 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
@@ -390,12 +390,6 @@ class CombinerHelper {
   /// Transform zext(trunc(x)) to x.
   bool matchCombineZextTrunc(MachineInstr &MI, Register &Reg);
 
-  /// Transform [asz]ext([asz]ext(x)) to [asz]ext x.
-  bool matchCombineExtOfExt(MachineInstr &MI,
-                            std::tuple<Register, unsigned> &MatchInfo);
-  void applyCombineExtOfExt(MachineInstr &MI,
-                            std::tuple<Register, unsigned> &MatchInfo);
-
   /// Transform trunc (shl x, K) to shl (trunc x), K
   ///    if K < VT.getScalarSizeInBits().
   ///
@@ -903,6 +897,9 @@ class CombinerHelper {
   // fold ((A-C1)+C2) -> (A+(C2-C1))
   bool matchFoldAMinusC1PlusC2(const MachineInstr &MI, BuildFnTy &MatchInfo);
 
+  bool matchExtOfExt(const MachineInstr &FirstMI, const MachineInstr &SecondMI,
+                     BuildFnTy &MatchInfo);
+
 private:
   /// Checks for legality of an indexed variant of \p LdSt.
   bool isIndexedLoadStoreLegal(GLoadStore &LdSt) const;
diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td
index c76596ce3e4f2d5..814c5e789cb3749 100644
--- a/llvm/include/llvm/Target/GlobalISel/Combine.td
+++ b/llvm/include/llvm/Target/GlobalISel/Combine.td
@@ -767,15 +767,6 @@ def zext_trunc_fold: GICombineRule <
   (apply [{ Helper.replaceSingleDefInstWithReg(*${root}, ${matchinfo}); }])
 >;
 
-// Fold ([asz]ext ([asz]ext x)) -> ([asz]ext x).
-def ext_ext_fold_matchinfo : GIDefMatchData<"std::tuple<Register, unsigned>">;
-def ext_ext_fold: GICombineRule <
-  (defs root:$root, ext_ext_fold_matchinfo:$matchinfo),
-  (match (wip_match_opcode G_ANYEXT, G_SEXT, G_ZEXT):$root,
-         [{ return Helper.matchCombineExtOfExt(*${root}, ${matchinfo}); }]),
-  (apply [{ Helper.applyCombineExtOfExt(*${root}, ${matchinfo}); }])
->;
-
 def not_cmp_fold_matchinfo : GIDefMatchData<"SmallVector<Register, 4>">;
 def not_cmp_fold : GICombineRule<
   (defs root:$d, not_cmp_fold_matchinfo:$info),
@@ -1850,13 +1841,36 @@ def select_of_zext : select_of_opcode<G_ZEXT>;
 def select_of_anyext : select_of_opcode<G_ANYEXT>;
 def select_of_truncate : select_of_opcode<G_TRUNC>;
 
+// Fold ([asz]ext ([asz]ext x)) -> ([asz]ext x).
+class ext_of_ext_opcodes<Instruction ext1Opcode, Instruction ext2Opcode> : GICombineRule <
+  (defs root:$root, build_fn_matchinfo:$matchinfo),
+  (match (ext2Opcode $second, $src):$Second,
+         (ext1Opcode $root, $second):$First,
+         [{ return Helper.matchExtOfExt(*${First}, *${Second}, ${matchinfo}); }]),
+  (apply [{ Helper.applyBuildFn(*${First}, ${matchinfo}); }])>;
+
+def zext_of_zext : ext_of_ext_opcodes<G_ZEXT, G_ZEXT>;
+def zext_of_anyext : ext_of_ext_opcodes<G_ZEXT, G_ANYEXT>;
+def sext_of_sext : ext_of_ext_opcodes<G_SEXT, G_SEXT>;
+def sext_of_anyext : ext_of_ext_opcodes<G_SEXT, G_ANYEXT>;
+def anyext_of_anyext : ext_of_ext_opcodes<G_ANYEXT, G_ANYEXT>;
+def anyext_of_zext : ext_of_ext_opcodes<G_ANYEXT, G_ZEXT>;
+def anyext_of_sext : ext_of_ext_opcodes<G_ANYEXT, G_SEXT>;
+
 def cast_combines: GICombineGroup<[
   truncate_of_zext,
   truncate_of_sext,
   truncate_of_anyext,
   select_of_zext,
   select_of_anyext,
-  select_of_truncate
+  select_of_truncate,
+  zext_of_zext,
+  zext_of_anyext,
+  sext_of_sext,
+  sext_of_anyext,
+  anyext_of_anyext,
+  anyext_of_zext,
+  anyext_of_sext
 ]>;
 
 
@@ -1928,7 +1942,7 @@ def all_combines : GICombineGroup<[integer_reassoc_combines, trivial_combines,
     reassocs, ptr_add_immed_chain,
     shl_ashr_to_sext_inreg, sext_inreg_of_load,
     width_reduction_combines, select_combines,
-    known_bits_simplifications, ext_ext_fold,
+    known_bits_simplifications,
     not_cmp_fold, opt_brcond_by_inverting_cond,
     unmerge_merge, unmerge_cst, unmerge_dead_to_trunc,
     unmerge_zext_to_zext, merge_unmerge, trunc_shift,
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index d363e6794cb017b..1517ae707c8cff1 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -2538,60 +2538,6 @@ bool CombinerHelper::matchCombineZextTrunc(MachineInstr &MI, Register &Reg) {
   return false;
 }
 
-bool CombinerHelper::matchCombineExtOfExt(
-    MachineInstr &MI, std::tuple<Register, unsigned> &MatchInfo) {
-  assert((MI.getOpcode() == TargetOpcode::G_ANYEXT ||
-          MI.getOpcode() == TargetOpcode::G_SEXT ||
-          MI.getOpcode() == TargetOpcode::G_ZEXT) &&
-         "Expected a G_[ASZ]EXT");
-  Register SrcReg = MI.getOperand(1).getReg();
-  Register OriginalSrcReg = getSrcRegIgnoringCopies(SrcReg, MRI);
-  if (OriginalSrcReg.isValid())
-    SrcReg = OriginalSrcReg;
-  MachineInstr *SrcMI = MRI.getVRegDef(SrcReg);
-  // Match exts with the same opcode, anyext([sz]ext) and sext(zext).
-  unsigned Opc = MI.getOpcode();
-  unsigned SrcOpc = SrcMI->getOpcode();
-  if (Opc == SrcOpc ||
-      (Opc == TargetOpcode::G_ANYEXT &&
-       (SrcOpc == TargetOpcode::G_SEXT || SrcOpc == TargetOpcode::G_ZEXT)) ||
-      (Opc == TargetOpcode::G_SEXT && SrcOpc == TargetOpcode::G_ZEXT)) {
-    MatchInfo = std::make_tuple(SrcMI->getOperand(1).getReg(), SrcOpc);
-    return true;
-  }
-  return false;
-}
-
-void CombinerHelper::applyCombineExtOfExt(
-    MachineInstr &MI, std::tuple<Register, unsigned> &MatchInfo) {
-  assert((MI.getOpcode() == TargetOpcode::G_ANYEXT ||
-          MI.getOpcode() == TargetOpcode::G_SEXT ||
-          MI.getOpcode() == TargetOpcode::G_ZEXT) &&
-         "Expected a G_[ASZ]EXT");
-
-  Register Reg = std::get<0>(MatchInfo);
-  unsigned SrcExtOp = std::get<1>(MatchInfo);
-
-  // Combine exts with the same opcode.
-  if (MI.getOpcode() == SrcExtOp) {
-    Observer.changingInstr(MI);
-    MI.getOperand(1).setReg(Reg);
-    Observer.changedInstr(MI);
-    return;
-  }
-
-  // Combine:
-  // - anyext([sz]ext x) to [sz]ext x
-  // - sext(zext x) to zext x
-  if (MI.getOpcode() == TargetOpcode::G_ANYEXT ||
-      (MI.getOpcode() == TargetOpcode::G_SEXT &&
-       SrcExtOp == TargetOpcode::G_ZEXT)) {
-    Register DstReg = MI.getOperand(0).getReg();
-    Builder.buildInstr(SrcExtOp, {DstReg}, {Reg});
-    MI.eraseFromParent();
-  }
-}
-
 static LLT getMidVTForTruncRightShiftCombine(LLT ShiftTy, LLT TruncTy) {
   const unsigned ShiftSize = ShiftTy.getScalarSizeInBits();
   const unsigned TruncSize = TruncTy.getScalarSizeInBits();
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelperCasts.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelperCasts.cpp
index 59295f7a65835c6..494d8da84445d15 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelperCasts.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelperCasts.cpp
@@ -209,3 +209,67 @@ bool CombinerHelper::matchCastOfSelect(const MachineInstr &CastMI,
 
   return true;
 }
+
+bool CombinerHelper::matchExtOfExt(const MachineInstr &FirstMI,
+                                   const MachineInstr &SecondMI,
+                                   BuildFnTy &MatchInfo) {
+  const GExtOp *First = cast<GExtOp>(&FirstMI);
+  const GExtOp *Second = cast<GExtOp>(&SecondMI);
+
+  Register Dst = First->getReg(0);
+  Register Src = Second->getSrcReg();
+  LLT DstTy = MRI.getType(Dst);
+  LLT SrcTy = MRI.getType(Src);
+
+  if (!MRI.hasOneNonDBGUse(Second->getReg(0)))
+    return false;
+
+  // ext of ext -> later ext
+  if (First->getOpcode() == Second->getOpcode() &&
+      isLegalOrBeforeLegalizer({Second->getOpcode(), {DstTy, SrcTy}})) {
+    if (Second->getOpcode() == TargetOpcode::G_ZEXT) {
+      MachineInstr::MIFlag Flag = MachineInstr::MIFlag::NoFlags;
+      if (Second->getFlag(MachineInstr::MIFlag::NonNeg))
+        Flag = MachineInstr::MIFlag::NonNeg;
+      MatchInfo = [=](MachineIRBuilder &B) { B.buildZExt(Dst, Src, Flag); };
+      return true;
+    }
+    // not zext -> no flags
+    MatchInfo = [=](MachineIRBuilder &B) {
+      B.buildInstr(Second->getOpcode(), {Dst}, {Src});
+    };
+    return true;
+  }
+
+  // anyext of sext/zext  -> sext/zext
+  // -> pick anyext as second ext, then ext of ext
+  if (First->getOpcode() == TargetOpcode::G_ANYEXT &&
+      isLegalOrBeforeLegalizer({Second->getOpcode(), {DstTy, SrcTy}})) {
+    if (Second->getOpcode() == TargetOpcode::G_ZEXT) {
+      MachineInstr::MIFlag Flag = MachineInstr::MIFlag::NoFlags;
+      if (Second->getFlag(MachineInstr::MIFlag::NonNeg))
+        Flag = MachineInstr::MIFlag::NonNeg;
+      MatchInfo = [=](MachineIRBuilder &B) { B.buildZExt(Dst, Src, Flag); };
+      return true;
+    }
+    MatchInfo = [=](MachineIRBuilder &B) { B.buildSExt(Dst, Src); };
+    return true;
+  }
+
+  // sext/zext of anyext -> sext/zext
+  // -> pick anyext as first ext, then ext of ext
+  if (Second->getOpcode() == TargetOpcode::G_ANYEXT &&
+      isLegalOrBeforeLegalizer({First->getOpcode(), {DstTy, SrcTy}})) {
+    if (First->getOpcode() == TargetOpcode::G_ZEXT) {
+      MachineInstr::MIFlag Flag = MachineInstr::MIFlag::NoFlags;
+      if (First->getFlag(MachineInstr::MIFlag::NonNeg))
+        Flag = MachineInstr::MIFlag::NonNeg;
+      MatchInfo = [=](MachineIRBuilder &B) { B.buildZExt(Dst, Src, Flag); };
+      return true;
+    }
+    MatchInfo = [=](MachineIRBuilder &B) { B.buildSExt(Dst, Src); };
+    return true;
+  }
+
+  return false;
+}
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-ext.mir b/llvm/test/CodeGen/AArch64/GlobalISel/combine-ext.mir
index a431a6764d06f7b..629238126380c77 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/combine-ext.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-ext.mir
@@ -251,8 +251,9 @@ body:             |
     ; CHECK: liveins: $h0
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s16) = COPY $h0
-    ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[COPY]](s16)
-    ; CHECK-NEXT: $x0 = COPY [[ZEXT]](s64)
+    ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[COPY]](s16)
+    ; CHECK-NEXT: [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[ZEXT]](s32)
+    ; CHECK-NEXT: $x0 = COPY [[SEXT]](s64)
     %0:_(s16) = COPY $h0
     %1:_(s32) = G_ZEXT %0(s16)
     %2:_(s64) = G_SEXT %1(s32)
@@ -267,8 +268,10 @@ body:             |
     ; CHECK: liveins: $h0
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s16) = COPY $h0
-    ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[COPY]](s16)
-    ; CHECK-NEXT: $x0 = COPY [[ZEXT]](s64)
+    ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[COPY]](s16)
+    ; CHECK-NEXT: [[ASSERT_ZEXT:%[0-9]+]]:_(s32) = G_ASSERT_ZEXT [[ZEXT]], 11
+    ; CHECK-NEXT: [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[ASSERT_ZEXT]](s32)
+    ; CHECK-NEXT: $x0 = COPY [[SEXT]](s64)
     %0:_(s16) = COPY $h0
     %1:_(s32) = G_ZEXT %0(s16)
     %2:_(s32) = G_ASSERT_ZEXT %1(s32), 11
@@ -284,8 +287,9 @@ body:             |
     ; CHECK: liveins: $s0
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $s0
-    ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(<2 x s64>) = G_ZEXT [[COPY]](<2 x s16>)
-    ; CHECK-NEXT: $q0 = COPY [[ZEXT]](<2 x s64>)
+    ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(<2 x s32>) = G_ZEXT [[COPY]](<2 x s16>)
+    ; CHECK-NEXT: [[SEXT:%[0-9]+]]:_(<2 x s64>) = G_SEXT [[ZEXT]](<2 x s32>)
+    ; CHECK-NEXT: $q0 = COPY [[SEXT]](<2 x s64>)
     %0:_(<2 x s16>) = COPY $s0
     %1:_(<2 x s32>) = G_ZEXT %0(<2 x s16>)
     %2:_(<2 x s64>) = G_SEXT %1(<2 x s32>)
@@ -340,3 +344,67 @@ body:             |
     %2:_(<2 x s64>) = G_ZEXT %1(<2 x s32>)
     $q0 = COPY %2(<2 x s64>)
 ...
+---
+name:            test_combine_zext_anyext
+body:             |
+  bb.1:
+  liveins: $h0
+    ; CHECK-LABEL: name: test_combine_zext_anyext
+    ; CHECK: liveins: $h0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s16) = COPY $h0
+    ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[COPY]](s16)
+    ; CHECK-NEXT: $x0 = COPY [[ZEXT]](s64)
+    %0:_(s16) = COPY $h0
+    %1:_(s32) = G_ANYEXT %0(s16)
+    %2:_(s64) = G_ZEXT %1(s32)
+    $x0 = COPY %2(s64)
+...
+---
+name:            test_combine_sext_anyext
+body:             |
+  bb.1:
+  liveins: $h0
+    ; CHECK-LABEL: name: test_combine_sext_anyext
+    ; CHECK: liveins: $h0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s16) = COPY $h0
+    ; CHECK-NEXT: [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[COPY]](s16)
+    ; CHECK-NEXT: $x0 = COPY [[SEXT]](s64)
+    %0:_(s16) = COPY $h0
+    %1:_(s32) = G_ANYEXT %0(s16)
+    %2:_(s64) = G_SEXT %1(s32)
+    $x0 = COPY %2(s64)
+...
+---
+name:            test_combine_nneg_zext_anyext
+body:             |
+  bb.1:
+  liveins: $h0
+    ; CHECK-LABEL: name: test_combine_nneg_zext_anyext
+    ; CHECK: liveins: $h0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s16) = COPY $h0
+    ; CHECK-NEXT: %2:_(s64) = nneg G_ZEXT [[COPY]](s16)
+    ; CHECK-NEXT: $x0 = COPY %2(s64)
+    %0:_(s16) = COPY $h0
+    %1:_(s32) = G_ANYEXT %0(s16)
+    %2:_(s64) = nneg G_ZEXT %1(s32)
+    $x0 = COPY %2(s64)
+...
+---
+name:            test_combine_anyext_nneg_zext
+body:             |
+  bb.1:
+  liveins: $h0
+    ; CHECK-LABEL: name: test_combine_anyext_nneg_zext
+    ; CHECK: liveins: $h0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s16) = COPY $h0
+    ; CHECK-NEXT: %2:_(s64) = nneg G_ZEXT [[COPY]](s16)
+    ; CHECK-NEXT: $x0 = COPY %2(s64)
+    %0:_(s16) = COPY $h0
+    %1:_(s32) = nneg G_ZEXT %0(s16)
+    %2:_(s64) = G_ANYEXT %1(s32)
+    $x0 = COPY %2(s64)
+...

From 3e5a4ac31abd9021dd7996287ebb96af45daef04 Mon Sep 17 00:00:00 2001
From: Angel Zhang <angel.zhang@amd.com>
Date: Fri, 16 Aug 2024 09:49:27 -0400
Subject: [PATCH 168/441] [mlir][spirv] Update documentation. NFC (#104584)

---
 mlir/include/mlir/Dialect/SPIRV/IR/SPIRVNonUniformOps.td | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVNonUniformOps.td b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVNonUniformOps.td
index e6276e6853fcb0d..c144dc52d4ea0c3 100644
--- a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVNonUniformOps.td
+++ b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVNonUniformOps.td
@@ -56,7 +56,7 @@ def SPIRV_GroupNonUniformBallotOp : SPIRV_Op<"GroupNonUniformBallot", []> {
     #### Example:
 
     ```mlir
-    %0 = spirv.GroupNonUniformBallot "SubGroup" %predicate : vector<4xi32>
+    %0 = spirv.GroupNonUniformBallot <Subgroup> %predicate : vector<4xi32>
     ```
   }];
 

From 8173ad703152e0b7f3689da76d5337ea8522e11c Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Fri, 16 Aug 2024 15:52:40 +0200
Subject: [PATCH 169/441] [InstCombine] Regenerate test checks (NFC)

---
 .../InstCombine/and-or-icmp-const-icmp.ll     | 72 +++++++++----------
 1 file changed, 36 insertions(+), 36 deletions(-)

diff --git a/llvm/test/Transforms/InstCombine/and-or-icmp-const-icmp.ll b/llvm/test/Transforms/InstCombine/and-or-icmp-const-icmp.ll
index 9143a2d0ccda249..9365f8281ccbd3b 100644
--- a/llvm/test/Transforms/InstCombine/and-or-icmp-const-icmp.ll
+++ b/llvm/test/Transforms/InstCombine/and-or-icmp-const-icmp.ll
@@ -13,8 +13,8 @@
 ; Basic tests
 ; ==============================================================================
 define i1 @eq_basic(i8 %x, i8 %y) {
-; CHECK-LABEL: define i1 @eq_basic
-; CHECK-SAME: (i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-LABEL: define i1 @eq_basic(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = add i8 [[X]], -1
 ; CHECK-NEXT:    [[OR:%.*]] = icmp uge i8 [[TMP1]], [[Y]]
 ; CHECK-NEXT:    ret i1 [[OR]]
@@ -26,8 +26,8 @@ define i1 @eq_basic(i8 %x, i8 %y) {
 }
 
 define i1 @ne_basic_equal_5(i8 %x, i8 %y) {
-; CHECK-LABEL: define i1 @ne_basic_equal_5
-; CHECK-SAME: (i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-LABEL: define i1 @ne_basic_equal_5(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = add i8 [[X]], -6
 ; CHECK-NEXT:    [[AND:%.*]] = icmp ult i8 [[TMP1]], [[Y]]
 ; CHECK-NEXT:    ret i1 [[AND]]
@@ -40,8 +40,8 @@ define i1 @ne_basic_equal_5(i8 %x, i8 %y) {
 }
 
 define i1 @eq_basic_equal_minus_1(i8 %x, i8 %y) {
-; CHECK-LABEL: define i1 @eq_basic_equal_minus_1
-; CHECK-SAME: (i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-LABEL: define i1 @eq_basic_equal_minus_1(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
 ; CHECK-NEXT:    [[OR:%.*]] = icmp uge i8 [[X]], [[Y]]
 ; CHECK-NEXT:    ret i1 [[OR]]
 ;
@@ -53,8 +53,8 @@ define i1 @eq_basic_equal_minus_1(i8 %x, i8 %y) {
 }
 
 define i1 @ne_basic_equal_minus_7(i8 %x, i8 %y) {
-; CHECK-LABEL: define i1 @ne_basic_equal_minus_7
-; CHECK-SAME: (i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-LABEL: define i1 @ne_basic_equal_minus_7(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = add i8 [[X]], 6
 ; CHECK-NEXT:    [[AND:%.*]] = icmp ult i8 [[TMP1]], [[Y]]
 ; CHECK-NEXT:    ret i1 [[AND]]
@@ -67,8 +67,8 @@ define i1 @ne_basic_equal_minus_7(i8 %x, i8 %y) {
 }
 
 define i1 @eq_basic_unequal(i8 %x, i8 %y) {
-; CHECK-LABEL: define i1 @eq_basic_unequal
-; CHECK-SAME: (i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-LABEL: define i1 @eq_basic_unequal(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
 ; CHECK-NEXT:    [[SUB:%.*]] = add i8 [[X]], -5
 ; CHECK-NEXT:    [[C1:%.*]] = icmp eq i8 [[X]], 6
 ; CHECK-NEXT:    [[C2:%.*]] = icmp ugt i8 [[SUB]], [[Y]]
@@ -83,8 +83,8 @@ define i1 @eq_basic_unequal(i8 %x, i8 %y) {
 }
 
 define i1 @ne_basic_unequal(i8 %x, i8 %y) {
-; CHECK-LABEL: define i1 @ne_basic_unequal
-; CHECK-SAME: (i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-LABEL: define i1 @ne_basic_unequal(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
 ; CHECK-NEXT:    [[ADD:%.*]] = add i8 [[X]], 7
 ; CHECK-NEXT:    [[C1:%.*]] = icmp ne i8 [[X]], -4
 ; CHECK-NEXT:    [[C2:%.*]] = icmp ule i8 [[ADD]], [[Y]]
@@ -102,8 +102,8 @@ define i1 @ne_basic_unequal(i8 %x, i8 %y) {
 ; Tests with multiple uses
 ; ==============================================================================
 define i1 @eq_multi_c1(i8 %x, i8 %y) {
-; CHECK-LABEL: define i1 @eq_multi_c1
-; CHECK-SAME: (i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-LABEL: define i1 @eq_multi_c1(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
 ; CHECK-NEXT:    [[C1:%.*]] = icmp eq i8 [[X]], 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = add i8 [[X]], -1
 ; CHECK-NEXT:    [[OR:%.*]] = icmp uge i8 [[TMP1]], [[Y]]
@@ -118,8 +118,8 @@ define i1 @eq_multi_c1(i8 %x, i8 %y) {
 }
 
 define i1 @ne_multi_c2(i8 %x, i8 %y) {
-; CHECK-LABEL: define i1 @ne_multi_c2
-; CHECK-SAME: (i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-LABEL: define i1 @ne_multi_c2(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
 ; CHECK-NEXT:    [[C2:%.*]] = icmp ule i8 [[X]], [[Y]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = add i8 [[X]], -1
 ; CHECK-NEXT:    [[AND:%.*]] = icmp ult i8 [[TMP1]], [[Y]]
@@ -137,8 +137,8 @@ define i1 @ne_multi_c2(i8 %x, i8 %y) {
 ; Tests with vector types
 ; ==============================================================================
 define <2 x i1> @eq_vector(<2 x i8> %x, <2 x i8> %y) {
-; CHECK-LABEL: define <2 x i1> @eq_vector
-; CHECK-SAME: (<2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]]) {
+; CHECK-LABEL: define <2 x i1> @eq_vector(
+; CHECK-SAME: <2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]]) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = add <2 x i8> [[X]], <i8 -1, i8 -1>
 ; CHECK-NEXT:    [[OR:%.*]] = icmp uge <2 x i8> [[TMP1]], [[Y]]
 ; CHECK-NEXT:    ret <2 x i1> [[OR]]
@@ -150,8 +150,8 @@ define <2 x i1> @eq_vector(<2 x i8> %x, <2 x i8> %y) {
 }
 
 define <2 x i1> @ne_vector_equal_5(<2 x i8> %x, <2 x i8> %y) {
-; CHECK-LABEL: define <2 x i1> @ne_vector_equal_5
-; CHECK-SAME: (<2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]]) {
+; CHECK-LABEL: define <2 x i1> @ne_vector_equal_5(
+; CHECK-SAME: <2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]]) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = add <2 x i8> [[X]], <i8 -6, i8 -6>
 ; CHECK-NEXT:    [[AND:%.*]] = icmp ult <2 x i8> [[TMP1]], [[Y]]
 ; CHECK-NEXT:    ret <2 x i1> [[AND]]
@@ -164,8 +164,8 @@ define <2 x i1> @ne_vector_equal_5(<2 x i8> %x, <2 x i8> %y) {
 }
 
 define <2 x i1> @eq_vector_equal_minus_1(<2 x i8> %x, <2 x i8> %y) {
-; CHECK-LABEL: define <2 x i1> @eq_vector_equal_minus_1
-; CHECK-SAME: (<2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]]) {
+; CHECK-LABEL: define <2 x i1> @eq_vector_equal_minus_1(
+; CHECK-SAME: <2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]]) {
 ; CHECK-NEXT:    [[OR:%.*]] = icmp uge <2 x i8> [[X]], [[Y]]
 ; CHECK-NEXT:    ret <2 x i1> [[OR]]
 ;
@@ -177,8 +177,8 @@ define <2 x i1> @eq_vector_equal_minus_1(<2 x i8> %x, <2 x i8> %y) {
 }
 
 define <2 x i1> @ne_vector_equal_minus_7(<2 x i8> %x, <2 x i8> %y) {
-; CHECK-LABEL: define <2 x i1> @ne_vector_equal_minus_7
-; CHECK-SAME: (<2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]]) {
+; CHECK-LABEL: define <2 x i1> @ne_vector_equal_minus_7(
+; CHECK-SAME: <2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]]) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = add <2 x i8> [[X]], <i8 6, i8 6>
 ; CHECK-NEXT:    [[AND:%.*]] = icmp ult <2 x i8> [[TMP1]], [[Y]]
 ; CHECK-NEXT:    ret <2 x i1> [[AND]]
@@ -191,8 +191,8 @@ define <2 x i1> @ne_vector_equal_minus_7(<2 x i8> %x, <2 x i8> %y) {
 }
 
 define <2 x i1> @eq_vector_unequal1(<2 x i8> %x, <2 x i8> %y) {
-; CHECK-LABEL: define <2 x i1> @eq_vector_unequal1
-; CHECK-SAME: (<2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]]) {
+; CHECK-LABEL: define <2 x i1> @eq_vector_unequal1(
+; CHECK-SAME: <2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]]) {
 ; CHECK-NEXT:    [[SUB:%.*]] = add <2 x i8> [[X]], <i8 -5, i8 -5>
 ; CHECK-NEXT:    [[C1:%.*]] = icmp eq <2 x i8> [[X]], <i8 2, i8 2>
 ; CHECK-NEXT:    [[C2:%.*]] = icmp ugt <2 x i8> [[SUB]], [[Y]]
@@ -207,8 +207,8 @@ define <2 x i1> @eq_vector_unequal1(<2 x i8> %x, <2 x i8> %y) {
 }
 
 define <2 x i1> @ne_vector_unequal2(<2 x i8> %x, <2 x i8> %y) {
-; CHECK-LABEL: define <2 x i1> @ne_vector_unequal2
-; CHECK-SAME: (<2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]]) {
+; CHECK-LABEL: define <2 x i1> @ne_vector_unequal2(
+; CHECK-SAME: <2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]]) {
 ; CHECK-NEXT:    [[ADD:%.*]] = add <2 x i8> [[X]], <i8 7, i8 7>
 ; CHECK-NEXT:    [[C1:%.*]] = icmp ne <2 x i8> [[X]], <i8 -3, i8 -3>
 ; CHECK-NEXT:    [[C2:%.*]] = icmp ule <2 x i8> [[ADD]], [[Y]]
@@ -226,8 +226,8 @@ define <2 x i1> @ne_vector_unequal2(<2 x i8> %x, <2 x i8> %y) {
 ; Tests with poison
 ; ==============================================================================
 define <2 x i1> @eq_vector_poison_icmp(<2 x i8> %x, <2 x i8> %y) {
-; CHECK-LABEL: define <2 x i1> @eq_vector_poison_icmp
-; CHECK-SAME: (<2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]]) {
+; CHECK-LABEL: define <2 x i1> @eq_vector_poison_icmp(
+; CHECK-SAME: <2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]]) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = add <2 x i8> [[X]], <i8 -6, i8 -6>
 ; CHECK-NEXT:    [[OR:%.*]] = icmp uge <2 x i8> [[TMP1]], [[Y]]
 ; CHECK-NEXT:    ret <2 x i1> [[OR]]
@@ -240,8 +240,8 @@ define <2 x i1> @eq_vector_poison_icmp(<2 x i8> %x, <2 x i8> %y) {
 }
 
 define <2 x i1> @eq_vector_poison_add(<2 x i8> %x, <2 x i8> %y) {
-; CHECK-LABEL: define <2 x i1> @eq_vector_poison_add
-; CHECK-SAME: (<2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]]) {
+; CHECK-LABEL: define <2 x i1> @eq_vector_poison_add(
+; CHECK-SAME: <2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]]) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = add <2 x i8> [[X]], <i8 -6, i8 -6>
 ; CHECK-NEXT:    [[OR:%.*]] = icmp uge <2 x i8> [[TMP1]], [[Y]]
 ; CHECK-NEXT:    ret <2 x i1> [[OR]]
@@ -257,8 +257,8 @@ define <2 x i1> @eq_vector_poison_add(<2 x i8> %x, <2 x i8> %y) {
 ; Tests with values commuted
 ; ==============================================================================
 define i1 @eq_commuted(i8 %x, i8 %py) {
-; CHECK-LABEL: define i1 @eq_commuted
-; CHECK-SAME: (i8 [[X:%.*]], i8 [[PY:%.*]]) {
+; CHECK-LABEL: define i1 @eq_commuted(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[PY:%.*]]) {
 ; CHECK-NEXT:    [[Y:%.*]] = sdiv i8 43, [[PY]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = add i8 [[X]], -1
 ; CHECK-NEXT:    [[OR:%.*]] = icmp uge i8 [[TMP1]], [[Y]]
@@ -272,8 +272,8 @@ define i1 @eq_commuted(i8 %x, i8 %py) {
 }
 
 define i1 @ne_commuted_equal_minus_1(i8 %x, i8 %py) {
-; CHECK-LABEL: define i1 @ne_commuted_equal_minus_1
-; CHECK-SAME: (i8 [[X:%.*]], i8 [[PY:%.*]]) {
+; CHECK-LABEL: define i1 @ne_commuted_equal_minus_1(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[PY:%.*]]) {
 ; CHECK-NEXT:    [[Y:%.*]] = sdiv i8 42, [[PY]]
 ; CHECK-NEXT:    [[AND:%.*]] = icmp ugt i8 [[Y]], [[X]]
 ; CHECK-NEXT:    ret i1 [[AND]]

From 397bcfef741315a68e75d174b8f746a11b42c0e2 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Fri, 16 Aug 2024 14:02:24 +0100
Subject: [PATCH 170/441] [X86] Fold extract_subvector(int_to_fp(x))
 vXi32/vXf32 cases to match existing fp_to_int folds

---
 llvm/lib/Target/X86/X86ISelLowering.cpp |  10 +-
 llvm/test/CodeGen/X86/pr50609.ll        |   2 +-
 llvm/test/CodeGen/X86/vec_int_to_fp.ll  | 124 ++++--------------------
 3 files changed, 27 insertions(+), 109 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index c6e1764edffc3a7..6385dab9c55e656 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -57203,11 +57203,13 @@ static SDValue combineEXTRACT_SUBVECTOR(SDNode *N, SelectionDAG &DAG,
       }
     }
     // v4i32 CVTPS2DQ(v4f32) / CVTPS2UDQ(v4f32).
-    if ((InOpcode == ISD::FP_TO_SINT ||
-         (InOpcode == ISD::FP_TO_UINT && Subtarget.hasVLX())) &&
-        VT == MVT::v4i32) {
+    // v4f32 CVTDQ2PS(v4i32) / CVTUDQ2PS(v4i32).
+    if ((InOpcode == ISD::FP_TO_SINT || InOpcode == ISD::SINT_TO_FP ||
+         ((InOpcode == ISD::FP_TO_UINT || InOpcode == ISD::UINT_TO_FP) &&
+          Subtarget.hasVLX())) &&
+        (VT == MVT::v4i32 || VT == MVT::v4f32)) {
       SDValue Src = InVec.getOperand(0);
-      if (Src.getValueType().getScalarType() == MVT::f32)
+      if (Src.getValueType().getScalarSizeInBits() == 32)
         return DAG.getNode(InOpcode, DL, VT,
                            extractSubVector(Src, IdxVal, DAG, DL, SizeInBits));
     }
diff --git a/llvm/test/CodeGen/X86/pr50609.ll b/llvm/test/CodeGen/X86/pr50609.ll
index ea85c312f38bd1c..3e39c567d79e6df 100644
--- a/llvm/test/CodeGen/X86/pr50609.ll
+++ b/llvm/test/CodeGen/X86/pr50609.ll
@@ -12,7 +12,7 @@ define void @PR50609(ptr noalias nocapture %RET, ptr noalias %aFOO, <16 x i32> %
 ; CHECK-NEXT:    vpsrld $30, %xmm3, %xmm3
 ; CHECK-NEXT:    vpaddd %xmm3, %xmm2, %xmm2
 ; CHECK-NEXT:    vpsrad $2, %xmm2, %xmm2
-; CHECK-NEXT:    vcvtdq2ps %ymm2, %ymm2
+; CHECK-NEXT:    vcvtdq2ps %xmm2, %xmm2
 ; CHECK-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[0,0,0,0]
 ; CHECK-NEXT:    vinsertf128 $1, %xmm2, %ymm2, %ymm2
 ; CHECK-NEXT:    vmaskmovps %ymm2, %ymm0, (%rdi)
diff --git a/llvm/test/CodeGen/X86/vec_int_to_fp.ll b/llvm/test/CodeGen/X86/vec_int_to_fp.ll
index e26de4be7066fa5..af841cf38b24aec 100644
--- a/llvm/test/CodeGen/X86/vec_int_to_fp.ll
+++ b/llvm/test/CodeGen/X86/vec_int_to_fp.ll
@@ -1478,32 +1478,11 @@ define <4 x float> @sitofp_8i16_to_4f32(<8 x i16> %a) {
 ; SSE41-NEXT:    cvtdq2ps %xmm0, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: sitofp_8i16_to_4f32:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
-; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; AVX1-NEXT:    vcvtdq2ps %ymm0, %ymm0
-; AVX1-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
-; AVX1-NEXT:    vzeroupper
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: sitofp_8i16_to_4f32:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
-; AVX2-NEXT:    vcvtdq2ps %ymm0, %ymm0
-; AVX2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
-; AVX2-NEXT:    vzeroupper
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: sitofp_8i16_to_4f32:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpmovsxwd %xmm0, %ymm0
-; AVX512-NEXT:    vcvtdq2ps %ymm0, %ymm0
-; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
-; AVX512-NEXT:    vzeroupper
-; AVX512-NEXT:    retq
+; AVX-LABEL: sitofp_8i16_to_4f32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmovsxwd %xmm0, %xmm0
+; AVX-NEXT:    vcvtdq2ps %xmm0, %xmm0
+; AVX-NEXT:    retq
   %cvt = sitofp <8 x i16> %a to <8 x float>
   %shuf = shufflevector <8 x float> %cvt, <8 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   ret <4 x float> %shuf
@@ -1549,32 +1528,11 @@ define <4 x float> @sitofp_16i8_to_4f32(<16 x i8> %a) {
 ; SSE41-NEXT:    cvtdq2ps %xmm0, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: sitofp_16i8_to_4f32:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; AVX1-NEXT:    vpmovsxbd %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; AVX1-NEXT:    vcvtdq2ps %ymm0, %ymm0
-; AVX1-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
-; AVX1-NEXT:    vzeroupper
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: sitofp_16i8_to_4f32:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpmovsxbd %xmm0, %ymm0
-; AVX2-NEXT:    vcvtdq2ps %ymm0, %ymm0
-; AVX2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
-; AVX2-NEXT:    vzeroupper
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: sitofp_16i8_to_4f32:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpmovsxbd %xmm0, %zmm0
-; AVX512-NEXT:    vcvtdq2ps %zmm0, %zmm0
-; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
-; AVX512-NEXT:    vzeroupper
-; AVX512-NEXT:    retq
+; AVX-LABEL: sitofp_16i8_to_4f32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmovsxbd %xmm0, %xmm0
+; AVX-NEXT:    vcvtdq2ps %xmm0, %xmm0
+; AVX-NEXT:    retq
   %cvt = sitofp <16 x i8> %a to <16 x float>
   %shuf = shufflevector <16 x float> %cvt, <16 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   ret <4 x float> %shuf
@@ -2354,32 +2312,11 @@ define <4 x float> @uitofp_8i16_to_4f32(<8 x i16> %a) {
 ; SSE41-NEXT:    cvtdq2ps %xmm0, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: uitofp_8i16_to_4f32:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
-; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX1-NEXT:    vcvtdq2ps %ymm0, %ymm0
-; AVX1-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
-; AVX1-NEXT:    vzeroupper
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: uitofp_8i16_to_4f32:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX2-NEXT:    vcvtdq2ps %ymm0, %ymm0
-; AVX2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
-; AVX2-NEXT:    vzeroupper
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: uitofp_8i16_to_4f32:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX512-NEXT:    vcvtdq2ps %ymm0, %ymm0
-; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
-; AVX512-NEXT:    vzeroupper
-; AVX512-NEXT:    retq
+; AVX-LABEL: uitofp_8i16_to_4f32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX-NEXT:    vcvtdq2ps %xmm0, %xmm0
+; AVX-NEXT:    retq
   %cvt = uitofp <8 x i16> %a to <8 x float>
   %shuf = shufflevector <8 x float> %cvt, <8 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   ret <4 x float> %shuf
@@ -2425,32 +2362,11 @@ define <4 x float> @uitofp_16i8_to_4f32(<16 x i8> %a) {
 ; SSE41-NEXT:    cvtdq2ps %xmm0, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: uitofp_16i8_to_4f32:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; AVX1-NEXT:    vcvtdq2ps %ymm0, %ymm0
-; AVX1-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
-; AVX1-NEXT:    vzeroupper
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: uitofp_16i8_to_4f32:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
-; AVX2-NEXT:    vcvtdq2ps %ymm0, %ymm0
-; AVX2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
-; AVX2-NEXT:    vzeroupper
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: uitofp_16i8_to_4f32:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
-; AVX512-NEXT:    vcvtdq2ps %zmm0, %zmm0
-; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
-; AVX512-NEXT:    vzeroupper
-; AVX512-NEXT:    retq
+; AVX-LABEL: uitofp_16i8_to_4f32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; AVX-NEXT:    vcvtdq2ps %xmm0, %xmm0
+; AVX-NEXT:    retq
   %cvt = uitofp <16 x i8> %a to <16 x float>
   %shuf = shufflevector <16 x float> %cvt, <16 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   ret <4 x float> %shuf

From 11ebc9d30eb5d58c41386bd0e5a350884d5d1020 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Fri, 16 Aug 2024 16:02:58 +0200
Subject: [PATCH 171/441] [InstCombine] Regenerate test checks (NFC)

---
 .../Transforms/InstCombine/and-xor-merge.ll   | 24 +++++------
 ...low-bit-mask-v2-and-icmp-eq-to-icmp-ule.ll | 42 +++++++++----------
 ...low-bit-mask-v2-and-icmp-ne-to-icmp-ugt.ll | 42 +++++++++----------
 ...low-bit-mask-v3-and-icmp-eq-to-icmp-ule.ll | 12 +++---
 ...low-bit-mask-v3-and-icmp-ne-to-icmp-ugt.ll | 12 +++---
 llvm/test/Transforms/InstCombine/cast.ll      | 12 +++---
 llvm/test/Transforms/InstCombine/demorgan.ll  |  4 +-
 .../Transforms/InstCombine/icmp-and-shift.ll  | 22 +++++-----
 8 files changed, 85 insertions(+), 85 deletions(-)

diff --git a/llvm/test/Transforms/InstCombine/and-xor-merge.ll b/llvm/test/Transforms/InstCombine/and-xor-merge.ll
index e6df4e32bae361c..80bdf67525faaea 100644
--- a/llvm/test/Transforms/InstCombine/and-xor-merge.ll
+++ b/llvm/test/Transforms/InstCombine/and-xor-merge.ll
@@ -43,8 +43,8 @@ define i32 @PR38781(i32 %a, i32 %b) {
 
 ; (a ^ 4) & (a ^ ~4) -> 0
 define i32 @PR75692_1(i32 %x) {
-; CHECK-LABEL: @PR75692_1
-; CHECK-NEXT:  ret i32 0
+; CHECK-LABEL: @PR75692_1(
+; CHECK-NEXT:    ret i32 0
 ;
   %t2 = xor i32 %x, 4
   %t3 = xor i32 %x, -5
@@ -54,11 +54,11 @@ define i32 @PR75692_1(i32 %x) {
 
 ; (a ^ 4) & (a ^ 3) is not zero
 define i32 @PR75692_2(i32 %x) {
-; CHECK-LABEL: @PR75692_2
-; CHECK-NEXT:  %t2 = xor i32 %x, 4
-; CHECK-NEXT:  %t3 = xor i32 %x, -4
-; CHECK-NEXT:  %t4 = and i32 %t2, %t3
-; CHECK-NEXT:  ret i32 %t4
+; CHECK-LABEL: @PR75692_2(
+; CHECK-NEXT:    [[T2:%.*]] = xor i32 [[X:%.*]], 4
+; CHECK-NEXT:    [[T3:%.*]] = xor i32 [[X]], -4
+; CHECK-NEXT:    [[T4:%.*]] = and i32 [[T2]], [[T3]]
+; CHECK-NEXT:    ret i32 [[T4]]
 ;
   %t2 = xor i32 %x, 4
   %t3 = xor i32 %x, -4
@@ -68,11 +68,11 @@ define i32 @PR75692_2(i32 %x) {
 
 ; (a ^ 4) & (b ^ ~4) is not zero, since a != b is possible
 define i32 @PR75692_3(i32 %x, i32 %y) {
-; CHECK-LABEL: @PR75692_3
-; CHECK-NEXT:  %t2 = xor i32 %x, 4
-; CHECK-NEXT:  %t3 = xor i32 %y, -5
-; CHECK-NEXT:  %t4 = and i32 %t2, %t3
-; CHECK-NEXT:  ret i32 %t4
+; CHECK-LABEL: @PR75692_3(
+; CHECK-NEXT:    [[T2:%.*]] = xor i32 [[X:%.*]], 4
+; CHECK-NEXT:    [[T3:%.*]] = xor i32 [[Y:%.*]], -5
+; CHECK-NEXT:    [[T4:%.*]] = and i32 [[T2]], [[T3]]
+; CHECK-NEXT:    ret i32 [[T4]]
 ;
   %t2 = xor i32 %x, 4
   %t3 = xor i32 %y, -5
diff --git a/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v2-and-icmp-eq-to-icmp-ule.ll b/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v2-and-icmp-eq-to-icmp-ule.ll
index 0a7de501ca0225f..443efbe1ecaf6d9 100644
--- a/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v2-and-icmp-eq-to-icmp-ule.ll
+++ b/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v2-and-icmp-eq-to-icmp-ule.ll
@@ -16,8 +16,8 @@
 
 define i1 @p0(i8 %x, i8 %y) {
 ; CHECK-LABEL: @p0(
-; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr i8 [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[RET:%.*]] = icmp eq i8 [[X_HIGHBITS]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp eq i8 [[TMP1]], 0
 ; CHECK-NEXT:    ret i1 [[RET]]
 ;
   %t0 = shl i8 -1, %y
@@ -33,8 +33,8 @@ define i1 @p0(i8 %x, i8 %y) {
 
 define <2 x i1> @p1_vec(<2 x i8> %x, <2 x i8> %y) {
 ; CHECK-LABEL: @p1_vec(
-; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr <2 x i8> [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[RET:%.*]] = icmp eq <2 x i8> [[X_HIGHBITS]], zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <2 x i8> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp eq <2 x i8> [[TMP1]], zeroinitializer
 ; CHECK-NEXT:    ret <2 x i1> [[RET]]
 ;
   %t0 = shl <2 x i8> <i8 -1, i8 -1>, %y
@@ -46,8 +46,8 @@ define <2 x i1> @p1_vec(<2 x i8> %x, <2 x i8> %y) {
 
 define <3 x i1> @p2_vec_poison0(<3 x i8> %x, <3 x i8> %y) {
 ; CHECK-LABEL: @p2_vec_poison0(
-; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr <3 x i8> [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[RET:%.*]] = icmp eq <3 x i8> [[X_HIGHBITS]], zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <3 x i8> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp eq <3 x i8> [[TMP1]], zeroinitializer
 ; CHECK-NEXT:    ret <3 x i1> [[RET]]
 ;
   %t0 = shl <3 x i8> <i8 -1, i8 poison, i8 -1>, %y
@@ -59,8 +59,8 @@ define <3 x i1> @p2_vec_poison0(<3 x i8> %x, <3 x i8> %y) {
 
 define <3 x i1> @p3_vec_poison0(<3 x i8> %x, <3 x i8> %y) {
 ; CHECK-LABEL: @p3_vec_poison0(
-; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr <3 x i8> [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[RET:%.*]] = icmp eq <3 x i8> [[X_HIGHBITS]], zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <3 x i8> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp eq <3 x i8> [[TMP1]], zeroinitializer
 ; CHECK-NEXT:    ret <3 x i1> [[RET]]
 ;
   %t0 = shl <3 x i8> <i8 -1, i8 -1, i8 -1>, %y
@@ -72,8 +72,8 @@ define <3 x i1> @p3_vec_poison0(<3 x i8> %x, <3 x i8> %y) {
 
 define <3 x i1> @p4_vec_poison2(<3 x i8> %x, <3 x i8> %y) {
 ; CHECK-LABEL: @p4_vec_poison2(
-; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr <3 x i8> [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[RET:%.*]] = icmp eq <3 x i8> [[X_HIGHBITS]], zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <3 x i8> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp eq <3 x i8> [[TMP1]], zeroinitializer
 ; CHECK-NEXT:    ret <3 x i1> [[RET]]
 ;
   %t0 = shl <3 x i8> <i8 -1, i8 poison, i8 -1>, %y
@@ -92,8 +92,8 @@ declare i8 @gen8()
 define i1 @c0(i8 %y) {
 ; CHECK-LABEL: @c0(
 ; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
-; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr i8 [[X]], [[Y:%.*]]
-; CHECK-NEXT:    [[RET:%.*]] = icmp eq i8 [[X_HIGHBITS]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i8 [[X]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp eq i8 [[TMP1]], 0
 ; CHECK-NEXT:    ret i1 [[RET]]
 ;
   %t0 = shl i8 -1, %y
@@ -107,8 +107,8 @@ define i1 @c0(i8 %y) {
 define i1 @c1(i8 %y) {
 ; CHECK-LABEL: @c1(
 ; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
-; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr i8 [[X]], [[Y:%.*]]
-; CHECK-NEXT:    [[RET:%.*]] = icmp eq i8 [[X_HIGHBITS]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i8 [[X]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp eq i8 [[TMP1]], 0
 ; CHECK-NEXT:    ret i1 [[RET]]
 ;
   %t0 = shl i8 -1, %y
@@ -122,8 +122,8 @@ define i1 @c1(i8 %y) {
 define i1 @c2(i8 %y) {
 ; CHECK-LABEL: @c2(
 ; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
-; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr i8 [[X]], [[Y:%.*]]
-; CHECK-NEXT:    [[RET:%.*]] = icmp eq i8 [[X_HIGHBITS]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i8 [[X]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp eq i8 [[TMP1]], 0
 ; CHECK-NEXT:    ret i1 [[RET]]
 ;
   %t0 = shl i8 -1, %y
@@ -144,8 +144,8 @@ define i1 @oneuse0(i8 %x, i8 %y) {
 ; CHECK-LABEL: @oneuse0(
 ; CHECK-NEXT:    [[T0:%.*]] = shl nsw i8 -1, [[Y:%.*]]
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
-; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = and i8 [[T0]], [[X:%.*]]
-; CHECK-NEXT:    [[RET:%.*]] = icmp eq i8 [[X_HIGHBITS]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[T0]], [[X:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp eq i8 [[TMP1]], 0
 ; CHECK-NEXT:    ret i1 [[RET]]
 ;
   %t0 = shl i8 -1, %y
@@ -285,9 +285,9 @@ define i1 @n1(i8 %x, i8 %y) {
 define i1 @n2(i8 %x, i8 %y) {
 ; CHECK-LABEL: @n2(
 ; CHECK-NEXT:    [[T0:%.*]] = shl nsw i8 -1, [[Y:%.*]]
-; CHECK-NEXT:    [[T1:%.*]] = xor i8 [[T0]], -2
-; CHECK-NEXT:    [[T2:%.*]] = and i8 [[T1]], [[X:%.*]]
-; CHECK-NEXT:    [[RET:%.*]] = icmp eq i8 [[T2]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i8 [[T0]], -2
+; CHECK-NEXT:    [[TMP2:%.*]] = and i8 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp eq i8 [[TMP2]], 0
 ; CHECK-NEXT:    ret i1 [[RET]]
 ;
   %t0 = shl i8 -1, %y
diff --git a/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v2-and-icmp-ne-to-icmp-ugt.ll b/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v2-and-icmp-ne-to-icmp-ugt.ll
index 54ff87676e71d9c..ffde4eae777cb28 100644
--- a/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v2-and-icmp-ne-to-icmp-ugt.ll
+++ b/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v2-and-icmp-ne-to-icmp-ugt.ll
@@ -16,8 +16,8 @@
 
 define i1 @p0(i8 %x, i8 %y) {
 ; CHECK-LABEL: @p0(
-; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr i8 [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[RET:%.*]] = icmp ne i8 [[X_HIGHBITS]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp ne i8 [[TMP1]], 0
 ; CHECK-NEXT:    ret i1 [[RET]]
 ;
   %t0 = shl i8 -1, %y
@@ -33,8 +33,8 @@ define i1 @p0(i8 %x, i8 %y) {
 
 define <2 x i1> @p1_vec(<2 x i8> %x, <2 x i8> %y) {
 ; CHECK-LABEL: @p1_vec(
-; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr <2 x i8> [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[RET:%.*]] = icmp ne <2 x i8> [[X_HIGHBITS]], zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <2 x i8> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp ne <2 x i8> [[TMP1]], zeroinitializer
 ; CHECK-NEXT:    ret <2 x i1> [[RET]]
 ;
   %t0 = shl <2 x i8> <i8 -1, i8 -1>, %y
@@ -46,8 +46,8 @@ define <2 x i1> @p1_vec(<2 x i8> %x, <2 x i8> %y) {
 
 define <3 x i1> @p2_vec_poison0(<3 x i8> %x, <3 x i8> %y) {
 ; CHECK-LABEL: @p2_vec_poison0(
-; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr <3 x i8> [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[RET:%.*]] = icmp ne <3 x i8> [[X_HIGHBITS]], zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <3 x i8> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp ne <3 x i8> [[TMP1]], zeroinitializer
 ; CHECK-NEXT:    ret <3 x i1> [[RET]]
 ;
   %t0 = shl <3 x i8> <i8 -1, i8 poison, i8 -1>, %y
@@ -59,8 +59,8 @@ define <3 x i1> @p2_vec_poison0(<3 x i8> %x, <3 x i8> %y) {
 
 define <3 x i1> @p3_vec_poison0(<3 x i8> %x, <3 x i8> %y) {
 ; CHECK-LABEL: @p3_vec_poison0(
-; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr <3 x i8> [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[RET:%.*]] = icmp ne <3 x i8> [[X_HIGHBITS]], zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <3 x i8> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp ne <3 x i8> [[TMP1]], zeroinitializer
 ; CHECK-NEXT:    ret <3 x i1> [[RET]]
 ;
   %t0 = shl <3 x i8> <i8 -1, i8 -1, i8 -1>, %y
@@ -72,8 +72,8 @@ define <3 x i1> @p3_vec_poison0(<3 x i8> %x, <3 x i8> %y) {
 
 define <3 x i1> @p4_vec_poison2(<3 x i8> %x, <3 x i8> %y) {
 ; CHECK-LABEL: @p4_vec_poison2(
-; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr <3 x i8> [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[RET:%.*]] = icmp ne <3 x i8> [[X_HIGHBITS]], zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr <3 x i8> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp ne <3 x i8> [[TMP1]], zeroinitializer
 ; CHECK-NEXT:    ret <3 x i1> [[RET]]
 ;
   %t0 = shl <3 x i8> <i8 -1, i8 poison, i8 -1>, %y
@@ -92,8 +92,8 @@ declare i8 @gen8()
 define i1 @c0(i8 %y) {
 ; CHECK-LABEL: @c0(
 ; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
-; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr i8 [[X]], [[Y:%.*]]
-; CHECK-NEXT:    [[RET:%.*]] = icmp ne i8 [[X_HIGHBITS]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i8 [[X]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp ne i8 [[TMP1]], 0
 ; CHECK-NEXT:    ret i1 [[RET]]
 ;
   %t0 = shl i8 -1, %y
@@ -107,8 +107,8 @@ define i1 @c0(i8 %y) {
 define i1 @c1(i8 %y) {
 ; CHECK-LABEL: @c1(
 ; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
-; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr i8 [[X]], [[Y:%.*]]
-; CHECK-NEXT:    [[RET:%.*]] = icmp ne i8 [[X_HIGHBITS]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i8 [[X]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp ne i8 [[TMP1]], 0
 ; CHECK-NEXT:    ret i1 [[RET]]
 ;
   %t0 = shl i8 -1, %y
@@ -122,8 +122,8 @@ define i1 @c1(i8 %y) {
 define i1 @c2(i8 %y) {
 ; CHECK-LABEL: @c2(
 ; CHECK-NEXT:    [[X:%.*]] = call i8 @gen8()
-; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = lshr i8 [[X]], [[Y:%.*]]
-; CHECK-NEXT:    [[RET:%.*]] = icmp ne i8 [[X_HIGHBITS]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i8 [[X]], [[Y:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp ne i8 [[TMP1]], 0
 ; CHECK-NEXT:    ret i1 [[RET]]
 ;
   %t0 = shl i8 -1, %y
@@ -144,8 +144,8 @@ define i1 @oneuse0(i8 %x, i8 %y) {
 ; CHECK-LABEL: @oneuse0(
 ; CHECK-NEXT:    [[T0:%.*]] = shl nsw i8 -1, [[Y:%.*]]
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
-; CHECK-NEXT:    [[X_HIGHBITS:%.*]] = and i8 [[T0]], [[X:%.*]]
-; CHECK-NEXT:    [[RET:%.*]] = icmp ne i8 [[X_HIGHBITS]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[T0]], [[X:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp ne i8 [[TMP1]], 0
 ; CHECK-NEXT:    ret i1 [[RET]]
 ;
   %t0 = shl i8 -1, %y
@@ -285,9 +285,9 @@ define i1 @n1(i8 %x, i8 %y) {
 define i1 @n2(i8 %x, i8 %y) {
 ; CHECK-LABEL: @n2(
 ; CHECK-NEXT:    [[T0:%.*]] = shl nsw i8 -1, [[Y:%.*]]
-; CHECK-NEXT:    [[T1:%.*]] = xor i8 [[T0]], -2
-; CHECK-NEXT:    [[T2:%.*]] = and i8 [[T1]], [[X:%.*]]
-; CHECK-NEXT:    [[RET:%.*]] = icmp ne i8 [[T2]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i8 [[T0]], -2
+; CHECK-NEXT:    [[TMP2:%.*]] = and i8 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp ne i8 [[TMP2]], 0
 ; CHECK-NEXT:    ret i1 [[RET]]
 ;
   %t0 = shl i8 -1, %y
diff --git a/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v3-and-icmp-eq-to-icmp-ule.ll b/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v3-and-icmp-eq-to-icmp-ule.ll
index c7c57b601eab384..946bb03e04f7e4e 100644
--- a/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v3-and-icmp-eq-to-icmp-ule.ll
+++ b/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v3-and-icmp-eq-to-icmp-ule.ll
@@ -251,9 +251,9 @@ define i1 @n1(i8 %x, i8 %y) {
 ; CHECK-LABEL: @n1(
 ; CHECK-NEXT:    [[T0:%.*]] = shl nsw i8 -1, [[Y:%.*]]
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
-; CHECK-NEXT:    [[T1:%.*]] = sub i8 0, [[T0]]
-; CHECK-NEXT:    [[T2:%.*]] = and i8 [[T1]], [[X:%.*]]
-; CHECK-NEXT:    [[RET:%.*]] = icmp eq i8 [[T2]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i8 0, [[T0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i8 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp eq i8 [[TMP2]], 0
 ; CHECK-NEXT:    ret i1 [[RET]]
 ;
   %t0 = shl i8 -1, %y ; not 1
@@ -268,9 +268,9 @@ define i1 @n2(i8 %x, i8 %y) {
 ; CHECK-LABEL: @n2(
 ; CHECK-NEXT:    [[T0:%.*]] = shl nuw i8 1, [[Y:%.*]]
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
-; CHECK-NEXT:    [[T1:%.*]] = sub nuw i8 -2, [[T0]]
-; CHECK-NEXT:    [[T2:%.*]] = and i8 [[T1]], [[X:%.*]]
-; CHECK-NEXT:    [[RET:%.*]] = icmp eq i8 [[T2]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = sub nuw i8 -2, [[T0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i8 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp eq i8 [[TMP2]], 0
 ; CHECK-NEXT:    ret i1 [[RET]]
 ;
   %t0 = shl i8 1, %y
diff --git a/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v3-and-icmp-ne-to-icmp-ugt.ll b/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v3-and-icmp-ne-to-icmp-ugt.ll
index d5826524f1637c2..63d406d54179fc3 100644
--- a/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v3-and-icmp-ne-to-icmp-ugt.ll
+++ b/llvm/test/Transforms/InstCombine/canonicalize-low-bit-mask-v3-and-icmp-ne-to-icmp-ugt.ll
@@ -251,9 +251,9 @@ define i1 @n1(i8 %x, i8 %y) {
 ; CHECK-LABEL: @n1(
 ; CHECK-NEXT:    [[T0:%.*]] = shl nsw i8 -1, [[Y:%.*]]
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
-; CHECK-NEXT:    [[T1:%.*]] = sub i8 0, [[T0]]
-; CHECK-NEXT:    [[T2:%.*]] = and i8 [[T1]], [[X:%.*]]
-; CHECK-NEXT:    [[RET:%.*]] = icmp ne i8 [[T2]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i8 0, [[T0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i8 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp ne i8 [[TMP2]], 0
 ; CHECK-NEXT:    ret i1 [[RET]]
 ;
   %t0 = shl i8 -1, %y ; not 1
@@ -268,9 +268,9 @@ define i1 @n2(i8 %x, i8 %y) {
 ; CHECK-LABEL: @n2(
 ; CHECK-NEXT:    [[T0:%.*]] = shl nuw i8 1, [[Y:%.*]]
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
-; CHECK-NEXT:    [[T1:%.*]] = sub nuw i8 -2, [[T0]]
-; CHECK-NEXT:    [[T2:%.*]] = and i8 [[T1]], [[X:%.*]]
-; CHECK-NEXT:    [[RET:%.*]] = icmp ne i8 [[T2]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = sub nuw i8 -2, [[T0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i8 [[TMP1]], [[X:%.*]]
+; CHECK-NEXT:    [[RET:%.*]] = icmp ne i8 [[TMP2]], 0
 ; CHECK-NEXT:    ret i1 [[RET]]
 ;
   %t0 = shl i8 1, %y
diff --git a/llvm/test/Transforms/InstCombine/cast.ll b/llvm/test/Transforms/InstCombine/cast.ll
index 564e82945c13f03..43c198b9e19d8eb 100644
--- a/llvm/test/Transforms/InstCombine/cast.ll
+++ b/llvm/test/Transforms/InstCombine/cast.ll
@@ -512,8 +512,8 @@ define <2 x i16> @test40vec_poison(<2 x i16> %a) {
 ; ALL-LABEL: @test40vec_poison(
 ; ALL-NEXT:    [[T21:%.*]] = lshr <2 x i16> [[A:%.*]], <i16 9, i16 poison>
 ; ALL-NEXT:    [[T5:%.*]] = shl <2 x i16> [[A]], <i16 8, i16 poison>
-; ALL-NEXT:    [[R:%.*]] = or disjoint <2 x i16> [[T21]], [[T5]]
-; ALL-NEXT:    ret <2 x i16> [[R]]
+; ALL-NEXT:    [[T32:%.*]] = or disjoint <2 x i16> [[T21]], [[T5]]
+; ALL-NEXT:    ret <2 x i16> [[T32]]
 ;
   %t = zext <2 x i16> %a to <2 x i32>
   %t21 = lshr <2 x i32> %t, <i32 9, i32 poison>
@@ -1994,8 +1994,8 @@ define <2 x i8> @trunc_lshr_zext_uniform(<2 x i8> %A) {
 
 define <2 x i8> @trunc_lshr_zext_uniform_poison(<2 x i8> %A) {
 ; ALL-LABEL: @trunc_lshr_zext_uniform_poison(
-; ALL-NEXT:    [[D:%.*]] = lshr <2 x i8> [[A:%.*]], <i8 6, i8 poison>
-; ALL-NEXT:    ret <2 x i8> [[D]]
+; ALL-NEXT:    [[C:%.*]] = lshr <2 x i8> [[A:%.*]], <i8 6, i8 poison>
+; ALL-NEXT:    ret <2 x i8> [[C]]
 ;
   %B = zext <2 x i8> %A to <2 x i32>
   %C = lshr <2 x i32> %B, <i32 6, i32 poison>
@@ -2016,8 +2016,8 @@ define <2 x i8> @trunc_lshr_zext_nonuniform(<2 x i8> %A) {
 
 define <3 x i8> @trunc_lshr_zext_nonuniform_poison(<3 x i8> %A) {
 ; ALL-LABEL: @trunc_lshr_zext_nonuniform_poison(
-; ALL-NEXT:    [[D:%.*]] = lshr <3 x i8> [[A:%.*]], <i8 6, i8 2, i8 poison>
-; ALL-NEXT:    ret <3 x i8> [[D]]
+; ALL-NEXT:    [[C:%.*]] = lshr <3 x i8> [[A:%.*]], <i8 6, i8 2, i8 poison>
+; ALL-NEXT:    ret <3 x i8> [[C]]
 ;
   %B = zext <3 x i8> %A to <3 x i32>
   %C = lshr <3 x i32> %B, <i32 6, i32 2, i32 poison>
diff --git a/llvm/test/Transforms/InstCombine/demorgan.ll b/llvm/test/Transforms/InstCombine/demorgan.ll
index c9196b6d49afffd..11052d38f9bc7e1 100644
--- a/llvm/test/Transforms/InstCombine/demorgan.ll
+++ b/llvm/test/Transforms/InstCombine/demorgan.ll
@@ -119,8 +119,8 @@ define i32 @test3(i32 %A, i32 %B) {
 
 define i32 @test4(i32 %A) {
 ; CHECK-LABEL: @test4(
-; CHECK-NEXT:    [[NOTC1:%.*]] = or i32 [[A:%.*]], -6
-; CHECK-NEXT:    ret i32 [[NOTC1]]
+; CHECK-NEXT:    [[NOTC:%.*]] = or i32 [[A:%.*]], -6
+; CHECK-NEXT:    ret i32 [[NOTC]]
 ;
   %nota = xor i32 %A, -1
   %c = and i32 %nota, 5
diff --git a/llvm/test/Transforms/InstCombine/icmp-and-shift.ll b/llvm/test/Transforms/InstCombine/icmp-and-shift.ll
index 65b9749ffc1bf52..2f797d726afe322 100644
--- a/llvm/test/Transforms/InstCombine/icmp-and-shift.ll
+++ b/llvm/test/Transforms/InstCombine/icmp-and-shift.ll
@@ -522,8 +522,8 @@ define i1 @slt_and_shl_one(i8 %x, i8 %y) {
 
 define i1 @fold_eq_lhs(i8 %x, i8 %y) {
 ; CHECK-LABEL: @fold_eq_lhs(
-; CHECK-NEXT:    [[AND:%.*]] = lshr i8 [[Y:%.*]], [[X:%.*]]
-; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[AND]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i8 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[TMP1]], 0
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %shl = shl i8 -1, %x
@@ -563,8 +563,8 @@ define i1 @fold_eq_lhs_fail_multiuse_shl(i8 %x, i8 %y) {
 define i1 @fold_ne_rhs(i8 %x, i8 %yy) {
 ; CHECK-LABEL: @fold_ne_rhs(
 ; CHECK-NEXT:    [[Y:%.*]] = xor i8 [[YY:%.*]], 123
-; CHECK-NEXT:    [[AND:%.*]] = lshr i8 [[Y]], [[X:%.*]]
-; CHECK-NEXT:    [[R:%.*]] = icmp ne i8 [[AND]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i8 [[Y]], [[X:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i8 [[TMP1]], 0
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %y = xor i8 %yy, 123
@@ -622,8 +622,8 @@ define i1 @test_shr_and_1_ne_0(i32 %a, i32 %b) {
 define i1 @test_const_shr_and_1_ne_0(i32 %b) {
 ; CHECK-LABEL: @test_const_shr_and_1_ne_0(
 ; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i32 1, [[B:%.*]]
-; CHECK-NEXT:    [[AND:%.*]] = and i32 [[TMP1]], 42
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[AND]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], 42
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[TMP2]], 0
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %shr = lshr i32 42, %b
@@ -635,9 +635,9 @@ define i1 @test_const_shr_and_1_ne_0(i32 %b) {
 define i1 @test_not_const_shr_and_1_ne_0(i32 %b) {
 ; CHECK-LABEL: @test_not_const_shr_and_1_ne_0(
 ; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i32 1, [[B:%.*]]
-; CHECK-NEXT:    [[AND:%.*]] = and i32 [[TMP1]], 42
-; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq i32 [[AND]], 0
-; CHECK-NEXT:    ret i1 [[CMP_NOT]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], 42
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP2]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %shr = lshr i32 42, %b
   %and = and i32 %shr, 1
@@ -648,8 +648,8 @@ define i1 @test_not_const_shr_and_1_ne_0(i32 %b) {
 define i1 @test_const_shr_exact_and_1_ne_0(i32 %b) {
 ; CHECK-LABEL: @test_const_shr_exact_and_1_ne_0(
 ; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i32 1, [[B:%.*]]
-; CHECK-NEXT:    [[AND:%.*]] = and i32 [[TMP1]], 42
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[AND]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], 42
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[TMP2]], 0
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %shr = lshr exact i32 42, %b

From b9fecea56f031c17665737b4844628027cf3c431 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Fri, 16 Aug 2024 16:15:23 +0200
Subject: [PATCH 172/441] [PhaseOrdering] Regenerate test checks (NFC)

---
 .../PhaseOrdering/ARM/arm_mult_q15.ll         | 52 +++++++++----------
 1 file changed, 26 insertions(+), 26 deletions(-)

diff --git a/llvm/test/Transforms/PhaseOrdering/ARM/arm_mult_q15.ll b/llvm/test/Transforms/PhaseOrdering/ARM/arm_mult_q15.ll
index c6126727598ef28..8b82f21e38c931b 100644
--- a/llvm/test/Transforms/PhaseOrdering/ARM/arm_mult_q15.ll
+++ b/llvm/test/Transforms/PhaseOrdering/ARM/arm_mult_q15.ll
@@ -15,7 +15,7 @@ define void @arm_mult_q15(ptr %pSrcA, ptr %pSrcB, ptr noalias %pDst, i32 %blockS
 ; CHECK-NEXT:    br i1 [[CMP_NOT2]], label [[WHILE_END:%.*]], label [[WHILE_BODY_PREHEADER:%.*]]
 ; CHECK:       while.body.preheader:
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[BLOCKSIZE]], 8
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[WHILE_BODY_PREHEADER16:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[WHILE_BODY_PREHEADER18:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[N_VEC:%.*]] = and i32 [[BLOCKSIZE]], -8
 ; CHECK-NEXT:    [[IND_END:%.*]] = and i32 [[BLOCKSIZE]], 7
@@ -28,27 +28,27 @@ define void @arm_mult_q15(ptr %pSrcA, ptr %pSrcB, ptr noalias %pDst, i32 %blockS
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP3:%.*]] = shl i32 [[INDEX]], 1
-; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[PSRCA]], i32 [[TMP3]]
-; CHECK-NEXT:    [[TMP4:%.*]] = shl i32 [[INDEX]], 1
-; CHECK-NEXT:    [[NEXT_GEP13:%.*]] = getelementptr i8, ptr [[PDST]], i32 [[TMP4]]
-; CHECK-NEXT:    [[TMP5:%.*]] = shl i32 [[INDEX]], 1
-; CHECK-NEXT:    [[NEXT_GEP14:%.*]] = getelementptr i8, ptr [[PSRCB]], i32 [[TMP5]]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = shl i32 [[INDEX]], 1
+; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[PSRCA]], i32 [[OFFSET_IDX]]
+; CHECK-NEXT:    [[OFFSET_IDX13:%.*]] = shl i32 [[INDEX]], 1
+; CHECK-NEXT:    [[NEXT_GEP14:%.*]] = getelementptr i8, ptr [[PDST]], i32 [[OFFSET_IDX13]]
+; CHECK-NEXT:    [[OFFSET_IDX15:%.*]] = shl i32 [[INDEX]], 1
+; CHECK-NEXT:    [[NEXT_GEP16:%.*]] = getelementptr i8, ptr [[PSRCB]], i32 [[OFFSET_IDX15]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x i16>, ptr [[NEXT_GEP]], align 2
-; CHECK-NEXT:    [[TMP6:%.*]] = sext <8 x i16> [[WIDE_LOAD]] to <8 x i32>
-; CHECK-NEXT:    [[WIDE_LOAD15:%.*]] = load <8 x i16>, ptr [[NEXT_GEP14]], align 2
-; CHECK-NEXT:    [[TMP7:%.*]] = sext <8 x i16> [[WIDE_LOAD15]] to <8 x i32>
-; CHECK-NEXT:    [[TMP8:%.*]] = mul nsw <8 x i32> [[TMP7]], [[TMP6]]
-; CHECK-NEXT:    [[TMP9:%.*]] = ashr <8 x i32> [[TMP8]], <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
-; CHECK-NEXT:    [[TMP10:%.*]] = tail call <8 x i32> @llvm.smin.v8i32(<8 x i32> [[TMP9]], <8 x i32> <i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767>)
-; CHECK-NEXT:    [[TMP11:%.*]] = trunc <8 x i32> [[TMP10]] to <8 x i16>
-; CHECK-NEXT:    store <8 x i16> [[TMP11]], ptr [[NEXT_GEP13]], align 2
+; CHECK-NEXT:    [[TMP3:%.*]] = sext <8 x i16> [[WIDE_LOAD]] to <8 x i32>
+; CHECK-NEXT:    [[WIDE_LOAD17:%.*]] = load <8 x i16>, ptr [[NEXT_GEP16]], align 2
+; CHECK-NEXT:    [[TMP4:%.*]] = sext <8 x i16> [[WIDE_LOAD17]] to <8 x i32>
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nsw <8 x i32> [[TMP4]], [[TMP3]]
+; CHECK-NEXT:    [[TMP6:%.*]] = ashr <8 x i32> [[TMP5]], <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT:    [[TMP7:%.*]] = tail call <8 x i32> @llvm.smin.v8i32(<8 x i32> [[TMP6]], <8 x i32> <i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767>)
+; CHECK-NEXT:    [[TMP8:%.*]] = trunc <8 x i32> [[TMP7]] to <8 x i16>
+; CHECK-NEXT:    store <8 x i16> [[TMP8]], ptr [[NEXT_GEP14]], align 2
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
-; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[N_VEC]], [[BLOCKSIZE]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[WHILE_END]], label [[WHILE_BODY_PREHEADER16]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[WHILE_END]], label [[WHILE_BODY_PREHEADER18]]
 ; CHECK:       while.body.preheader18:
 ; CHECK-NEXT:    [[BLKCNT_06_PH:%.*]] = phi i32 [ [[BLOCKSIZE]], [[WHILE_BODY_PREHEADER]] ], [ [[IND_END]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    [[PSRCA_ADDR_05_PH:%.*]] = phi ptr [ [[PSRCA]], [[WHILE_BODY_PREHEADER]] ], [ [[IND_END7]], [[MIDDLE_BLOCK]] ]
@@ -56,16 +56,16 @@ define void @arm_mult_q15(ptr %pSrcA, ptr %pSrcB, ptr noalias %pDst, i32 %blockS
 ; CHECK-NEXT:    [[PSRCB_ADDR_03_PH:%.*]] = phi ptr [ [[PSRCB]], [[WHILE_BODY_PREHEADER]] ], [ [[IND_END11]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    br label [[WHILE_BODY:%.*]]
 ; CHECK:       while.body:
-; CHECK-NEXT:    [[BLKCNT_06:%.*]] = phi i32 [ [[DEC:%.*]], [[WHILE_BODY]] ], [ [[BLKCNT_06_PH]], [[WHILE_BODY_PREHEADER16]] ]
-; CHECK-NEXT:    [[PSRCA_ADDR_05:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], [[WHILE_BODY]] ], [ [[PSRCA_ADDR_05_PH]], [[WHILE_BODY_PREHEADER16]] ]
-; CHECK-NEXT:    [[PDST_ADDR_04:%.*]] = phi ptr [ [[INCDEC_PTR4:%.*]], [[WHILE_BODY]] ], [ [[PDST_ADDR_04_PH]], [[WHILE_BODY_PREHEADER16]] ]
-; CHECK-NEXT:    [[PSRCB_ADDR_03:%.*]] = phi ptr [ [[INCDEC_PTR1:%.*]], [[WHILE_BODY]] ], [ [[PSRCB_ADDR_03_PH]], [[WHILE_BODY_PREHEADER16]] ]
+; CHECK-NEXT:    [[BLKCNT_06:%.*]] = phi i32 [ [[DEC:%.*]], [[WHILE_BODY]] ], [ [[BLKCNT_06_PH]], [[WHILE_BODY_PREHEADER18]] ]
+; CHECK-NEXT:    [[PSRCA_ADDR_05:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], [[WHILE_BODY]] ], [ [[PSRCA_ADDR_05_PH]], [[WHILE_BODY_PREHEADER18]] ]
+; CHECK-NEXT:    [[PDST_ADDR_04:%.*]] = phi ptr [ [[INCDEC_PTR4:%.*]], [[WHILE_BODY]] ], [ [[PDST_ADDR_04_PH]], [[WHILE_BODY_PREHEADER18]] ]
+; CHECK-NEXT:    [[PSRCB_ADDR_03:%.*]] = phi ptr [ [[INCDEC_PTR1:%.*]], [[WHILE_BODY]] ], [ [[PSRCB_ADDR_03_PH]], [[WHILE_BODY_PREHEADER18]] ]
 ; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds i8, ptr [[PSRCA_ADDR_05]], i32 2
-; CHECK-NEXT:    [[TMP13:%.*]] = load i16, ptr [[PSRCA_ADDR_05]], align 2
-; CHECK-NEXT:    [[CONV:%.*]] = sext i16 [[TMP13]] to i32
+; CHECK-NEXT:    [[TMP10:%.*]] = load i16, ptr [[PSRCA_ADDR_05]], align 2
+; CHECK-NEXT:    [[CONV:%.*]] = sext i16 [[TMP10]] to i32
 ; CHECK-NEXT:    [[INCDEC_PTR1]] = getelementptr inbounds i8, ptr [[PSRCB_ADDR_03]], i32 2
-; CHECK-NEXT:    [[TMP14:%.*]] = load i16, ptr [[PSRCB_ADDR_03]], align 2
-; CHECK-NEXT:    [[CONV2:%.*]] = sext i16 [[TMP14]] to i32
+; CHECK-NEXT:    [[TMP11:%.*]] = load i16, ptr [[PSRCB_ADDR_03]], align 2
+; CHECK-NEXT:    [[CONV2:%.*]] = sext i16 [[TMP11]] to i32
 ; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[CONV2]], [[CONV]]
 ; CHECK-NEXT:    [[SHR:%.*]] = ashr i32 [[MUL]], 15
 ; CHECK-NEXT:    [[SPEC_SELECT_I:%.*]] = tail call i32 @llvm.smin.i32(i32 [[SHR]], i32 32767)

From 14d57e21e1cc76de554314015e11a9d6e9b797f5 Mon Sep 17 00:00:00 2001
From: Kadir Cetinkaya <kadircet@google.com>
Date: Fri, 16 Aug 2024 16:16:05 +0200
Subject: [PATCH 173/441] [include-cleaner] fix 32-bit buildbots after
 a426ffdee1ca7814f2684b6

---
 clang-tools-extra/include-cleaner/unittests/WalkASTTest.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang-tools-extra/include-cleaner/unittests/WalkASTTest.cpp b/clang-tools-extra/include-cleaner/unittests/WalkASTTest.cpp
index 79371f5978fc334..b0a4473d4ad2b79 100644
--- a/clang-tools-extra/include-cleaner/unittests/WalkASTTest.cpp
+++ b/clang-tools-extra/include-cleaner/unittests/WalkASTTest.cpp
@@ -557,7 +557,7 @@ TEST(WalkAST, FriendDecl) {
 }
 
 TEST(WalkAST, OperatorNewDelete) {
-  testWalk("void* $explicit^operator new(unsigned long, void*);",
+  testWalk("void* $explicit^operator new(decltype(sizeof(int)), void*);",
            "struct Bar { void foo() { Bar b; ^new (&b) Bar; } };");
   testWalk("struct A { static void $explicit^operator delete(void*); };",
            "void foo() { A a; ^delete &a; }");

From 3c3fc4ce77da21214e1e20b8c42607b4dcdef5a7 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Fri, 16 Aug 2024 16:28:51 +0200
Subject: [PATCH 174/441] [InstCombine] Add nsw tests for A + -B fold (NFC)

---
 llvm/test/Transforms/InstCombine/add.ll | 42 +++++++++++++++++++++++--
 1 file changed, 40 insertions(+), 2 deletions(-)

diff --git a/llvm/test/Transforms/InstCombine/add.ll b/llvm/test/Transforms/InstCombine/add.ll
index adb61cd4b9692fb..c61a9006e020d30 100644
--- a/llvm/test/Transforms/InstCombine/add.ll
+++ b/llvm/test/Transforms/InstCombine/add.ll
@@ -99,16 +99,54 @@ define i32 @test3(i32 %A) {
 }
 
 ; D = B + -A = B - A
-define i32 @test4(i32 %A, i32 %B) {
+define i32 @test4(i32 %A, i32 %BB) {
 ; CHECK-LABEL: @test4(
-; CHECK-NEXT:    [[D:%.*]] = sub i32 [[B:%.*]], [[A:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = xor i32 [[BB:%.*]], 1
+; CHECK-NEXT:    [[D:%.*]] = sub i32 [[B]], [[A:%.*]]
 ; CHECK-NEXT:    ret i32 [[D]]
 ;
+  %B = xor i32 %BB, 1 ; thwart complexity-based canonicalization
   %C = sub i32 0, %A
   %D = add i32 %B, %C
   ret i32 %D
 }
 
+define i32 @test4_both_nsw(i32 %A, i32 %BB) {
+; CHECK-LABEL: @test4_both_nsw(
+; CHECK-NEXT:    [[B:%.*]] = xor i32 [[BB:%.*]], 1
+; CHECK-NEXT:    [[D:%.*]] = sub i32 [[B]], [[A:%.*]]
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %B = xor i32 %BB, 1 ; thwart complexity-based canonicalization
+  %C = sub nsw i32 0, %A
+  %D = add nsw i32 %B, %C
+  ret i32 %D
+}
+
+define i32 @test4_neg_nsw(i32 %A, i32 %BB) {
+; CHECK-LABEL: @test4_neg_nsw(
+; CHECK-NEXT:    [[B:%.*]] = xor i32 [[BB:%.*]], 1
+; CHECK-NEXT:    [[D:%.*]] = sub i32 [[B]], [[A:%.*]]
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %B = xor i32 %BB, 1 ; thwart complexity-based canonicalization
+  %C = sub nsw i32 0, %A
+  %D = add i32 %B, %C
+  ret i32 %D
+}
+
+define i32 @test4_add_nsw(i32 %A, i32 %BB) {
+; CHECK-LABEL: @test4_add_nsw(
+; CHECK-NEXT:    [[B:%.*]] = xor i32 [[BB:%.*]], 1
+; CHECK-NEXT:    [[D:%.*]] = sub i32 [[B]], [[A:%.*]]
+; CHECK-NEXT:    ret i32 [[D]]
+;
+  %B = xor i32 %BB, 1 ; thwart complexity-based canonicalization
+  %C = sub i32 0, %A
+  %D = add nsw i32 %B, %C
+  ret i32 %D
+}
+
 ; D = -A + B = B - A
 define i32 @test5(i32 %A, i32 %B) {
 ; CHECK-LABEL: @test5(

From dd9a99f2b634d95072ae49ebcbe5598877de4985 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Fri, 16 Aug 2024 16:31:12 +0200
Subject: [PATCH 175/441] [InstCombine] Preserve nsw in A + -B fold

This was already done for -B + A, but not for A + -B.

Proof: https://alive2.llvm.org/ce/z/F3V2yZ
---
 llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp | 8 ++++++--
 llvm/test/Transforms/InstCombine/add.ll               | 2 +-
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
index 3bd086230cbec5a..123f810bacfb6e4 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
@@ -1548,8 +1548,12 @@ Instruction *InstCombinerImpl::visitAdd(BinaryOperator &I) {
   }
 
   // A + -B  -->  A - B
-  if (match(RHS, m_Neg(m_Value(B))))
-    return BinaryOperator::CreateSub(LHS, B);
+  if (match(RHS, m_Neg(m_Value(B)))) {
+    auto *Sub = BinaryOperator::CreateSub(LHS, B);
+    auto *OBO = cast<OverflowingBinaryOperator>(RHS);
+    Sub->setHasNoSignedWrap(I.hasNoSignedWrap() && OBO->hasNoSignedWrap());
+    return Sub;
+  }
 
   if (Value *V = checkForNegativeOperand(I, Builder))
     return replaceInstUsesWith(I, V);
diff --git a/llvm/test/Transforms/InstCombine/add.ll b/llvm/test/Transforms/InstCombine/add.ll
index c61a9006e020d30..b1f21e58de1e2ad 100644
--- a/llvm/test/Transforms/InstCombine/add.ll
+++ b/llvm/test/Transforms/InstCombine/add.ll
@@ -114,7 +114,7 @@ define i32 @test4(i32 %A, i32 %BB) {
 define i32 @test4_both_nsw(i32 %A, i32 %BB) {
 ; CHECK-LABEL: @test4_both_nsw(
 ; CHECK-NEXT:    [[B:%.*]] = xor i32 [[BB:%.*]], 1
-; CHECK-NEXT:    [[D:%.*]] = sub i32 [[B]], [[A:%.*]]
+; CHECK-NEXT:    [[D:%.*]] = sub nsw i32 [[B]], [[A:%.*]]
 ; CHECK-NEXT:    ret i32 [[D]]
 ;
   %B = xor i32 %BB, 1 ; thwart complexity-based canonicalization

From 83bf1d661213cb8ec8e79ac085f793009721aaf5 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Fri, 16 Aug 2024 16:42:58 +0200
Subject: [PATCH 176/441] [InstCombine] Thwart complexity-based
 canonicalization in sqrt test (NFC)

---
 llvm/test/Transforms/InstCombine/fast-math.ll | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/llvm/test/Transforms/InstCombine/fast-math.ll b/llvm/test/Transforms/InstCombine/fast-math.ll
index da403555ebe2489..7f83328d360d66c 100644
--- a/llvm/test/Transforms/InstCombine/fast-math.ll
+++ b/llvm/test/Transforms/InstCombine/fast-math.ll
@@ -709,13 +709,15 @@ define double @sqrt_intrinsic_three_args5(double %x, double %y) {
   ret double %sqrt
 }
 
-define double @sqrt_intrinsic_three_args6(double %x, double %y) {
+define double @sqrt_intrinsic_three_args6(double %x, ptr %yp) {
 ; CHECK-LABEL: @sqrt_intrinsic_three_args6(
-; CHECK-NEXT:    [[FABS:%.*]] = call fast double @llvm.fabs.f64(double [[X:%.*]])
-; CHECK-NEXT:    [[SQRT1:%.*]] = call fast double @llvm.sqrt.f64(double [[Y:%.*]])
-; CHECK-NEXT:    [[SQRT:%.*]] = fmul fast double [[FABS]], [[SQRT1]]
+; CHECK-NEXT:    [[Y:%.*]] = load double, ptr [[YP:%.*]], align 8
+; CHECK-NEXT:    [[MUL:%.*]] = fmul fast double [[X:%.*]], [[X]]
+; CHECK-NEXT:    [[MUL2:%.*]] = fmul fast double [[Y]], [[MUL]]
+; CHECK-NEXT:    [[SQRT:%.*]] = call fast double @llvm.sqrt.f64(double [[MUL2]])
 ; CHECK-NEXT:    ret double [[SQRT]]
 ;
+  %y = load double, ptr %yp ; thwart complexity-based canonicalization
   %mul = fmul fast double %x, %x
   %mul2 = fmul fast double %y, %mul
   %sqrt = call fast double @llvm.sqrt.f64(double %mul2)

From 60bffe221a1d615ffc7c6b632287d0fbd27ef863 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Fri, 16 Aug 2024 16:47:13 +0200
Subject: [PATCH 177/441] [InstCombine] Handle commuted variant of sqrt
 transform

---
 .../lib/Transforms/Utils/SimplifyLibCalls.cpp | 19 +++++++++++--------
 llvm/test/Transforms/InstCombine/fast-math.ll |  6 +++---
 2 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp b/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
index be4d4590cfd839d..fb2efe581ac6bb2 100644
--- a/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
@@ -2758,14 +2758,17 @@ Value *LibCallSimplifier::optimizeSqrt(CallInst *CI, IRBuilderBase &B) {
     // Note: We don't bother looking any deeper than this first level or for
     // variations of this pattern because instcombine's visitFMUL and/or the
     // reassociation pass should give us this form.
-    Value *OtherMul0, *OtherMul1;
-    if (match(Op0, m_FMul(m_Value(OtherMul0), m_Value(OtherMul1)))) {
-      // Pattern: sqrt((x * y) * z)
-      if (OtherMul0 == OtherMul1 && cast<Instruction>(Op0)->isFast()) {
-        // Matched: sqrt((x * x) * z)
-        RepeatOp = OtherMul0;
-        OtherOp = Op1;
-      }
+    Value *MulOp;
+    if (match(Op0, m_FMul(m_Value(MulOp), m_Deferred(MulOp))) &&
+        cast<Instruction>(Op0)->isFast()) {
+      // Pattern: sqrt((x * x) * z)
+      RepeatOp = MulOp;
+      OtherOp = Op1;
+    } else if (match(Op1, m_FMul(m_Value(MulOp), m_Deferred(MulOp))) &&
+               cast<Instruction>(Op1)->isFast()) {
+      // Pattern: sqrt(z * (x * x))
+      RepeatOp = MulOp;
+      OtherOp = Op0;
     }
   }
   if (!RepeatOp)
diff --git a/llvm/test/Transforms/InstCombine/fast-math.ll b/llvm/test/Transforms/InstCombine/fast-math.ll
index 7f83328d360d66c..d7c90e82ab520a8 100644
--- a/llvm/test/Transforms/InstCombine/fast-math.ll
+++ b/llvm/test/Transforms/InstCombine/fast-math.ll
@@ -712,9 +712,9 @@ define double @sqrt_intrinsic_three_args5(double %x, double %y) {
 define double @sqrt_intrinsic_three_args6(double %x, ptr %yp) {
 ; CHECK-LABEL: @sqrt_intrinsic_three_args6(
 ; CHECK-NEXT:    [[Y:%.*]] = load double, ptr [[YP:%.*]], align 8
-; CHECK-NEXT:    [[MUL:%.*]] = fmul fast double [[X:%.*]], [[X]]
-; CHECK-NEXT:    [[MUL2:%.*]] = fmul fast double [[Y]], [[MUL]]
-; CHECK-NEXT:    [[SQRT:%.*]] = call fast double @llvm.sqrt.f64(double [[MUL2]])
+; CHECK-NEXT:    [[FABS:%.*]] = call fast double @llvm.fabs.f64(double [[X:%.*]])
+; CHECK-NEXT:    [[SQRT1:%.*]] = call fast double @llvm.sqrt.f64(double [[Y]])
+; CHECK-NEXT:    [[SQRT:%.*]] = fmul fast double [[FABS]], [[SQRT1]]
 ; CHECK-NEXT:    ret double [[SQRT]]
 ;
   %y = load double, ptr %yp ; thwart complexity-based canonicalization

From 99696b35bc8a0054e0b0c1a26e8dd5049fa8c41b Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2@gmail.com>
Date: Fri, 16 Aug 2024 11:08:34 -0400
Subject: [PATCH 178/441] [libc++] Fix rejects-valid in std::span copy
 construction (#104500)

Trying to copy-construct a std::span from another std::span holding an
incomplete type would fail as we evaluate the SFINAE for the range-based
constructor. The problem was that we checked for __is_std_span after
checking for the range being a contiguous_range, which hard-errored
because of arithmetic on a pointer to incomplete type.

As a drive-by, refactor the whole test and format it.

Fixes #104496
---
 libcxx/include/span                           |   2 +-
 .../views/views.span/span.cons/copy.pass.cpp  | 126 ++++++++++++------
 2 files changed, 86 insertions(+), 42 deletions(-)

diff --git a/libcxx/include/span b/libcxx/include/span
index 60d76d830f0f318..da631cdc3f90e6b 100644
--- a/libcxx/include/span
+++ b/libcxx/include/span
@@ -206,10 +206,10 @@ struct __is_std_span<span<_Tp, _Sz>> : true_type {};
 
 template <class _Range, class _ElementType>
 concept __span_compatible_range =
+    !__is_std_span<remove_cvref_t<_Range>>::value &&                //
     ranges::contiguous_range<_Range> &&                             //
     ranges::sized_range<_Range> &&                                  //
     (ranges::borrowed_range<_Range> || is_const_v<_ElementType>) && //
-    !__is_std_span<remove_cvref_t<_Range>>::value &&                //
     !__is_std_array<remove_cvref_t<_Range>>::value &&               //
     !is_array_v<remove_cvref_t<_Range>> &&                          //
     is_convertible_v<remove_reference_t<ranges::range_reference_t<_Range>> (*)[], _ElementType (*)[]>;
diff --git a/libcxx/test/std/containers/views/views.span/span.cons/copy.pass.cpp b/libcxx/test/std/containers/views/views.span/span.cons/copy.pass.cpp
index 28f13e122ddc5e1..d3990fd60a459aa 100644
--- a/libcxx/test/std/containers/views/views.span/span.cons/copy.pass.cpp
+++ b/libcxx/test/std/containers/views/views.span/span.cons/copy.pass.cpp
@@ -5,6 +5,7 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
+
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 
 // <span>
@@ -14,58 +15,101 @@
 #include <span>
 #include <cassert>
 #include <string>
+#include <utility>
 
 #include "test_macros.h"
 
-template <typename T>
-constexpr bool doCopy(const T &rhs)
-{
-    ASSERT_NOEXCEPT(T{rhs});
-    T lhs{rhs};
-    return lhs.data() == rhs.data()
-     &&    lhs.size() == rhs.size();
-}
+template <class T>
+constexpr void test() {
+  ASSERT_NOEXCEPT(std::span<T>(std::declval<std::span<T> const&>()));
+  ASSERT_NOEXCEPT(std::span<T>{std::declval<std::span<T> const&>()});
 
-struct A{};
-
-template <typename T>
-void testCV ()
-{
-    int  arr[] = {1,2,3};
-    assert((doCopy(std::span<T>  ()          )));
-    assert((doCopy(std::span<T,0>()          )));
-    assert((doCopy(std::span<T>  (&arr[0], 1))));
-    assert((doCopy(std::span<T,1>(&arr[0], 1))));
-    assert((doCopy(std::span<T>  (&arr[0], 2))));
-    assert((doCopy(std::span<T,2>(&arr[0], 2))));
+  // dynamic_extent
+  {
+    std::span<T> x;
+    std::span<T> copy(x);
+    assert(copy.data() == x.data());
+    assert(copy.size() == x.size());
+  }
+  {
+    T array[3] = {};
+    std::span<T> x(array, 3);
+    std::span<T> copy(x);
+    assert(copy.data() == array);
+    assert(copy.size() == 3);
+  }
+  {
+    T array[3] = {};
+    std::span<T> x(array, 2);
+    std::span<T> copy(x);
+    assert(copy.data() == array);
+    assert(copy.size() == 2);
+  }
+
+  // static extent
+  {
+    std::span<T, 0> x;
+    std::span<T, 0> copy(x);
+    assert(copy.data() == x.data());
+    assert(copy.size() == x.size());
+  }
+  {
+    T array[3] = {};
+    std::span<T, 3> x(array);
+    std::span<T, 3> copy(x);
+    assert(copy.data() == array);
+    assert(copy.size() == 3);
+  }
+  {
+    T array[2] = {};
+    std::span<T, 2> x(array);
+    std::span<T, 2> copy(x);
+    assert(copy.data() == array);
+    assert(copy.size() == 2);
+  }
 }
 
+struct Foo {};
+
+constexpr bool test_all() {
+  test<int>();
+  test<const int>();
+  test<volatile int>();
+  test<const volatile int>();
 
-int main(int, char**)
-{
-    constexpr int carr[] = {1,2,3};
+  test<long>();
+  test<const long>();
+  test<volatile long>();
+  test<const volatile long>();
 
-    static_assert(doCopy(std::span<      int>  ()),            "");
-    static_assert(doCopy(std::span<      int,0>()),            "");
-    static_assert(doCopy(std::span<const int>  (&carr[0], 1)), "");
-    static_assert(doCopy(std::span<const int,1>(&carr[0], 1)), "");
-    static_assert(doCopy(std::span<const int>  (&carr[0], 2)), "");
-    static_assert(doCopy(std::span<const int,2>(&carr[0], 2)), "");
+  test<double>();
+  test<const double>();
+  test<volatile double>();
+  test<const volatile double>();
 
-    static_assert(doCopy(std::span<long>()),   "");
-    static_assert(doCopy(std::span<double>()), "");
-    static_assert(doCopy(std::span<A>()),      "");
+  // Note: Can't test non-fundamental types with volatile because we require `T*` to be indirectly_readable,
+  //       which isn't the case when T is volatile.
+  test<Foo>();
+  test<const Foo>();
 
-    std::string s;
-    assert(doCopy(std::span<std::string>   ()     ));
-    assert(doCopy(std::span<std::string, 0>()     ));
-    assert(doCopy(std::span<std::string>   (&s, 1)));
-    assert(doCopy(std::span<std::string, 1>(&s, 1)));
+  test<std::string>();
+  test<const std::string>();
+
+  // Regression test for https://github.com/llvm/llvm-project/issues/104496
+  {
+    struct Incomplete;
+    std::span<Incomplete> x;
+    std::span<Incomplete> copy(x);
+    assert(copy.data() == x.data());
+    assert(copy.size() == x.size());
+  }
+
+  return true;
+}
 
-    testCV<               int>();
-    testCV<const          int>();
-    testCV<      volatile int>();
-    testCV<const volatile int>();
+int main(int, char**) {
+  test_all();
+  static_assert(test_all());
 
   return 0;
 }

From ef6e7affbb7b0eb4976c1019c788bcadfc34ecd6 Mon Sep 17 00:00:00 2001
From: Rafael Ubal <rubal@mathworks.com>
Date: Fri, 16 Aug 2024 11:10:48 -0400
Subject: [PATCH 179/441] [mlir] [tosa] Bug fixes in shape inference pass
 (#104146)

This change addresses 2 bugs in the TOSA shape inference pass
(`--tosa-infer-shapes`). The included unit test contains a detailed
description of the issues.

- Input IR

```
func.func @main(%arg0: tensor<1x2x8xf32>) {
  %0 = tosa.cast %arg0 : (tensor<1x2x8xf32>) -> tensor<?x2x8xf32>

  %c0 = arith.constant 0 : index
  %dim = tensor.dim %0, %c0 : tensor<?x2x8xf32>

  %expanded_0 = tensor.expand_shape %0 [[0], [1, 2], [3]] output_shape [%dim, 1, 4, 8] : tensor<?x2x8xf32> into tensor<?x1x2x8xf32>
  %expanded_1 = tensor.expand_shape %0 [[0], [1, 2], [3]] output_shape [%dim, 1, 4, 8] : tensor<?x2x8xf32> into tensor<?x1x2x8xf32>
  return
}
```

- Output IR

```
module {
  func.func @main(%arg0: tensor<1x2x8xf32>) {
  %0 = tosa.cast %arg0 : (tensor<1x2x8xf32>) -> tensor<1x2x8xf32>

  // This cast was previously inserted between both 'tensor.expand_shape' ops.
  %cast = tensor.cast %0 : tensor<1x2x8xf32> to tensor<?x2x8xf32>

  %c0 = arith.constant 0 : index
  %dim = tensor.dim %0, %c0 : tensor<1x2x8xf32>

  // The operand of the first 'tensor.expand_shape' op was not previously updated
  // from '%0' to '%cast' due to an invalidation of the iterator traversing the
  // use list of the 'tosa.cast' op.
  %expanded_0 = tensor.expand_shape %cast [[0], [1, 2], [3]] output_shape [%dim, 1, 4, 8] : tensor<?x2x8xf32> into tensor<?x1x2x8xf32>
  %expanded_1 = tensor.expand_shape %cast [[0], [1, 2], [3]] output_shape [%dim, 1, 4, 8] : tensor<?x2x8xf32> into tensor<?x1x2x8xf32>
  return
}
```
---
 .../Tosa/Transforms/TosaInferShapes.cpp       | 34 ++++++++---
 mlir/test/Dialect/Tosa/tosa-infer-shapes.mlir | 56 ++++++++++++++++++-
 2 files changed, 81 insertions(+), 9 deletions(-)

diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaInferShapes.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaInferShapes.cpp
index b1d5720541846f3..358353eb955c16c 100644
--- a/mlir/lib/Dialect/Tosa/Transforms/TosaInferShapes.cpp
+++ b/mlir/lib/Dialect/Tosa/Transforms/TosaInferShapes.cpp
@@ -88,18 +88,36 @@ class TypeModificationState {
     // For each use whose type changed, cast the value with the new type back to
     // the old type.
     for (auto [value, oldType] : oldTypes) {
-      tensor::CastOp castedValue;
-      for (auto &use : value.getUses()) {
-        if (canBeRefined(use.getOwner()))
+      // The call to 'use->set()' in the body of the loop below invalidates the
+      // iterator used to traverse op uses, so it is important to make a copy of
+      // these first.
+      llvm::SmallVector<OpOperand *> uses = llvm::map_to_vector(
+          value.getUses(),
+          [](OpOperand &use) -> OpOperand * {
+            return &use;
+          });
+
+      // A 'tensor.cast' op is emitted only if needed. Once emitted, it is
+      // cached and reused by all consumers.
+      tensor::CastOp castValue;
+
+      // Traverse all uses
+      for (OpOperand *use : uses) {
+        if (canBeRefined(use->getOwner()))
           continue;
 
-        // Cache the cast to avoid generating duplicates
-        if (!castedValue) {
-          ImplicitLocOpBuilder builder{value.getLoc(), use.getOwner()};
-          castedValue = builder.create<tensor::CastOp>(oldType, value);
+        if (!castValue) {
+          // Set the insertion point as far back as possible, since new
+          // consumers of the 'tensor.cast' op generated in future iterations
+          // are likely to be further up in the code due to the order in which
+          // they appear in the use list.
+          OpBuilder builder{value.getContext()};
+          builder.setInsertionPointAfter(value.getDefiningOp());
+          castValue =
+              builder.create<tensor::CastOp>(value.getLoc(), oldType, value);
         }
 
-        use.set(castedValue);
+        use->set(castValue);
       }
     }
 
diff --git a/mlir/test/Dialect/Tosa/tosa-infer-shapes.mlir b/mlir/test/Dialect/Tosa/tosa-infer-shapes.mlir
index 3224f88968f3d27..d46de740800e937 100644
--- a/mlir/test/Dialect/Tosa/tosa-infer-shapes.mlir
+++ b/mlir/test/Dialect/Tosa/tosa-infer-shapes.mlir
@@ -1373,4 +1373,58 @@ func.func @test_tosa_use_def_chain(%arg0: tensor<1x32x32x3xf32>, %arg1: tensor<1
   // CHECK: (tensor<1x32x32x16xf32>) -> tensor<1x16x16x16xf32>
   %1 = tosa.max_pool2d %0 {kernel = array<i64: 2, 2>, pad = array<i64: 0, 0, 0, 0>, stride = array<i64: 2, 2>} : (tensor<?x32x32x16xf32>) -> tensor<?x16x16x16xf32>
   return %1 : tensor<?x16x16x16xf32>
-}
\ No newline at end of file
+}
+
+// -----
+
+// This test locks two bug fixes manifested in the code below.
+//
+// 1. Context
+//
+// When shape propagation hits an operation that does not support shape
+// inference (here 'tensor.expand_shape'), it must revert the currently
+// inferred shape of its consumers back to the originally expected input
+// type to avoid potential op verification errors. This type reversal is
+// done through an additional 'tensor.cast' op.
+//
+//
+// 2. Preserving list of non-inferrable consumers
+//
+// When multiple non-inferrable consumers of a shape-inferred value are found
+// (here, the 2 occurrences of 'tensor.expand_shape' consuming the output of
+// 'tosa.cast'), their input argument ('%0') must be altered to consume the
+// output the new 'tensor.cast' op. While these replacements occur, the use list
+// of the producer ('tosa.cast') is also implicitly altered, invalidating any
+// iterators associated with it. It is therefore necessary to create a copy of
+// this use list ahead of time. Before this bug fix, the second
+// 'tensor.expand_shape' op below was not updated correctly.
+//
+// 3. Guaranteeing def-use order
+//
+// When emitting the 'tensor.cast' op, it is important to guarantee that its
+// output value is defined before all of its consumers (here, both of the
+// 'tensor.expand_shape' ops. In a previous version of the code, this insertion
+// occurred right before the first encountered consumer. Since use lists are
+// saved in reverse order, the 'tensor.cast' op was inserted before the second
+// 'tensor.expand_shape' op, leading to a def-use order violation when the
+// first 'tensor.expand_shape' op was later updated. The current implementation
+// sets the insertion point right after the producer of the last shape-inferred
+// value (here 'tosa.cast'), which guarantees correct def-use order for all
+// future operand updates.
+
+// CHECK-LABEL: test_multiple_non_inferrable_consumers
+// CHECK-SAME: %[[ARG:.*]]: tensor<1x2x8xf32>
+func.func @test_multiple_non_inferrable_consumers(%arg0: tensor<1x2x8xf32>) {
+  // CHECK: %[[TOSA_CAST:.*]] = tosa.cast %[[ARG]] : (tensor<1x2x8xf32>) -> tensor<1x2x8xf32>
+  // CHECK: %[[TENSOR_CAST:.*]] = tensor.cast %[[TOSA_CAST]] : tensor<1x2x8xf32> to tensor<?x2x8xf32>
+  %0 = tosa.cast %arg0 : (tensor<1x2x8xf32>) -> tensor<?x2x8xf32>
+
+  %c0 = arith.constant 0 : index
+  %dim = tensor.dim %0, %c0 : tensor<?x2x8xf32>
+
+  // CHECK: tensor.expand_shape %[[TENSOR_CAST]]
+  // CHECK: tensor.expand_shape %[[TENSOR_CAST]]
+  %expanded_0 = tensor.expand_shape %0 [[0], [1, 2], [3]] output_shape [%dim, 1, 4, 8] : tensor<?x2x8xf32> into tensor<?x1x2x8xf32>
+  %expanded_1 = tensor.expand_shape %0 [[0], [1, 2], [3]] output_shape [%dim, 1, 4, 8] : tensor<?x2x8xf32> into tensor<?x1x2x8xf32>
+  return
+}

From a07aba5d44204a7ca0d891a3da05af9960081e4c Mon Sep 17 00:00:00 2001
From: Timm Baeder <tbaeder@redhat.com>
Date: Fri, 16 Aug 2024 17:13:12 +0200
Subject: [PATCH 180/441] [clang] Rename all AST/Interp stuff to AST/ByteCode
 (#104552)

"Interp" clashes with the clang interpreter and people often confuse
this.
---
 clang/docs/ClangFormattedStatus.rst           |  2 +-
 clang/docs/tools/clang-formatted-files.txt    | 36 ++++++------
 clang/lib/AST/ASTContext.cpp                  |  2 +-
 clang/lib/AST/{Interp => ByteCode}/Boolean.h  | 15 +++--
 .../{Interp => ByteCode}/ByteCodeEmitter.cpp  |  9 ++-
 .../{Interp => ByteCode}/ByteCodeEmitter.h    |  2 +-
 .../lib/AST/{Interp => ByteCode}/Compiler.cpp |  5 +-
 clang/lib/AST/{Interp => ByteCode}/Compiler.h |  0
 .../lib/AST/{Interp => ByteCode}/Context.cpp  |  0
 clang/lib/AST/{Interp => ByteCode}/Context.h  |  0
 .../AST/{Interp => ByteCode}/Descriptor.cpp   |  0
 .../lib/AST/{Interp => ByteCode}/Descriptor.h |  2 +-
 clang/lib/AST/{Interp => ByteCode}/Disasm.cpp |  0
 .../{Interp => ByteCode}/DynamicAllocator.cpp |  0
 .../{Interp => ByteCode}/DynamicAllocator.h   |  0
 .../AST/{Interp => ByteCode}/EvalEmitter.cpp  |  4 +-
 .../AST/{Interp => ByteCode}/EvalEmitter.h    |  0
 .../{Interp => ByteCode}/EvaluationResult.cpp |  0
 .../{Interp => ByteCode}/EvaluationResult.h   |  0
 .../lib/AST/{Interp => ByteCode}/Floating.cpp |  0
 clang/lib/AST/{Interp => ByteCode}/Floating.h |  0
 clang/lib/AST/{Interp => ByteCode}/Frame.cpp  |  0
 clang/lib/AST/{Interp => ByteCode}/Frame.h    |  0
 .../lib/AST/{Interp => ByteCode}/Function.cpp |  0
 clang/lib/AST/{Interp => ByteCode}/Function.h |  0
 .../{Interp => ByteCode}/FunctionPointer.h    |  0
 clang/lib/AST/{Interp => ByteCode}/Integral.h | 42 +++++++++-----
 .../lib/AST/{Interp => ByteCode}/IntegralAP.h |  0
 clang/lib/AST/{Interp => ByteCode}/Interp.cpp |  0
 clang/lib/AST/{Interp => ByteCode}/Interp.h   | 15 ++---
 .../AST/{Interp => ByteCode}/InterpBlock.cpp  |  0
 .../AST/{Interp => ByteCode}/InterpBlock.h    |  2 +-
 .../{Interp => ByteCode}/InterpBuiltin.cpp    |  0
 .../AST/{Interp => ByteCode}/InterpFrame.cpp  |  2 +-
 .../AST/{Interp => ByteCode}/InterpFrame.h    |  2 +-
 .../AST/{Interp => ByteCode}/InterpShared.cpp |  0
 .../AST/{Interp => ByteCode}/InterpShared.h   |  0
 .../AST/{Interp => ByteCode}/InterpStack.cpp  |  4 +-
 .../AST/{Interp => ByteCode}/InterpStack.h    |  5 +-
 .../AST/{Interp => ByteCode}/InterpState.cpp  |  0
 .../AST/{Interp => ByteCode}/InterpState.h    |  0
 .../{Interp => ByteCode}/MemberPointer.cpp    |  0
 .../AST/{Interp => ByteCode}/MemberPointer.h  |  0
 clang/lib/AST/{Interp => ByteCode}/Opcode.h   |  0
 clang/lib/AST/{Interp => ByteCode}/Opcodes.td |  0
 .../lib/AST/{Interp => ByteCode}/Pointer.cpp  |  0
 clang/lib/AST/{Interp => ByteCode}/Pointer.h  |  2 +-
 .../lib/AST/{Interp => ByteCode}/PrimType.cpp |  0
 clang/lib/AST/{Interp => ByteCode}/PrimType.h | 57 ++++++++++++++-----
 .../lib/AST/{Interp => ByteCode}/Primitives.h |  0
 .../lib/AST/{Interp => ByteCode}/Program.cpp  | 43 +++++++-------
 clang/lib/AST/{Interp => ByteCode}/Program.h  | 16 ++----
 clang/lib/AST/{Interp => ByteCode}/Record.cpp |  2 +-
 clang/lib/AST/{Interp => ByteCode}/Record.h   |  0
 clang/lib/AST/{Interp => ByteCode}/Source.cpp |  0
 clang/lib/AST/{Interp => ByteCode}/Source.h   |  0
 clang/lib/AST/{Interp => ByteCode}/State.cpp  |  0
 clang/lib/AST/{Interp => ByteCode}/State.h    |  0
 clang/lib/AST/CMakeLists.txt                  | 52 ++++++++---------
 clang/lib/AST/ExprConstShared.h               |  2 +-
 clang/lib/AST/ExprConstant.cpp                |  6 +-
 .../test/AST/{Interp => ByteCode}/arrays.cpp  |  0
 clang/test/AST/{Interp => ByteCode}/atomic.c  |  0
 .../test/AST/{Interp => ByteCode}/atomic.cpp  |  0
 .../AST/{Interp => ByteCode}/bitfields.cpp    |  0
 .../builtin-align-cxx.cpp                     |  0
 .../builtin-constant-p.cpp                    |  0
 .../builtin-functions.cpp                     |  0
 .../AST/{Interp => ByteCode}/builtins.cpp     |  0
 clang/test/AST/{Interp => ByteCode}/c.c       |  0
 clang/test/AST/{Interp => ByteCode}/c23.c     |  0
 .../test/AST/{Interp => ByteCode}/codegen.cpp |  0
 clang/test/AST/{Interp => ByteCode}/comma.cpp |  0
 clang/test/AST/{Interp => ByteCode}/complex.c |  0
 .../test/AST/{Interp => ByteCode}/complex.cpp |  0
 clang/test/AST/{Interp => ByteCode}/cond.cpp  |  0
 .../AST/{Interp => ByteCode}/const-eval.c     |  0
 .../{Interp => ByteCode}/const-fpfeatures.cpp |  0
 .../const-temporaries.cpp                     |  0
 .../constexpr-frame-describe.cpp              |  0
 .../constexpr-nqueens.cpp                     |  0
 .../constexpr-subobj-initialization.cpp       |  0
 .../{Interp => ByteCode}/crash-GH49103-2.cpp  |  0
 clang/test/AST/{Interp => ByteCode}/cxx03.cpp |  0
 clang/test/AST/{Interp => ByteCode}/cxx11.cpp |  0
 clang/test/AST/{Interp => ByteCode}/cxx17.cpp |  0
 clang/test/AST/{Interp => ByteCode}/cxx20.cpp |  0
 clang/test/AST/{Interp => ByteCode}/cxx23.cpp |  0
 clang/test/AST/{Interp => ByteCode}/cxx26.cpp |  0
 clang/test/AST/{Interp => ByteCode}/cxx2a.cpp |  0
 clang/test/AST/{Interp => ByteCode}/cxx98.cpp |  0
 .../AST/{Interp => ByteCode}/depth-limit.cpp  |  0
 .../AST/{Interp => ByteCode}/depth-limit2.cpp |  0
 .../{Interp => ByteCode}/enums-targets.cpp    |  0
 clang/test/AST/{Interp => ByteCode}/enums.cpp |  0
 .../AST/{Interp => ByteCode}/eval-order.cpp   |  0
 .../test/AST/{Interp => ByteCode}/floats.cpp  |  0
 .../AST/{Interp => ByteCode}/functions.cpp    |  0
 clang/test/AST/{Interp => ByteCode}/hlsl.hlsl |  0
 clang/test/AST/{Interp => ByteCode}/if.cpp    |  0
 clang/test/AST/{Interp => ByteCode}/intap.cpp |  0
 .../test/AST/{Interp => ByteCode}/invalid.cpp |  0
 .../test/AST/{Interp => ByteCode}/lambda.cpp  |  0
 .../AST/{Interp => ByteCode}/lifetimes.cpp    |  0
 .../AST/{Interp => ByteCode}/literals.cpp     |  0
 clang/test/AST/{Interp => ByteCode}/loops.cpp |  0
 .../{Interp => ByteCode}/memberpointers.cpp   |  0
 clang/test/AST/{Interp => ByteCode}/ms.cpp    |  0
 .../test/AST/{Interp => ByteCode}/mutable.cpp |  0
 .../AST/{Interp => ByteCode}/new-delete.cpp   |  0
 .../AST/{Interp => ByteCode}/nullable.cpp     |  0
 clang/test/AST/{Interp => ByteCode}/objc.mm   |  0
 clang/test/AST/{Interp => ByteCode}/opencl.cl |  0
 .../{Interp => ByteCode}/pointer-addition.c   |  0
 .../test/AST/{Interp => ByteCode}/records.cpp |  0
 .../AST/{Interp => ByteCode}/references.cpp   |  0
 .../test/AST/{Interp => ByteCode}/shifts.cpp  |  0
 .../AST/{Interp => ByteCode}/spaceship.cpp    |  0
 .../test/AST/{Interp => ByteCode}/switch.cpp  |  0
 clang/test/AST/{Interp => ByteCode}/sycl.cpp  |  0
 .../test/AST/{Interp => ByteCode}/unions.cpp  |  0
 .../test/AST/{Interp => ByteCode}/vectors.cpp |  0
 clang/test/AST/{Interp => ByteCode}/weak.cpp  |  0
 .../AST/{Interp => ByteCode}/CMakeLists.txt   |  0
 .../AST/{Interp => ByteCode}/Descriptor.cpp   | 28 ++++-----
 .../AST/{Interp => ByteCode}/toAPValue.cpp    |  6 +-
 clang/unittests/AST/CMakeLists.txt            |  2 +-
 127 files changed, 198 insertions(+), 167 deletions(-)
 rename clang/lib/AST/{Interp => ByteCode}/Boolean.h (96%)
 rename clang/lib/AST/{Interp => ByteCode}/ByteCodeEmitter.cpp (97%)
 rename clang/lib/AST/{Interp => ByteCode}/ByteCodeEmitter.h (97%)
 rename clang/lib/AST/{Interp => ByteCode}/Compiler.cpp (99%)
 rename clang/lib/AST/{Interp => ByteCode}/Compiler.h (100%)
 rename clang/lib/AST/{Interp => ByteCode}/Context.cpp (100%)
 rename clang/lib/AST/{Interp => ByteCode}/Context.h (100%)
 rename clang/lib/AST/{Interp => ByteCode}/Descriptor.cpp (100%)
 rename clang/lib/AST/{Interp => ByteCode}/Descriptor.h (99%)
 rename clang/lib/AST/{Interp => ByteCode}/Disasm.cpp (100%)
 rename clang/lib/AST/{Interp => ByteCode}/DynamicAllocator.cpp (100%)
 rename clang/lib/AST/{Interp => ByteCode}/DynamicAllocator.h (100%)
 rename clang/lib/AST/{Interp => ByteCode}/EvalEmitter.cpp (99%)
 rename clang/lib/AST/{Interp => ByteCode}/EvalEmitter.h (100%)
 rename clang/lib/AST/{Interp => ByteCode}/EvaluationResult.cpp (100%)
 rename clang/lib/AST/{Interp => ByteCode}/EvaluationResult.h (100%)
 rename clang/lib/AST/{Interp => ByteCode}/Floating.cpp (100%)
 rename clang/lib/AST/{Interp => ByteCode}/Floating.h (100%)
 rename clang/lib/AST/{Interp => ByteCode}/Frame.cpp (100%)
 rename clang/lib/AST/{Interp => ByteCode}/Frame.h (100%)
 rename clang/lib/AST/{Interp => ByteCode}/Function.cpp (100%)
 rename clang/lib/AST/{Interp => ByteCode}/Function.h (100%)
 rename clang/lib/AST/{Interp => ByteCode}/FunctionPointer.h (100%)
 rename clang/lib/AST/{Interp => ByteCode}/Integral.h (93%)
 rename clang/lib/AST/{Interp => ByteCode}/IntegralAP.h (100%)
 rename clang/lib/AST/{Interp => ByteCode}/Interp.cpp (100%)
 rename clang/lib/AST/{Interp => ByteCode}/Interp.h (99%)
 rename clang/lib/AST/{Interp => ByteCode}/InterpBlock.cpp (100%)
 rename clang/lib/AST/{Interp => ByteCode}/InterpBlock.h (100%)
 rename clang/lib/AST/{Interp => ByteCode}/InterpBuiltin.cpp (100%)
 rename clang/lib/AST/{Interp => ByteCode}/InterpFrame.cpp (99%)
 rename clang/lib/AST/{Interp => ByteCode}/InterpFrame.h (99%)
 rename clang/lib/AST/{Interp => ByteCode}/InterpShared.cpp (100%)
 rename clang/lib/AST/{Interp => ByteCode}/InterpShared.h (100%)
 rename clang/lib/AST/{Interp => ByteCode}/InterpStack.cpp (98%)
 rename clang/lib/AST/{Interp => ByteCode}/InterpStack.h (97%)
 rename clang/lib/AST/{Interp => ByteCode}/InterpState.cpp (100%)
 rename clang/lib/AST/{Interp => ByteCode}/InterpState.h (100%)
 rename clang/lib/AST/{Interp => ByteCode}/MemberPointer.cpp (100%)
 rename clang/lib/AST/{Interp => ByteCode}/MemberPointer.h (100%)
 rename clang/lib/AST/{Interp => ByteCode}/Opcode.h (100%)
 rename clang/lib/AST/{Interp => ByteCode}/Opcodes.td (100%)
 rename clang/lib/AST/{Interp => ByteCode}/Pointer.cpp (100%)
 rename clang/lib/AST/{Interp => ByteCode}/Pointer.h (99%)
 rename clang/lib/AST/{Interp => ByteCode}/PrimType.cpp (100%)
 rename clang/lib/AST/{Interp => ByteCode}/PrimType.h (83%)
 rename clang/lib/AST/{Interp => ByteCode}/Primitives.h (100%)
 rename clang/lib/AST/{Interp => ByteCode}/Program.cpp (95%)
 rename clang/lib/AST/{Interp => ByteCode}/Program.h (96%)
 rename clang/lib/AST/{Interp => ByteCode}/Record.cpp (96%)
 rename clang/lib/AST/{Interp => ByteCode}/Record.h (100%)
 rename clang/lib/AST/{Interp => ByteCode}/Source.cpp (100%)
 rename clang/lib/AST/{Interp => ByteCode}/Source.h (100%)
 rename clang/lib/AST/{Interp => ByteCode}/State.cpp (100%)
 rename clang/lib/AST/{Interp => ByteCode}/State.h (100%)
 rename clang/test/AST/{Interp => ByteCode}/arrays.cpp (100%)
 rename clang/test/AST/{Interp => ByteCode}/atomic.c (100%)
 rename clang/test/AST/{Interp => ByteCode}/atomic.cpp (100%)
 rename clang/test/AST/{Interp => ByteCode}/bitfields.cpp (100%)
 rename clang/test/AST/{Interp => ByteCode}/builtin-align-cxx.cpp (100%)
 rename clang/test/AST/{Interp => ByteCode}/builtin-constant-p.cpp (100%)
 rename clang/test/AST/{Interp => ByteCode}/builtin-functions.cpp (100%)
 rename clang/test/AST/{Interp => ByteCode}/builtins.cpp (100%)
 rename clang/test/AST/{Interp => ByteCode}/c.c (100%)
 rename clang/test/AST/{Interp => ByteCode}/c23.c (100%)
 rename clang/test/AST/{Interp => ByteCode}/codegen.cpp (100%)
 rename clang/test/AST/{Interp => ByteCode}/comma.cpp (100%)
 rename clang/test/AST/{Interp => ByteCode}/complex.c (100%)
 rename clang/test/AST/{Interp => ByteCode}/complex.cpp (100%)
 rename clang/test/AST/{Interp => ByteCode}/cond.cpp (100%)
 rename clang/test/AST/{Interp => ByteCode}/const-eval.c (100%)
 rename clang/test/AST/{Interp => ByteCode}/const-fpfeatures.cpp (100%)
 rename clang/test/AST/{Interp => ByteCode}/const-temporaries.cpp (100%)
 rename clang/test/AST/{Interp => ByteCode}/constexpr-frame-describe.cpp (100%)
 rename clang/test/AST/{Interp => ByteCode}/constexpr-nqueens.cpp (100%)
 rename clang/test/AST/{Interp => ByteCode}/constexpr-subobj-initialization.cpp (100%)
 rename clang/test/AST/{Interp => ByteCode}/crash-GH49103-2.cpp (100%)
 rename clang/test/AST/{Interp => ByteCode}/cxx03.cpp (100%)
 rename clang/test/AST/{Interp => ByteCode}/cxx11.cpp (100%)
 rename clang/test/AST/{Interp => ByteCode}/cxx17.cpp (100%)
 rename clang/test/AST/{Interp => ByteCode}/cxx20.cpp (100%)
 rename clang/test/AST/{Interp => ByteCode}/cxx23.cpp (100%)
 rename clang/test/AST/{Interp => ByteCode}/cxx26.cpp (100%)
 rename clang/test/AST/{Interp => ByteCode}/cxx2a.cpp (100%)
 rename clang/test/AST/{Interp => ByteCode}/cxx98.cpp (100%)
 rename clang/test/AST/{Interp => ByteCode}/depth-limit.cpp (100%)
 rename clang/test/AST/{Interp => ByteCode}/depth-limit2.cpp (100%)
 rename clang/test/AST/{Interp => ByteCode}/enums-targets.cpp (100%)
 rename clang/test/AST/{Interp => ByteCode}/enums.cpp (100%)
 rename clang/test/AST/{Interp => ByteCode}/eval-order.cpp (100%)
 rename clang/test/AST/{Interp => ByteCode}/floats.cpp (100%)
 rename clang/test/AST/{Interp => ByteCode}/functions.cpp (100%)
 rename clang/test/AST/{Interp => ByteCode}/hlsl.hlsl (100%)
 rename clang/test/AST/{Interp => ByteCode}/if.cpp (100%)
 rename clang/test/AST/{Interp => ByteCode}/intap.cpp (100%)
 rename clang/test/AST/{Interp => ByteCode}/invalid.cpp (100%)
 rename clang/test/AST/{Interp => ByteCode}/lambda.cpp (100%)
 rename clang/test/AST/{Interp => ByteCode}/lifetimes.cpp (100%)
 rename clang/test/AST/{Interp => ByteCode}/literals.cpp (100%)
 rename clang/test/AST/{Interp => ByteCode}/loops.cpp (100%)
 rename clang/test/AST/{Interp => ByteCode}/memberpointers.cpp (100%)
 rename clang/test/AST/{Interp => ByteCode}/ms.cpp (100%)
 rename clang/test/AST/{Interp => ByteCode}/mutable.cpp (100%)
 rename clang/test/AST/{Interp => ByteCode}/new-delete.cpp (100%)
 rename clang/test/AST/{Interp => ByteCode}/nullable.cpp (100%)
 rename clang/test/AST/{Interp => ByteCode}/objc.mm (100%)
 rename clang/test/AST/{Interp => ByteCode}/opencl.cl (100%)
 rename clang/test/AST/{Interp => ByteCode}/pointer-addition.c (100%)
 rename clang/test/AST/{Interp => ByteCode}/records.cpp (100%)
 rename clang/test/AST/{Interp => ByteCode}/references.cpp (100%)
 rename clang/test/AST/{Interp => ByteCode}/shifts.cpp (100%)
 rename clang/test/AST/{Interp => ByteCode}/spaceship.cpp (100%)
 rename clang/test/AST/{Interp => ByteCode}/switch.cpp (100%)
 rename clang/test/AST/{Interp => ByteCode}/sycl.cpp (100%)
 rename clang/test/AST/{Interp => ByteCode}/unions.cpp (100%)
 rename clang/test/AST/{Interp => ByteCode}/vectors.cpp (100%)
 rename clang/test/AST/{Interp => ByteCode}/weak.cpp (100%)
 rename clang/unittests/AST/{Interp => ByteCode}/CMakeLists.txt (100%)
 rename clang/unittests/AST/{Interp => ByteCode}/Descriptor.cpp (95%)
 rename clang/unittests/AST/{Interp => ByteCode}/toAPValue.cpp (98%)

diff --git a/clang/docs/ClangFormattedStatus.rst b/clang/docs/ClangFormattedStatus.rst
index 68c13076ccbb4ec..0ee0782879ef6b4 100644
--- a/clang/docs/ClangFormattedStatus.rst
+++ b/clang/docs/ClangFormattedStatus.rst
@@ -439,7 +439,7 @@ tree in terms of conformance to :doc:`ClangFormat` as of: March 06, 2022 17:32:2
      - `2`
      - `79`
      - :part:`2%`
-   * - clang/lib/AST/Interp
+   * - clang/lib/AST/ByteCode
      - `44`
      - `18`
      - `26`
diff --git a/clang/docs/tools/clang-formatted-files.txt b/clang/docs/tools/clang-formatted-files.txt
index 62871133a680753..fc07357986d989f 100644
--- a/clang/docs/tools/clang-formatted-files.txt
+++ b/clang/docs/tools/clang-formatted-files.txt
@@ -322,24 +322,24 @@ clang/lib/APINotes/APINotesTypes.cpp
 clang/lib/APINotes/APINotesYAMLCompiler.cpp
 clang/lib/AST/DataCollection.cpp
 clang/lib/AST/Linkage.h
-clang/lib/AST/Interp/ByteCodeGenError.cpp
-clang/lib/AST/Interp/ByteCodeGenError.h
-clang/lib/AST/Interp/Context.cpp
-clang/lib/AST/Interp/Context.h
-clang/lib/AST/Interp/Descriptor.cpp
-clang/lib/AST/Interp/Disasm.cpp
-clang/lib/AST/Interp/EvalEmitter.h
-clang/lib/AST/Interp/Frame.cpp
-clang/lib/AST/Interp/Frame.h
-clang/lib/AST/Interp/InterpState.h
-clang/lib/AST/Interp/Opcode.h
-clang/lib/AST/Interp/Pointer.cpp
-clang/lib/AST/Interp/PrimType.cpp
-clang/lib/AST/Interp/Record.h
-clang/lib/AST/Interp/Source.cpp
-clang/lib/AST/Interp/Source.h
-clang/lib/AST/Interp/State.cpp
-clang/lib/AST/Interp/State.h
+clang/lib/AST/ByteCode/ByteCodeGenError.cpp
+clang/lib/AST/ByteCode/ByteCodeGenError.h
+clang/lib/AST/ByteCode/Context.cpp
+clang/lib/AST/ByteCode/Context.h
+clang/lib/AST/ByteCode/Descriptor.cpp
+clang/lib/AST/ByteCode/Disasm.cpp
+clang/lib/AST/ByteCode/EvalEmitter.h
+clang/lib/AST/ByteCode/Frame.cpp
+clang/lib/AST/ByteCode/Frame.h
+clang/lib/AST/ByteCode/InterpState.h
+clang/lib/AST/ByteCode/Opcode.h
+clang/lib/AST/ByteCode/Pointer.cpp
+clang/lib/AST/ByteCode/PrimType.cpp
+clang/lib/AST/ByteCode/Record.h
+clang/lib/AST/ByteCode/Source.cpp
+clang/lib/AST/ByteCode/Source.h
+clang/lib/AST/ByteCode/State.cpp
+clang/lib/AST/ByteCode/State.h
 clang/lib/ASTMatchers/GtestMatchers.cpp
 clang/lib/ASTMatchers/Dynamic/Marshallers.cpp
 clang/lib/Basic/Attributes.cpp
diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index 0dbbe0043f70317..f35a025fc8edfc9 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -11,8 +11,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/AST/ASTContext.h"
+#include "ByteCode/Context.h"
 #include "CXXABI.h"
-#include "Interp/Context.h"
 #include "clang/AST/APValue.h"
 #include "clang/AST/ASTConcept.h"
 #include "clang/AST/ASTMutationListener.h"
diff --git a/clang/lib/AST/Interp/Boolean.h b/clang/lib/AST/ByteCode/Boolean.h
similarity index 96%
rename from clang/lib/AST/Interp/Boolean.h
rename to clang/lib/AST/ByteCode/Boolean.h
index 23f728603676464..0a2e0fe332110e9 100644
--- a/clang/lib/AST/Interp/Boolean.h
+++ b/clang/lib/AST/ByteCode/Boolean.h
@@ -9,25 +9,25 @@
 #ifndef LLVM_CLANG_AST_INTERP_BOOLEAN_H
 #define LLVM_CLANG_AST_INTERP_BOOLEAN_H
 
-#include <cstddef>
-#include <cstdint>
 #include "Integral.h"
 #include "clang/AST/APValue.h"
 #include "clang/AST/ComparisonCategories.h"
 #include "llvm/ADT/APSInt.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
+#include <cstddef>
+#include <cstdint>
 
 namespace clang {
 namespace interp {
 
 /// Wrapper around boolean types.
 class Boolean final {
- private:
+private:
   /// Underlying boolean.
   bool V;
 
- public:
+public:
   /// Zero-initializes a boolean.
   Boolean() : V(false) {}
   explicit Boolean(bool V) : V(V) {}
@@ -104,8 +104,7 @@ class Boolean final {
 
   static Boolean zero() { return from(false); }
 
-  template <typename T>
-  static Boolean from(T Value, unsigned NumBits) {
+  template <typename T> static Boolean from(T Value, unsigned NumBits) {
     return Boolean(Value);
   }
 
@@ -153,7 +152,7 @@ inline llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const Boolean &B) {
   return OS;
 }
 
-}  // namespace interp
-}  // namespace clang
+} // namespace interp
+} // namespace clang
 
 #endif
diff --git a/clang/lib/AST/Interp/ByteCodeEmitter.cpp b/clang/lib/AST/ByteCode/ByteCodeEmitter.cpp
similarity index 97%
rename from clang/lib/AST/Interp/ByteCodeEmitter.cpp
rename to clang/lib/AST/ByteCode/ByteCodeEmitter.cpp
index a01fa15dc0b7dc9..1f6bdaff50b5bff 100644
--- a/clang/lib/AST/Interp/ByteCodeEmitter.cpp
+++ b/clang/lib/AST/ByteCode/ByteCodeEmitter.cpp
@@ -206,8 +206,7 @@ void ByteCodeEmitter::emitLabel(LabelTy Label) {
   const size_t Target = Code.size();
   LabelOffsets.insert({Label, Target});
 
-  if (auto It = LabelRelocs.find(Label);
-      It != LabelRelocs.end()) {
+  if (auto It = LabelRelocs.find(Label); It != LabelRelocs.end()) {
     for (unsigned Reloc : It->second) {
       using namespace llvm::support;
 
@@ -228,8 +227,7 @@ int32_t ByteCodeEmitter::getOffset(LabelTy Label) {
   assert(aligned(Position));
 
   // If target is known, compute jump offset.
-  if (auto It = LabelOffsets.find(Label);
-      It != LabelOffsets.end())
+  if (auto It = LabelOffsets.find(Label); It != LabelOffsets.end())
     return It->second - Position;
 
   // Otherwise, record relocation and return dummy offset.
@@ -308,7 +306,8 @@ void emit(Program &P, std::vector<std::byte> &Code, const IntegralAP<true> &Val,
 }
 
 template <typename... Tys>
-bool ByteCodeEmitter::emitOp(Opcode Op, const Tys &... Args, const SourceInfo &SI) {
+bool ByteCodeEmitter::emitOp(Opcode Op, const Tys &...Args,
+                             const SourceInfo &SI) {
   bool Success = true;
 
   // The opcode is followed by arguments. The source info is
diff --git a/clang/lib/AST/Interp/ByteCodeEmitter.h b/clang/lib/AST/ByteCode/ByteCodeEmitter.h
similarity index 97%
rename from clang/lib/AST/Interp/ByteCodeEmitter.h
rename to clang/lib/AST/ByteCode/ByteCodeEmitter.h
index a19a25c2f9e8ecc..59e12defd8e52b5 100644
--- a/clang/lib/AST/Interp/ByteCodeEmitter.h
+++ b/clang/lib/AST/ByteCode/ByteCodeEmitter.h
@@ -92,7 +92,7 @@ class ByteCodeEmitter {
 
   /// Emits an opcode.
   template <typename... Tys>
-  bool emitOp(Opcode Op, const Tys &... Args, const SourceInfo &L);
+  bool emitOp(Opcode Op, const Tys &...Args, const SourceInfo &L);
 
 protected:
 #define GET_LINK_PROTO
diff --git a/clang/lib/AST/Interp/Compiler.cpp b/clang/lib/AST/ByteCode/Compiler.cpp
similarity index 99%
rename from clang/lib/AST/Interp/Compiler.cpp
rename to clang/lib/AST/ByteCode/Compiler.cpp
index 5e1f507ca2b1785..c3ecaa40b86f2a1 100644
--- a/clang/lib/AST/Interp/Compiler.cpp
+++ b/clang/lib/AST/ByteCode/Compiler.cpp
@@ -3722,7 +3722,8 @@ bool Compiler<Emitter>::visitDeclAndReturn(const VarDecl *VD,
 }
 
 template <class Emitter>
-VarCreationState Compiler<Emitter>::visitVarDecl(const VarDecl *VD, bool Toplevel) {
+VarCreationState Compiler<Emitter>::visitVarDecl(const VarDecl *VD,
+                                                 bool Toplevel) {
   // We don't know what to do with these, so just return false.
   if (VD->getType().isNull())
     return false;
@@ -5124,7 +5125,7 @@ bool Compiler<Emitter>::VisitUnaryOperator(const UnaryOperator *E) {
     if (!this->visit(SubExpr))
       return false;
     return DiscardResult ? this->emitPop(*T, E) : this->emitNeg(*T, E);
-  case UO_Plus:                // +x
+  case UO_Plus: // +x
     if (!T)
       return this->emitError(E);
 
diff --git a/clang/lib/AST/Interp/Compiler.h b/clang/lib/AST/ByteCode/Compiler.h
similarity index 100%
rename from clang/lib/AST/Interp/Compiler.h
rename to clang/lib/AST/ByteCode/Compiler.h
diff --git a/clang/lib/AST/Interp/Context.cpp b/clang/lib/AST/ByteCode/Context.cpp
similarity index 100%
rename from clang/lib/AST/Interp/Context.cpp
rename to clang/lib/AST/ByteCode/Context.cpp
diff --git a/clang/lib/AST/Interp/Context.h b/clang/lib/AST/ByteCode/Context.h
similarity index 100%
rename from clang/lib/AST/Interp/Context.h
rename to clang/lib/AST/ByteCode/Context.h
diff --git a/clang/lib/AST/Interp/Descriptor.cpp b/clang/lib/AST/ByteCode/Descriptor.cpp
similarity index 100%
rename from clang/lib/AST/Interp/Descriptor.cpp
rename to clang/lib/AST/ByteCode/Descriptor.cpp
diff --git a/clang/lib/AST/Interp/Descriptor.h b/clang/lib/AST/ByteCode/Descriptor.h
similarity index 99%
rename from clang/lib/AST/Interp/Descriptor.h
rename to clang/lib/AST/ByteCode/Descriptor.h
index 6f1adeb898c4307..41899c3bd6831a2 100644
--- a/clang/lib/AST/Interp/Descriptor.h
+++ b/clang/lib/AST/ByteCode/Descriptor.h
@@ -229,7 +229,7 @@ struct Descriptor final {
   /// Returns the allocated size, including metadata.
   unsigned getAllocSize() const { return AllocSize; }
   /// returns the size of an element when the structure is viewed as an array.
-  unsigned getElemSize()  const { return ElemSize; }
+  unsigned getElemSize() const { return ElemSize; }
   /// Returns the size of the metadata.
   unsigned getMetadataSize() const { return MDSize; }
 
diff --git a/clang/lib/AST/Interp/Disasm.cpp b/clang/lib/AST/ByteCode/Disasm.cpp
similarity index 100%
rename from clang/lib/AST/Interp/Disasm.cpp
rename to clang/lib/AST/ByteCode/Disasm.cpp
diff --git a/clang/lib/AST/Interp/DynamicAllocator.cpp b/clang/lib/AST/ByteCode/DynamicAllocator.cpp
similarity index 100%
rename from clang/lib/AST/Interp/DynamicAllocator.cpp
rename to clang/lib/AST/ByteCode/DynamicAllocator.cpp
diff --git a/clang/lib/AST/Interp/DynamicAllocator.h b/clang/lib/AST/ByteCode/DynamicAllocator.h
similarity index 100%
rename from clang/lib/AST/Interp/DynamicAllocator.h
rename to clang/lib/AST/ByteCode/DynamicAllocator.h
diff --git a/clang/lib/AST/Interp/EvalEmitter.cpp b/clang/lib/AST/ByteCode/EvalEmitter.cpp
similarity index 99%
rename from clang/lib/AST/Interp/EvalEmitter.cpp
rename to clang/lib/AST/ByteCode/EvalEmitter.cpp
index 08536536ac3c2eb..e36d86c814e17fd 100644
--- a/clang/lib/AST/Interp/EvalEmitter.cpp
+++ b/clang/lib/AST/ByteCode/EvalEmitter.cpp
@@ -76,9 +76,7 @@ EvaluationResult EvalEmitter::interpretDecl(const VarDecl *VD,
   return std::move(this->EvalResult);
 }
 
-void EvalEmitter::emitLabel(LabelTy Label) {
-  CurrentLabel = Label;
-}
+void EvalEmitter::emitLabel(LabelTy Label) { CurrentLabel = Label; }
 
 EvalEmitter::LabelTy EvalEmitter::getLabel() { return NextLabel++; }
 
diff --git a/clang/lib/AST/Interp/EvalEmitter.h b/clang/lib/AST/ByteCode/EvalEmitter.h
similarity index 100%
rename from clang/lib/AST/Interp/EvalEmitter.h
rename to clang/lib/AST/ByteCode/EvalEmitter.h
diff --git a/clang/lib/AST/Interp/EvaluationResult.cpp b/clang/lib/AST/ByteCode/EvaluationResult.cpp
similarity index 100%
rename from clang/lib/AST/Interp/EvaluationResult.cpp
rename to clang/lib/AST/ByteCode/EvaluationResult.cpp
diff --git a/clang/lib/AST/Interp/EvaluationResult.h b/clang/lib/AST/ByteCode/EvaluationResult.h
similarity index 100%
rename from clang/lib/AST/Interp/EvaluationResult.h
rename to clang/lib/AST/ByteCode/EvaluationResult.h
diff --git a/clang/lib/AST/Interp/Floating.cpp b/clang/lib/AST/ByteCode/Floating.cpp
similarity index 100%
rename from clang/lib/AST/Interp/Floating.cpp
rename to clang/lib/AST/ByteCode/Floating.cpp
diff --git a/clang/lib/AST/Interp/Floating.h b/clang/lib/AST/ByteCode/Floating.h
similarity index 100%
rename from clang/lib/AST/Interp/Floating.h
rename to clang/lib/AST/ByteCode/Floating.h
diff --git a/clang/lib/AST/Interp/Frame.cpp b/clang/lib/AST/ByteCode/Frame.cpp
similarity index 100%
rename from clang/lib/AST/Interp/Frame.cpp
rename to clang/lib/AST/ByteCode/Frame.cpp
diff --git a/clang/lib/AST/Interp/Frame.h b/clang/lib/AST/ByteCode/Frame.h
similarity index 100%
rename from clang/lib/AST/Interp/Frame.h
rename to clang/lib/AST/ByteCode/Frame.h
diff --git a/clang/lib/AST/Interp/Function.cpp b/clang/lib/AST/ByteCode/Function.cpp
similarity index 100%
rename from clang/lib/AST/Interp/Function.cpp
rename to clang/lib/AST/ByteCode/Function.cpp
diff --git a/clang/lib/AST/Interp/Function.h b/clang/lib/AST/ByteCode/Function.h
similarity index 100%
rename from clang/lib/AST/Interp/Function.h
rename to clang/lib/AST/ByteCode/Function.h
diff --git a/clang/lib/AST/Interp/FunctionPointer.h b/clang/lib/AST/ByteCode/FunctionPointer.h
similarity index 100%
rename from clang/lib/AST/Interp/FunctionPointer.h
rename to clang/lib/AST/ByteCode/FunctionPointer.h
diff --git a/clang/lib/AST/Interp/Integral.h b/clang/lib/AST/ByteCode/Integral.h
similarity index 93%
rename from clang/lib/AST/Interp/Integral.h
rename to clang/lib/AST/ByteCode/Integral.h
index aafdd02676c966f..e06ec1669259da4 100644
--- a/clang/lib/AST/Interp/Integral.h
+++ b/clang/lib/AST/ByteCode/Integral.h
@@ -13,8 +13,8 @@
 #ifndef LLVM_CLANG_AST_INTERP_INTEGRAL_H
 #define LLVM_CLANG_AST_INTERP_INTEGRAL_H
 
-#include "clang/AST/ComparisonCategories.h"
 #include "clang/AST/APValue.h"
+#include "clang/AST/ComparisonCategories.h"
 #include "llvm/ADT/APSInt.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
@@ -33,14 +33,30 @@ template <bool Signed> class IntegralAP;
 
 // Helper structure to select the representation.
 template <unsigned Bits, bool Signed> struct Repr;
-template <> struct Repr<8, false> { using Type = uint8_t; };
-template <> struct Repr<16, false> { using Type = uint16_t; };
-template <> struct Repr<32, false> { using Type = uint32_t; };
-template <> struct Repr<64, false> { using Type = uint64_t; };
-template <> struct Repr<8, true> { using Type = int8_t; };
-template <> struct Repr<16, true> { using Type = int16_t; };
-template <> struct Repr<32, true> { using Type = int32_t; };
-template <> struct Repr<64, true> { using Type = int64_t; };
+template <> struct Repr<8, false> {
+  using Type = uint8_t;
+};
+template <> struct Repr<16, false> {
+  using Type = uint16_t;
+};
+template <> struct Repr<32, false> {
+  using Type = uint32_t;
+};
+template <> struct Repr<64, false> {
+  using Type = uint64_t;
+};
+template <> struct Repr<8, true> {
+  using Type = int8_t;
+};
+template <> struct Repr<16, true> {
+  using Type = int16_t;
+};
+template <> struct Repr<32, true> {
+  using Type = int32_t;
+};
+template <> struct Repr<64, true> {
+  using Type = int64_t;
+};
 
 /// Wrapper around numeric types.
 ///
@@ -159,12 +175,8 @@ template <unsigned Bits, bool Signed> class Integral final {
 
   void print(llvm::raw_ostream &OS) const { OS << V; }
 
-  static Integral min(unsigned NumBits) {
-    return Integral(Min);
-  }
-  static Integral max(unsigned NumBits) {
-    return Integral(Max);
-  }
+  static Integral min(unsigned NumBits) { return Integral(Min); }
+  static Integral max(unsigned NumBits) { return Integral(Max); }
 
   template <typename ValT> static Integral from(ValT Value) {
     if constexpr (std::is_integral<ValT>::value)
diff --git a/clang/lib/AST/Interp/IntegralAP.h b/clang/lib/AST/ByteCode/IntegralAP.h
similarity index 100%
rename from clang/lib/AST/Interp/IntegralAP.h
rename to clang/lib/AST/ByteCode/IntegralAP.h
diff --git a/clang/lib/AST/Interp/Interp.cpp b/clang/lib/AST/ByteCode/Interp.cpp
similarity index 100%
rename from clang/lib/AST/Interp/Interp.cpp
rename to clang/lib/AST/ByteCode/Interp.cpp
diff --git a/clang/lib/AST/Interp/Interp.h b/clang/lib/AST/ByteCode/Interp.h
similarity index 99%
rename from clang/lib/AST/Interp/Interp.h
rename to clang/lib/AST/ByteCode/Interp.h
index c2d73f32f0b20c9..c3ce21cb13770ed 100644
--- a/clang/lib/AST/Interp/Interp.h
+++ b/clang/lib/AST/ByteCode/Interp.h
@@ -1226,7 +1226,7 @@ template <PrimType Name, class T = typename PrimConv<Name>::T>
 bool GetField(InterpState &S, CodePtr OpPC, uint32_t I) {
   const Pointer &Obj = S.Stk.peek<Pointer>();
   if (!CheckNull(S, OpPC, Obj, CSK_Field))
-      return false;
+    return false;
   if (!CheckRange(S, OpPC, Obj, CSK_Field))
     return false;
   const Pointer &Field = Obj.atField(I);
@@ -1542,7 +1542,7 @@ inline bool GetPtrActiveField(InterpState &S, CodePtr OpPC, uint32_t Off) {
 }
 
 inline bool GetPtrActiveThisField(InterpState &S, CodePtr OpPC, uint32_t Off) {
- if (S.checkingPotentialConstantExpression())
+  if (S.checkingPotentialConstantExpression())
     return false;
   const Pointer &This = S.Current->getThis();
   if (!CheckThis(S, OpPC, This))
@@ -2053,7 +2053,7 @@ template <PrimType TIn, PrimType TOut> bool Cast(InterpState &S, CodePtr OpPC) {
 /// 1) Pops a Floating from the stack.
 /// 2) Pushes a new floating on the stack that uses the given semantics.
 inline bool CastFP(InterpState &S, CodePtr OpPC, const llvm::fltSemantics *Sem,
-            llvm::RoundingMode RM) {
+                   llvm::RoundingMode RM) {
   Floating F = S.Stk.pop<Floating>();
   Floating Result = F.toSemantics(Sem, RM);
   S.Stk.push<Floating>(Result);
@@ -2313,9 +2313,9 @@ inline bool DoShift(InterpState &S, CodePtr OpPC, LT &LHS, RT &RHS) {
     if (!S.noteUndefinedBehavior())
       return false;
     RHS = -RHS;
-    return DoShift < LT, RT,
-           Dir == ShiftDir::Left ? ShiftDir::Right
-                                 : ShiftDir::Left > (S, OpPC, LHS, RHS);
+    return DoShift<LT, RT,
+                   Dir == ShiftDir::Left ? ShiftDir::Right : ShiftDir::Left>(
+        S, OpPC, LHS, RHS);
   }
 
   if constexpr (Dir == ShiftDir::Left) {
@@ -2467,7 +2467,8 @@ inline bool ArrayElemPop(InterpState &S, CodePtr OpPC, uint32_t Index) {
 }
 
 template <PrimType Name, class T = typename PrimConv<Name>::T>
-inline bool CopyArray(InterpState &S, CodePtr OpPC, uint32_t SrcIndex, uint32_t DestIndex, uint32_t Size) {
+inline bool CopyArray(InterpState &S, CodePtr OpPC, uint32_t SrcIndex,
+                      uint32_t DestIndex, uint32_t Size) {
   const auto &SrcPtr = S.Stk.pop<Pointer>();
   const auto &DestPtr = S.Stk.peek<Pointer>();
 
diff --git a/clang/lib/AST/Interp/InterpBlock.cpp b/clang/lib/AST/ByteCode/InterpBlock.cpp
similarity index 100%
rename from clang/lib/AST/Interp/InterpBlock.cpp
rename to clang/lib/AST/ByteCode/InterpBlock.cpp
diff --git a/clang/lib/AST/Interp/InterpBlock.h b/clang/lib/AST/ByteCode/InterpBlock.h
similarity index 100%
rename from clang/lib/AST/Interp/InterpBlock.h
rename to clang/lib/AST/ByteCode/InterpBlock.h
index 20cf8cb9d3810e9..a5cd58e3a655a0c 100644
--- a/clang/lib/AST/Interp/InterpBlock.h
+++ b/clang/lib/AST/ByteCode/InterpBlock.h
@@ -14,10 +14,10 @@
 #define LLVM_CLANG_AST_INTERP_BLOCK_H
 
 #include "Descriptor.h"
+#include "clang/AST/ComparisonCategories.h"
 #include "clang/AST/Decl.h"
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/Expr.h"
-#include "clang/AST/ComparisonCategories.h"
 #include "llvm/ADT/PointerUnion.h"
 #include "llvm/Support/raw_ostream.h"
 
diff --git a/clang/lib/AST/Interp/InterpBuiltin.cpp b/clang/lib/AST/ByteCode/InterpBuiltin.cpp
similarity index 100%
rename from clang/lib/AST/Interp/InterpBuiltin.cpp
rename to clang/lib/AST/ByteCode/InterpBuiltin.cpp
diff --git a/clang/lib/AST/Interp/InterpFrame.cpp b/clang/lib/AST/ByteCode/InterpFrame.cpp
similarity index 99%
rename from clang/lib/AST/Interp/InterpFrame.cpp
rename to clang/lib/AST/ByteCode/InterpFrame.cpp
index 4530f28d694e844..8b55b61cbbfa7e5 100644
--- a/clang/lib/AST/Interp/InterpFrame.cpp
+++ b/clang/lib/AST/ByteCode/InterpFrame.cpp
@@ -137,7 +137,7 @@ void print(llvm::raw_ostream &OS, const Pointer &P, ASTContext &Ctx,
   if (!Ty->isReferenceType())
     OS << "&";
   llvm::SmallVector<Pointer, 2> Levels;
-  for (Pointer F = P; !F.isRoot(); ) {
+  for (Pointer F = P; !F.isRoot();) {
     Levels.push_back(F);
     F = F.isArrayElement() ? F.getArray().expand() : F.getBase();
   }
diff --git a/clang/lib/AST/Interp/InterpFrame.h b/clang/lib/AST/ByteCode/InterpFrame.h
similarity index 99%
rename from clang/lib/AST/Interp/InterpFrame.h
rename to clang/lib/AST/ByteCode/InterpFrame.h
index 91b9b41b5d33431..1e0d2b1d4424b14 100644
--- a/clang/lib/AST/Interp/InterpFrame.h
+++ b/clang/lib/AST/ByteCode/InterpFrame.h
@@ -91,7 +91,7 @@ class InterpFrame final : public Frame {
 
   /// Mutates a local copy of a parameter.
   template <typename T> void setParam(unsigned Offset, const T &Value) {
-     getParamPointer(Offset).deref<T>() = Value;
+    getParamPointer(Offset).deref<T>() = Value;
   }
 
   /// Returns a pointer to an argument - lazily creates a block.
diff --git a/clang/lib/AST/Interp/InterpShared.cpp b/clang/lib/AST/ByteCode/InterpShared.cpp
similarity index 100%
rename from clang/lib/AST/Interp/InterpShared.cpp
rename to clang/lib/AST/ByteCode/InterpShared.cpp
diff --git a/clang/lib/AST/Interp/InterpShared.h b/clang/lib/AST/ByteCode/InterpShared.h
similarity index 100%
rename from clang/lib/AST/Interp/InterpShared.h
rename to clang/lib/AST/ByteCode/InterpShared.h
diff --git a/clang/lib/AST/Interp/InterpStack.cpp b/clang/lib/AST/ByteCode/InterpStack.cpp
similarity index 98%
rename from clang/lib/AST/Interp/InterpStack.cpp
rename to clang/lib/AST/ByteCode/InterpStack.cpp
index c7024740d322ef2..b8cdaeee72166cf 100644
--- a/clang/lib/AST/Interp/InterpStack.cpp
+++ b/clang/lib/AST/ByteCode/InterpStack.cpp
@@ -18,9 +18,7 @@
 using namespace clang;
 using namespace clang::interp;
 
-InterpStack::~InterpStack() {
-  clear();
-}
+InterpStack::~InterpStack() { clear(); }
 
 void InterpStack::clear() {
   if (Chunk && Chunk->Next)
diff --git a/clang/lib/AST/Interp/InterpStack.h b/clang/lib/AST/ByteCode/InterpStack.h
similarity index 97%
rename from clang/lib/AST/Interp/InterpStack.h
rename to clang/lib/AST/ByteCode/InterpStack.h
index 4966e2870de6bd9..153d17f0d70e1bd 100644
--- a/clang/lib/AST/Interp/InterpStack.h
+++ b/clang/lib/AST/ByteCode/InterpStack.h
@@ -32,7 +32,7 @@ class InterpStack final {
   ~InterpStack();
 
   /// Constructs a value in place on the top of the stack.
-  template <typename T, typename... Tys> void push(Tys &&... Args) {
+  template <typename T, typename... Tys> void push(Tys &&...Args) {
     new (grow(aligned_size<T>())) T(std::forward<Tys>(Args)...);
 #ifndef NDEBUG
     ItemTypes.push_back(toPrimType<T>());
@@ -153,8 +153,7 @@ class InterpStack final {
   template <typename T> static constexpr PrimType toPrimType() {
     if constexpr (std::is_same_v<T, Pointer>)
       return PT_Ptr;
-    else if constexpr (std::is_same_v<T, bool> ||
-                       std::is_same_v<T, Boolean>)
+    else if constexpr (std::is_same_v<T, bool> || std::is_same_v<T, Boolean>)
       return PT_Bool;
     else if constexpr (std::is_same_v<T, int8_t> ||
                        std::is_same_v<T, Integral<8, true>>)
diff --git a/clang/lib/AST/Interp/InterpState.cpp b/clang/lib/AST/ByteCode/InterpState.cpp
similarity index 100%
rename from clang/lib/AST/Interp/InterpState.cpp
rename to clang/lib/AST/ByteCode/InterpState.cpp
diff --git a/clang/lib/AST/Interp/InterpState.h b/clang/lib/AST/ByteCode/InterpState.h
similarity index 100%
rename from clang/lib/AST/Interp/InterpState.h
rename to clang/lib/AST/ByteCode/InterpState.h
diff --git a/clang/lib/AST/Interp/MemberPointer.cpp b/clang/lib/AST/ByteCode/MemberPointer.cpp
similarity index 100%
rename from clang/lib/AST/Interp/MemberPointer.cpp
rename to clang/lib/AST/ByteCode/MemberPointer.cpp
diff --git a/clang/lib/AST/Interp/MemberPointer.h b/clang/lib/AST/ByteCode/MemberPointer.h
similarity index 100%
rename from clang/lib/AST/Interp/MemberPointer.h
rename to clang/lib/AST/ByteCode/MemberPointer.h
diff --git a/clang/lib/AST/Interp/Opcode.h b/clang/lib/AST/ByteCode/Opcode.h
similarity index 100%
rename from clang/lib/AST/Interp/Opcode.h
rename to clang/lib/AST/ByteCode/Opcode.h
diff --git a/clang/lib/AST/Interp/Opcodes.td b/clang/lib/AST/ByteCode/Opcodes.td
similarity index 100%
rename from clang/lib/AST/Interp/Opcodes.td
rename to clang/lib/AST/ByteCode/Opcodes.td
diff --git a/clang/lib/AST/Interp/Pointer.cpp b/clang/lib/AST/ByteCode/Pointer.cpp
similarity index 100%
rename from clang/lib/AST/Interp/Pointer.cpp
rename to clang/lib/AST/ByteCode/Pointer.cpp
diff --git a/clang/lib/AST/Interp/Pointer.h b/clang/lib/AST/ByteCode/Pointer.h
similarity index 99%
rename from clang/lib/AST/Interp/Pointer.h
rename to clang/lib/AST/ByteCode/Pointer.h
index 01ccb88ec03b292..8db081c0ec82b33 100644
--- a/clang/lib/AST/Interp/Pointer.h
+++ b/clang/lib/AST/ByteCode/Pointer.h
@@ -173,7 +173,7 @@ class Pointer {
 
   /// Subtract the given offset from the current Base and Offset
   /// of the pointer.
-  [[nodiscard]]  Pointer atFieldSub(unsigned Off) const {
+  [[nodiscard]] Pointer atFieldSub(unsigned Off) const {
     assert(Offset >= Off);
     unsigned O = Offset - Off;
     return Pointer(asBlockPointer().Pointee, O, O);
diff --git a/clang/lib/AST/Interp/PrimType.cpp b/clang/lib/AST/ByteCode/PrimType.cpp
similarity index 100%
rename from clang/lib/AST/Interp/PrimType.cpp
rename to clang/lib/AST/ByteCode/PrimType.cpp
diff --git a/clang/lib/AST/Interp/PrimType.h b/clang/lib/AST/ByteCode/PrimType.h
similarity index 83%
rename from clang/lib/AST/Interp/PrimType.h
rename to clang/lib/AST/ByteCode/PrimType.h
index 20fb5e81774d6ff..bb2f59d86e98d78 100644
--- a/clang/lib/AST/Interp/PrimType.h
+++ b/clang/lib/AST/ByteCode/PrimType.h
@@ -73,23 +73,45 @@ constexpr bool isIntegralType(PrimType T) { return T <= PT_Bool; }
 
 /// Mapping from primitive types to their representation.
 template <PrimType T> struct PrimConv;
-template <> struct PrimConv<PT_Sint8> { using T = Integral<8, true>; };
-template <> struct PrimConv<PT_Uint8> { using T = Integral<8, false>; };
-template <> struct PrimConv<PT_Sint16> { using T = Integral<16, true>; };
-template <> struct PrimConv<PT_Uint16> { using T = Integral<16, false>; };
-template <> struct PrimConv<PT_Sint32> { using T = Integral<32, true>; };
-template <> struct PrimConv<PT_Uint32> { using T = Integral<32, false>; };
-template <> struct PrimConv<PT_Sint64> { using T = Integral<64, true>; };
-template <> struct PrimConv<PT_Uint64> { using T = Integral<64, false>; };
+template <> struct PrimConv<PT_Sint8> {
+  using T = Integral<8, true>;
+};
+template <> struct PrimConv<PT_Uint8> {
+  using T = Integral<8, false>;
+};
+template <> struct PrimConv<PT_Sint16> {
+  using T = Integral<16, true>;
+};
+template <> struct PrimConv<PT_Uint16> {
+  using T = Integral<16, false>;
+};
+template <> struct PrimConv<PT_Sint32> {
+  using T = Integral<32, true>;
+};
+template <> struct PrimConv<PT_Uint32> {
+  using T = Integral<32, false>;
+};
+template <> struct PrimConv<PT_Sint64> {
+  using T = Integral<64, true>;
+};
+template <> struct PrimConv<PT_Uint64> {
+  using T = Integral<64, false>;
+};
 template <> struct PrimConv<PT_IntAP> {
   using T = IntegralAP<false>;
 };
 template <> struct PrimConv<PT_IntAPS> {
   using T = IntegralAP<true>;
 };
-template <> struct PrimConv<PT_Float> { using T = Floating; };
-template <> struct PrimConv<PT_Bool> { using T = Boolean; };
-template <> struct PrimConv<PT_Ptr> { using T = Pointer; };
+template <> struct PrimConv<PT_Float> {
+  using T = Floating;
+};
+template <> struct PrimConv<PT_Bool> {
+  using T = Boolean;
+};
+template <> struct PrimConv<PT_Ptr> {
+  using T = Pointer;
+};
 template <> struct PrimConv<PT_FnPtr> {
   using T = FunctionPointer;
 };
@@ -117,8 +139,12 @@ static inline bool aligned(const void *P) {
 
 /// Helper macro to simplify type switches.
 /// The macro implicitly exposes a type T in the scope of the inner block.
-#define TYPE_SWITCH_CASE(Name, B) \
-  case Name: { using T = PrimConv<Name>::T; B; break; }
+#define TYPE_SWITCH_CASE(Name, B)                                              \
+  case Name: {                                                                 \
+    using T = PrimConv<Name>::T;                                               \
+    B;                                                                         \
+    break;                                                                     \
+  }
 #define TYPE_SWITCH(Expr, B)                                                   \
   do {                                                                         \
     switch (Expr) {                                                            \
@@ -181,7 +207,10 @@ static inline bool aligned(const void *P) {
   do {                                                                         \
     switch (Expr) {                                                            \
       TYPE_SWITCH_CASE(PT_Ptr, B)                                              \
-      default: { D; break; }                                                   \
+    default: {                                                                 \
+      D;                                                                       \
+      break;                                                                   \
+    }                                                                          \
     }                                                                          \
   } while (0)
 #endif
diff --git a/clang/lib/AST/Interp/Primitives.h b/clang/lib/AST/ByteCode/Primitives.h
similarity index 100%
rename from clang/lib/AST/Interp/Primitives.h
rename to clang/lib/AST/ByteCode/Primitives.h
diff --git a/clang/lib/AST/Interp/Program.cpp b/clang/lib/AST/ByteCode/Program.cpp
similarity index 95%
rename from clang/lib/AST/Interp/Program.cpp
rename to clang/lib/AST/ByteCode/Program.cpp
index 9109956dc652cb2..25b99115ddd777e 100644
--- a/clang/lib/AST/Interp/Program.cpp
+++ b/clang/lib/AST/ByteCode/Program.cpp
@@ -76,26 +76,26 @@ unsigned Program::createGlobalString(const StringLiteral *S) {
     Pointer Field = Ptr.atIndex(I).narrow();
     const uint32_t CodePoint = I == N ? 0 : S->getCodeUnit(I);
     switch (CharType) {
-      case PT_Sint8: {
-        using T = PrimConv<PT_Sint8>::T;
-        Field.deref<T>() = T::from(CodePoint, BitWidth);
-        Field.initialize();
-        break;
-      }
-      case PT_Uint16: {
-        using T = PrimConv<PT_Uint16>::T;
-        Field.deref<T>() = T::from(CodePoint, BitWidth);
-        Field.initialize();
-        break;
-      }
-      case PT_Uint32: {
-        using T = PrimConv<PT_Uint32>::T;
-        Field.deref<T>() = T::from(CodePoint, BitWidth);
-        Field.initialize();
-        break;
-      }
-      default:
-        llvm_unreachable("unsupported character type");
+    case PT_Sint8: {
+      using T = PrimConv<PT_Sint8>::T;
+      Field.deref<T>() = T::from(CodePoint, BitWidth);
+      Field.initialize();
+      break;
+    }
+    case PT_Uint16: {
+      using T = PrimConv<PT_Uint16>::T;
+      Field.deref<T>() = T::from(CodePoint, BitWidth);
+      Field.initialize();
+      break;
+    }
+    case PT_Uint32: {
+      using T = PrimConv<PT_Uint32>::T;
+      Field.deref<T>() = T::from(CodePoint, BitWidth);
+      Field.initialize();
+      break;
+    }
+    default:
+      llvm_unreachable("unsupported character type");
     }
   }
   return I;
@@ -394,8 +394,7 @@ Descriptor *Program::createDescriptor(const DeclTy &D, const Type *Ty,
             D, ElemTy.getTypePtr(), std::nullopt, IsConst, IsTemporary);
         if (!ElemDesc)
           return nullptr;
-        unsigned ElemSize =
-            ElemDesc->getAllocSize() + sizeof(InlineDescriptor);
+        unsigned ElemSize = ElemDesc->getAllocSize() + sizeof(InlineDescriptor);
         if (std::numeric_limits<unsigned>::max() / ElemSize <= NumElems)
           return {};
         return allocateDescriptor(D, ElemDesc, MDSize, NumElems, IsConst,
diff --git a/clang/lib/AST/Interp/Program.h b/clang/lib/AST/ByteCode/Program.h
similarity index 96%
rename from clang/lib/AST/Interp/Program.h
rename to clang/lib/AST/ByteCode/Program.h
index 1cabc5212180f28..7f69d9790fc79ac 100644
--- a/clang/lib/AST/Interp/Program.h
+++ b/clang/lib/AST/ByteCode/Program.h
@@ -13,8 +13,6 @@
 #ifndef LLVM_CLANG_AST_INTERP_PROGRAM_H
 #define LLVM_CLANG_AST_INTERP_PROGRAM_H
 
-#include <map>
-#include <vector>
 #include "Function.h"
 #include "Pointer.h"
 #include "PrimType.h"
@@ -24,6 +22,8 @@
 #include "llvm/ADT/PointerUnion.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Allocator.h"
+#include <map>
+#include <vector>
 
 namespace clang {
 class RecordDecl;
@@ -94,15 +94,14 @@ class Program final {
 
   /// Creates a new function from a code range.
   template <typename... Ts>
-  Function *createFunction(const FunctionDecl *Def, Ts &&... Args) {
+  Function *createFunction(const FunctionDecl *Def, Ts &&...Args) {
     Def = Def->getCanonicalDecl();
     auto *Func = new Function(*this, Def, std::forward<Ts>(Args)...);
     Funcs.insert({Def, std::unique_ptr<Function>(Func)});
     return Func;
   }
   /// Creates an anonymous function.
-  template <typename... Ts>
-  Function *createFunction(Ts &&... Args) {
+  template <typename... Ts> Function *createFunction(Ts &&...Args) {
     auto *Func = new Function(*this, std::forward<Ts>(Args)...);
     AnonFuncs.emplace_back(Func);
     return Func;
@@ -213,8 +212,7 @@ class Program final {
   llvm::DenseMap<const ValueDecl *, unsigned> DummyVariables;
 
   /// Creates a new descriptor.
-  template <typename... Ts>
-  Descriptor *allocateDescriptor(Ts &&... Args) {
+  template <typename... Ts> Descriptor *allocateDescriptor(Ts &&...Args) {
     return new (Allocator) Descriptor(std::forward<Ts>(Args)...);
   }
 
@@ -232,9 +230,7 @@ class Program final {
   }
 
   /// Ends a global declaration.
-  void endDeclaration() {
-    CurrentDeclaration = NoDeclaration;
-  }
+  void endDeclaration() { CurrentDeclaration = NoDeclaration; }
 
 public:
   /// Dumps the disassembled bytecode to \c llvm::errs().
diff --git a/clang/lib/AST/Interp/Record.cpp b/clang/lib/AST/ByteCode/Record.cpp
similarity index 96%
rename from clang/lib/AST/Interp/Record.cpp
rename to clang/lib/AST/ByteCode/Record.cpp
index e7b741f2d458c84..ec1b55da347af66 100644
--- a/clang/lib/AST/Interp/Record.cpp
+++ b/clang/lib/AST/ByteCode/Record.cpp
@@ -18,7 +18,7 @@ Record::Record(const RecordDecl *Decl, BaseList &&SrcBases,
     : Decl(Decl), Bases(std::move(SrcBases)), Fields(std::move(SrcFields)),
       BaseSize(BaseSize), VirtualSize(VirtualSize), IsUnion(Decl->isUnion()) {
   for (Base &V : SrcVirtualBases)
-    VirtualBases.push_back({ V.Decl, V.Offset + BaseSize, V.Desc, V.R });
+    VirtualBases.push_back({V.Decl, V.Offset + BaseSize, V.Desc, V.R});
 
   for (Base &B : Bases)
     BaseMap[B.Decl] = &B;
diff --git a/clang/lib/AST/Interp/Record.h b/clang/lib/AST/ByteCode/Record.h
similarity index 100%
rename from clang/lib/AST/Interp/Record.h
rename to clang/lib/AST/ByteCode/Record.h
diff --git a/clang/lib/AST/Interp/Source.cpp b/clang/lib/AST/ByteCode/Source.cpp
similarity index 100%
rename from clang/lib/AST/Interp/Source.cpp
rename to clang/lib/AST/ByteCode/Source.cpp
diff --git a/clang/lib/AST/Interp/Source.h b/clang/lib/AST/ByteCode/Source.h
similarity index 100%
rename from clang/lib/AST/Interp/Source.h
rename to clang/lib/AST/ByteCode/Source.h
diff --git a/clang/lib/AST/Interp/State.cpp b/clang/lib/AST/ByteCode/State.cpp
similarity index 100%
rename from clang/lib/AST/Interp/State.cpp
rename to clang/lib/AST/ByteCode/State.cpp
diff --git a/clang/lib/AST/Interp/State.h b/clang/lib/AST/ByteCode/State.h
similarity index 100%
rename from clang/lib/AST/Interp/State.h
rename to clang/lib/AST/ByteCode/State.h
diff --git a/clang/lib/AST/CMakeLists.txt b/clang/lib/AST/CMakeLists.txt
index 70aecb781c2ff2f..041252b6830e0a7 100644
--- a/clang/lib/AST/CMakeLists.txt
+++ b/clang/lib/AST/CMakeLists.txt
@@ -11,7 +11,7 @@ set(LLVM_LINK_COMPONENTS
 # another library which depends on clangAST.
 clang_tablegen(Opcodes.inc
   -gen-clang-opcodes
-  SOURCE Interp/Opcodes.td
+  SOURCE ByteCode/Opcodes.td
   TARGET Opcodes)
 
 clang_tablegen(AttrDocTable.inc -gen-clang-attr-doc-table
@@ -64,31 +64,31 @@ add_clang_library(clangAST
   ExternalASTSource.cpp
   FormatString.cpp
   InheritViz.cpp
-  Interp/ByteCodeEmitter.cpp
-  Interp/Compiler.cpp
-  Interp/Context.cpp
-  Interp/Descriptor.cpp
-  Interp/Disasm.cpp
-  Interp/EvalEmitter.cpp
-  Interp/Frame.cpp
-  Interp/Function.cpp
-  Interp/InterpBuiltin.cpp
-  Interp/Floating.cpp
-  Interp/EvaluationResult.cpp
-  Interp/DynamicAllocator.cpp
-  Interp/Interp.cpp
-  Interp/InterpBlock.cpp
-  Interp/InterpFrame.cpp
-  Interp/InterpStack.cpp
-  Interp/InterpState.cpp
-  Interp/Pointer.cpp
-  Interp/PrimType.cpp
-  Interp/Program.cpp
-  Interp/Record.cpp
-  Interp/Source.cpp
-  Interp/State.cpp
-  Interp/MemberPointer.cpp
-  Interp/InterpShared.cpp
+  ByteCode/ByteCodeEmitter.cpp
+  ByteCode/Compiler.cpp
+  ByteCode/Context.cpp
+  ByteCode/Descriptor.cpp
+  ByteCode/Disasm.cpp
+  ByteCode/EvalEmitter.cpp
+  ByteCode/Frame.cpp
+  ByteCode/Function.cpp
+  ByteCode/InterpBuiltin.cpp
+  ByteCode/Floating.cpp
+  ByteCode/EvaluationResult.cpp
+  ByteCode/DynamicAllocator.cpp
+  ByteCode/Interp.cpp
+  ByteCode/InterpBlock.cpp
+  ByteCode/InterpFrame.cpp
+  ByteCode/InterpStack.cpp
+  ByteCode/InterpState.cpp
+  ByteCode/Pointer.cpp
+  ByteCode/PrimType.cpp
+  ByteCode/Program.cpp
+  ByteCode/Record.cpp
+  ByteCode/Source.cpp
+  ByteCode/State.cpp
+  ByteCode/MemberPointer.cpp
+  ByteCode/InterpShared.cpp
   ItaniumCXXABI.cpp
   ItaniumMangle.cpp
   JSONNodeDumper.cpp
diff --git a/clang/lib/AST/ExprConstShared.h b/clang/lib/AST/ExprConstShared.h
index 2a7088e4e371a1f..efe8ee986d29b3d 100644
--- a/clang/lib/AST/ExprConstShared.h
+++ b/clang/lib/AST/ExprConstShared.h
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 //
 // Shared functionality between the new constant expression
-// interpreter (AST/Interp/) and the current one (ExprConstant.cpp).
+// interpreter (AST/ByteCode/) and the current one (ExprConstant.cpp).
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp
index 09edbb6641650a0..7bfc63ffd81e288 100644
--- a/clang/lib/AST/ExprConstant.cpp
+++ b/clang/lib/AST/ExprConstant.cpp
@@ -32,10 +32,10 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "ByteCode/Context.h"
+#include "ByteCode/Frame.h"
+#include "ByteCode/State.h"
 #include "ExprConstShared.h"
-#include "Interp/Context.h"
-#include "Interp/Frame.h"
-#include "Interp/State.h"
 #include "clang/AST/APValue.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/ASTDiagnostic.h"
diff --git a/clang/test/AST/Interp/arrays.cpp b/clang/test/AST/ByteCode/arrays.cpp
similarity index 100%
rename from clang/test/AST/Interp/arrays.cpp
rename to clang/test/AST/ByteCode/arrays.cpp
diff --git a/clang/test/AST/Interp/atomic.c b/clang/test/AST/ByteCode/atomic.c
similarity index 100%
rename from clang/test/AST/Interp/atomic.c
rename to clang/test/AST/ByteCode/atomic.c
diff --git a/clang/test/AST/Interp/atomic.cpp b/clang/test/AST/ByteCode/atomic.cpp
similarity index 100%
rename from clang/test/AST/Interp/atomic.cpp
rename to clang/test/AST/ByteCode/atomic.cpp
diff --git a/clang/test/AST/Interp/bitfields.cpp b/clang/test/AST/ByteCode/bitfields.cpp
similarity index 100%
rename from clang/test/AST/Interp/bitfields.cpp
rename to clang/test/AST/ByteCode/bitfields.cpp
diff --git a/clang/test/AST/Interp/builtin-align-cxx.cpp b/clang/test/AST/ByteCode/builtin-align-cxx.cpp
similarity index 100%
rename from clang/test/AST/Interp/builtin-align-cxx.cpp
rename to clang/test/AST/ByteCode/builtin-align-cxx.cpp
diff --git a/clang/test/AST/Interp/builtin-constant-p.cpp b/clang/test/AST/ByteCode/builtin-constant-p.cpp
similarity index 100%
rename from clang/test/AST/Interp/builtin-constant-p.cpp
rename to clang/test/AST/ByteCode/builtin-constant-p.cpp
diff --git a/clang/test/AST/Interp/builtin-functions.cpp b/clang/test/AST/ByteCode/builtin-functions.cpp
similarity index 100%
rename from clang/test/AST/Interp/builtin-functions.cpp
rename to clang/test/AST/ByteCode/builtin-functions.cpp
diff --git a/clang/test/AST/Interp/builtins.cpp b/clang/test/AST/ByteCode/builtins.cpp
similarity index 100%
rename from clang/test/AST/Interp/builtins.cpp
rename to clang/test/AST/ByteCode/builtins.cpp
diff --git a/clang/test/AST/Interp/c.c b/clang/test/AST/ByteCode/c.c
similarity index 100%
rename from clang/test/AST/Interp/c.c
rename to clang/test/AST/ByteCode/c.c
diff --git a/clang/test/AST/Interp/c23.c b/clang/test/AST/ByteCode/c23.c
similarity index 100%
rename from clang/test/AST/Interp/c23.c
rename to clang/test/AST/ByteCode/c23.c
diff --git a/clang/test/AST/Interp/codegen.cpp b/clang/test/AST/ByteCode/codegen.cpp
similarity index 100%
rename from clang/test/AST/Interp/codegen.cpp
rename to clang/test/AST/ByteCode/codegen.cpp
diff --git a/clang/test/AST/Interp/comma.cpp b/clang/test/AST/ByteCode/comma.cpp
similarity index 100%
rename from clang/test/AST/Interp/comma.cpp
rename to clang/test/AST/ByteCode/comma.cpp
diff --git a/clang/test/AST/Interp/complex.c b/clang/test/AST/ByteCode/complex.c
similarity index 100%
rename from clang/test/AST/Interp/complex.c
rename to clang/test/AST/ByteCode/complex.c
diff --git a/clang/test/AST/Interp/complex.cpp b/clang/test/AST/ByteCode/complex.cpp
similarity index 100%
rename from clang/test/AST/Interp/complex.cpp
rename to clang/test/AST/ByteCode/complex.cpp
diff --git a/clang/test/AST/Interp/cond.cpp b/clang/test/AST/ByteCode/cond.cpp
similarity index 100%
rename from clang/test/AST/Interp/cond.cpp
rename to clang/test/AST/ByteCode/cond.cpp
diff --git a/clang/test/AST/Interp/const-eval.c b/clang/test/AST/ByteCode/const-eval.c
similarity index 100%
rename from clang/test/AST/Interp/const-eval.c
rename to clang/test/AST/ByteCode/const-eval.c
diff --git a/clang/test/AST/Interp/const-fpfeatures.cpp b/clang/test/AST/ByteCode/const-fpfeatures.cpp
similarity index 100%
rename from clang/test/AST/Interp/const-fpfeatures.cpp
rename to clang/test/AST/ByteCode/const-fpfeatures.cpp
diff --git a/clang/test/AST/Interp/const-temporaries.cpp b/clang/test/AST/ByteCode/const-temporaries.cpp
similarity index 100%
rename from clang/test/AST/Interp/const-temporaries.cpp
rename to clang/test/AST/ByteCode/const-temporaries.cpp
diff --git a/clang/test/AST/Interp/constexpr-frame-describe.cpp b/clang/test/AST/ByteCode/constexpr-frame-describe.cpp
similarity index 100%
rename from clang/test/AST/Interp/constexpr-frame-describe.cpp
rename to clang/test/AST/ByteCode/constexpr-frame-describe.cpp
diff --git a/clang/test/AST/Interp/constexpr-nqueens.cpp b/clang/test/AST/ByteCode/constexpr-nqueens.cpp
similarity index 100%
rename from clang/test/AST/Interp/constexpr-nqueens.cpp
rename to clang/test/AST/ByteCode/constexpr-nqueens.cpp
diff --git a/clang/test/AST/Interp/constexpr-subobj-initialization.cpp b/clang/test/AST/ByteCode/constexpr-subobj-initialization.cpp
similarity index 100%
rename from clang/test/AST/Interp/constexpr-subobj-initialization.cpp
rename to clang/test/AST/ByteCode/constexpr-subobj-initialization.cpp
diff --git a/clang/test/AST/Interp/crash-GH49103-2.cpp b/clang/test/AST/ByteCode/crash-GH49103-2.cpp
similarity index 100%
rename from clang/test/AST/Interp/crash-GH49103-2.cpp
rename to clang/test/AST/ByteCode/crash-GH49103-2.cpp
diff --git a/clang/test/AST/Interp/cxx03.cpp b/clang/test/AST/ByteCode/cxx03.cpp
similarity index 100%
rename from clang/test/AST/Interp/cxx03.cpp
rename to clang/test/AST/ByteCode/cxx03.cpp
diff --git a/clang/test/AST/Interp/cxx11.cpp b/clang/test/AST/ByteCode/cxx11.cpp
similarity index 100%
rename from clang/test/AST/Interp/cxx11.cpp
rename to clang/test/AST/ByteCode/cxx11.cpp
diff --git a/clang/test/AST/Interp/cxx17.cpp b/clang/test/AST/ByteCode/cxx17.cpp
similarity index 100%
rename from clang/test/AST/Interp/cxx17.cpp
rename to clang/test/AST/ByteCode/cxx17.cpp
diff --git a/clang/test/AST/Interp/cxx20.cpp b/clang/test/AST/ByteCode/cxx20.cpp
similarity index 100%
rename from clang/test/AST/Interp/cxx20.cpp
rename to clang/test/AST/ByteCode/cxx20.cpp
diff --git a/clang/test/AST/Interp/cxx23.cpp b/clang/test/AST/ByteCode/cxx23.cpp
similarity index 100%
rename from clang/test/AST/Interp/cxx23.cpp
rename to clang/test/AST/ByteCode/cxx23.cpp
diff --git a/clang/test/AST/Interp/cxx26.cpp b/clang/test/AST/ByteCode/cxx26.cpp
similarity index 100%
rename from clang/test/AST/Interp/cxx26.cpp
rename to clang/test/AST/ByteCode/cxx26.cpp
diff --git a/clang/test/AST/Interp/cxx2a.cpp b/clang/test/AST/ByteCode/cxx2a.cpp
similarity index 100%
rename from clang/test/AST/Interp/cxx2a.cpp
rename to clang/test/AST/ByteCode/cxx2a.cpp
diff --git a/clang/test/AST/Interp/cxx98.cpp b/clang/test/AST/ByteCode/cxx98.cpp
similarity index 100%
rename from clang/test/AST/Interp/cxx98.cpp
rename to clang/test/AST/ByteCode/cxx98.cpp
diff --git a/clang/test/AST/Interp/depth-limit.cpp b/clang/test/AST/ByteCode/depth-limit.cpp
similarity index 100%
rename from clang/test/AST/Interp/depth-limit.cpp
rename to clang/test/AST/ByteCode/depth-limit.cpp
diff --git a/clang/test/AST/Interp/depth-limit2.cpp b/clang/test/AST/ByteCode/depth-limit2.cpp
similarity index 100%
rename from clang/test/AST/Interp/depth-limit2.cpp
rename to clang/test/AST/ByteCode/depth-limit2.cpp
diff --git a/clang/test/AST/Interp/enums-targets.cpp b/clang/test/AST/ByteCode/enums-targets.cpp
similarity index 100%
rename from clang/test/AST/Interp/enums-targets.cpp
rename to clang/test/AST/ByteCode/enums-targets.cpp
diff --git a/clang/test/AST/Interp/enums.cpp b/clang/test/AST/ByteCode/enums.cpp
similarity index 100%
rename from clang/test/AST/Interp/enums.cpp
rename to clang/test/AST/ByteCode/enums.cpp
diff --git a/clang/test/AST/Interp/eval-order.cpp b/clang/test/AST/ByteCode/eval-order.cpp
similarity index 100%
rename from clang/test/AST/Interp/eval-order.cpp
rename to clang/test/AST/ByteCode/eval-order.cpp
diff --git a/clang/test/AST/Interp/floats.cpp b/clang/test/AST/ByteCode/floats.cpp
similarity index 100%
rename from clang/test/AST/Interp/floats.cpp
rename to clang/test/AST/ByteCode/floats.cpp
diff --git a/clang/test/AST/Interp/functions.cpp b/clang/test/AST/ByteCode/functions.cpp
similarity index 100%
rename from clang/test/AST/Interp/functions.cpp
rename to clang/test/AST/ByteCode/functions.cpp
diff --git a/clang/test/AST/Interp/hlsl.hlsl b/clang/test/AST/ByteCode/hlsl.hlsl
similarity index 100%
rename from clang/test/AST/Interp/hlsl.hlsl
rename to clang/test/AST/ByteCode/hlsl.hlsl
diff --git a/clang/test/AST/Interp/if.cpp b/clang/test/AST/ByteCode/if.cpp
similarity index 100%
rename from clang/test/AST/Interp/if.cpp
rename to clang/test/AST/ByteCode/if.cpp
diff --git a/clang/test/AST/Interp/intap.cpp b/clang/test/AST/ByteCode/intap.cpp
similarity index 100%
rename from clang/test/AST/Interp/intap.cpp
rename to clang/test/AST/ByteCode/intap.cpp
diff --git a/clang/test/AST/Interp/invalid.cpp b/clang/test/AST/ByteCode/invalid.cpp
similarity index 100%
rename from clang/test/AST/Interp/invalid.cpp
rename to clang/test/AST/ByteCode/invalid.cpp
diff --git a/clang/test/AST/Interp/lambda.cpp b/clang/test/AST/ByteCode/lambda.cpp
similarity index 100%
rename from clang/test/AST/Interp/lambda.cpp
rename to clang/test/AST/ByteCode/lambda.cpp
diff --git a/clang/test/AST/Interp/lifetimes.cpp b/clang/test/AST/ByteCode/lifetimes.cpp
similarity index 100%
rename from clang/test/AST/Interp/lifetimes.cpp
rename to clang/test/AST/ByteCode/lifetimes.cpp
diff --git a/clang/test/AST/Interp/literals.cpp b/clang/test/AST/ByteCode/literals.cpp
similarity index 100%
rename from clang/test/AST/Interp/literals.cpp
rename to clang/test/AST/ByteCode/literals.cpp
diff --git a/clang/test/AST/Interp/loops.cpp b/clang/test/AST/ByteCode/loops.cpp
similarity index 100%
rename from clang/test/AST/Interp/loops.cpp
rename to clang/test/AST/ByteCode/loops.cpp
diff --git a/clang/test/AST/Interp/memberpointers.cpp b/clang/test/AST/ByteCode/memberpointers.cpp
similarity index 100%
rename from clang/test/AST/Interp/memberpointers.cpp
rename to clang/test/AST/ByteCode/memberpointers.cpp
diff --git a/clang/test/AST/Interp/ms.cpp b/clang/test/AST/ByteCode/ms.cpp
similarity index 100%
rename from clang/test/AST/Interp/ms.cpp
rename to clang/test/AST/ByteCode/ms.cpp
diff --git a/clang/test/AST/Interp/mutable.cpp b/clang/test/AST/ByteCode/mutable.cpp
similarity index 100%
rename from clang/test/AST/Interp/mutable.cpp
rename to clang/test/AST/ByteCode/mutable.cpp
diff --git a/clang/test/AST/Interp/new-delete.cpp b/clang/test/AST/ByteCode/new-delete.cpp
similarity index 100%
rename from clang/test/AST/Interp/new-delete.cpp
rename to clang/test/AST/ByteCode/new-delete.cpp
diff --git a/clang/test/AST/Interp/nullable.cpp b/clang/test/AST/ByteCode/nullable.cpp
similarity index 100%
rename from clang/test/AST/Interp/nullable.cpp
rename to clang/test/AST/ByteCode/nullable.cpp
diff --git a/clang/test/AST/Interp/objc.mm b/clang/test/AST/ByteCode/objc.mm
similarity index 100%
rename from clang/test/AST/Interp/objc.mm
rename to clang/test/AST/ByteCode/objc.mm
diff --git a/clang/test/AST/Interp/opencl.cl b/clang/test/AST/ByteCode/opencl.cl
similarity index 100%
rename from clang/test/AST/Interp/opencl.cl
rename to clang/test/AST/ByteCode/opencl.cl
diff --git a/clang/test/AST/Interp/pointer-addition.c b/clang/test/AST/ByteCode/pointer-addition.c
similarity index 100%
rename from clang/test/AST/Interp/pointer-addition.c
rename to clang/test/AST/ByteCode/pointer-addition.c
diff --git a/clang/test/AST/Interp/records.cpp b/clang/test/AST/ByteCode/records.cpp
similarity index 100%
rename from clang/test/AST/Interp/records.cpp
rename to clang/test/AST/ByteCode/records.cpp
diff --git a/clang/test/AST/Interp/references.cpp b/clang/test/AST/ByteCode/references.cpp
similarity index 100%
rename from clang/test/AST/Interp/references.cpp
rename to clang/test/AST/ByteCode/references.cpp
diff --git a/clang/test/AST/Interp/shifts.cpp b/clang/test/AST/ByteCode/shifts.cpp
similarity index 100%
rename from clang/test/AST/Interp/shifts.cpp
rename to clang/test/AST/ByteCode/shifts.cpp
diff --git a/clang/test/AST/Interp/spaceship.cpp b/clang/test/AST/ByteCode/spaceship.cpp
similarity index 100%
rename from clang/test/AST/Interp/spaceship.cpp
rename to clang/test/AST/ByteCode/spaceship.cpp
diff --git a/clang/test/AST/Interp/switch.cpp b/clang/test/AST/ByteCode/switch.cpp
similarity index 100%
rename from clang/test/AST/Interp/switch.cpp
rename to clang/test/AST/ByteCode/switch.cpp
diff --git a/clang/test/AST/Interp/sycl.cpp b/clang/test/AST/ByteCode/sycl.cpp
similarity index 100%
rename from clang/test/AST/Interp/sycl.cpp
rename to clang/test/AST/ByteCode/sycl.cpp
diff --git a/clang/test/AST/Interp/unions.cpp b/clang/test/AST/ByteCode/unions.cpp
similarity index 100%
rename from clang/test/AST/Interp/unions.cpp
rename to clang/test/AST/ByteCode/unions.cpp
diff --git a/clang/test/AST/Interp/vectors.cpp b/clang/test/AST/ByteCode/vectors.cpp
similarity index 100%
rename from clang/test/AST/Interp/vectors.cpp
rename to clang/test/AST/ByteCode/vectors.cpp
diff --git a/clang/test/AST/Interp/weak.cpp b/clang/test/AST/ByteCode/weak.cpp
similarity index 100%
rename from clang/test/AST/Interp/weak.cpp
rename to clang/test/AST/ByteCode/weak.cpp
diff --git a/clang/unittests/AST/Interp/CMakeLists.txt b/clang/unittests/AST/ByteCode/CMakeLists.txt
similarity index 100%
rename from clang/unittests/AST/Interp/CMakeLists.txt
rename to clang/unittests/AST/ByteCode/CMakeLists.txt
diff --git a/clang/unittests/AST/Interp/Descriptor.cpp b/clang/unittests/AST/ByteCode/Descriptor.cpp
similarity index 95%
rename from clang/unittests/AST/Interp/Descriptor.cpp
rename to clang/unittests/AST/ByteCode/Descriptor.cpp
index abcf1bb0714cc8c..9e09c7f15ac9a28 100644
--- a/clang/unittests/AST/Interp/Descriptor.cpp
+++ b/clang/unittests/AST/ByteCode/Descriptor.cpp
@@ -1,6 +1,6 @@
-#include "../../../lib/AST/Interp/Descriptor.h"
-#include "../../../lib/AST/Interp/Context.h"
-#include "../../../lib/AST/Interp/Program.h"
+#include "../../../lib/AST/ByteCode/Descriptor.h"
+#include "../../../lib/AST/ByteCode/Context.h"
+#include "../../../lib/AST/ByteCode/Program.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/Decl.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
@@ -15,17 +15,17 @@ using namespace clang::ast_matchers;
 /// Inspect generated Descriptors as well as the pointers we create.
 ///
 TEST(Descriptor, Primitives) {
-  constexpr char Code[] =
-      "struct A { bool a; bool b; };\n"
-      "struct S {\n"
-      "  float f;\n"
-      "  char s[4];\n"
-      "  A a[3];\n"
-      "  short l[3][3];\n"
-      "  int EmptyA[0];\n"
-      "};\n"
-      "constexpr S d = {0.0, \"foo\", {{true, false}, {false, true}, {false, false}},\n"
-      "  {{1, 2, 3}, {4, 5, 6}, {7, 8, 9}}, {}};\n";
+  constexpr char Code[] = "struct A { bool a; bool b; };\n"
+                          "struct S {\n"
+                          "  float f;\n"
+                          "  char s[4];\n"
+                          "  A a[3];\n"
+                          "  short l[3][3];\n"
+                          "  int EmptyA[0];\n"
+                          "};\n"
+                          "constexpr S d = {0.0, \"foo\", {{true, false}, "
+                          "{false, true}, {false, false}},\n"
+                          "  {{1, 2, 3}, {4, 5, 6}, {7, 8, 9}}, {}};\n";
 
   auto AST = tooling::buildASTFromCodeWithArgs(
       Code, {"-fexperimental-new-constant-interpreter"});
diff --git a/clang/unittests/AST/Interp/toAPValue.cpp b/clang/unittests/AST/ByteCode/toAPValue.cpp
similarity index 98%
rename from clang/unittests/AST/Interp/toAPValue.cpp
rename to clang/unittests/AST/ByteCode/toAPValue.cpp
index 5ec607a8243492d..3f141878fb9594d 100644
--- a/clang/unittests/AST/Interp/toAPValue.cpp
+++ b/clang/unittests/AST/ByteCode/toAPValue.cpp
@@ -1,6 +1,6 @@
-#include "../../../lib/AST/Interp/Context.h"
-#include "../../../lib/AST/Interp/Descriptor.h"
-#include "../../../lib/AST/Interp/Program.h"
+#include "../../../lib/AST/ByteCode/Context.h"
+#include "../../../lib/AST/ByteCode/Descriptor.h"
+#include "../../../lib/AST/ByteCode/Program.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/Decl.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
diff --git a/clang/unittests/AST/CMakeLists.txt b/clang/unittests/AST/CMakeLists.txt
index 51245599736cfbb..40d2e1ff77a6015 100644
--- a/clang/unittests/AST/CMakeLists.txt
+++ b/clang/unittests/AST/CMakeLists.txt
@@ -5,7 +5,7 @@ set(LLVM_LINK_COMPONENTS
   )
 
 
-add_subdirectory(Interp)
+add_subdirectory(ByteCode)
 
 add_clang_unittest(ASTTests
   ASTContextParentMapTest.cpp

From 2fe59d5259ec921668d87d111be55db0b047aa68 Mon Sep 17 00:00:00 2001
From: Robin Caloudis <robin.caloudis@gmx.de>
Date: Fri, 16 Aug 2024 17:18:49 +0200
Subject: [PATCH 181/441] [libc++][math] Fix acceptance of convertible types in
 `std::isnan()` and `std::isinf()` (#98952)

Following up on https://github.com/llvm/llvm-project/pull/98841.

Changes:
- Properly test convertible types for `std::isnan()` and `std::inf()`
- Tighten conditional in `cmath.pass.cpp` (Find insights on `_LIBCPP_PREFERRED_OVERLOAD` below)
- Tighten preprocessor guard in `traits.h`

Insights into why `_LIBCPP_PREFERRED_OVERLOAD` is needed:

(i) When libc++ is layered on top of glibc on Linux, glibc's `math.h` is
    included. When compiling with `-std=c++03`, this header brings the
    function declaration of `isinf(double)` [1] and `isnan(double)` [2]
    into scope. This differs from the C99 Standard as only the macros
    `#define isnan(arg)` and `#define isinf(arg)` are expected.

    Therefore, libc++ needs to respect the presense of the `double` overload
    and cannot redefine it as it will conflict with the declaration already
    in scope. For `-std=c++11` and beyond this issue is fixed, as glibc
    guards both the `isinf` and `isnan` by preprocessor macros.

(ii) When libc++ is layered on top of Bionic's libc, `math.h` exposes a
     function prototype for `isinf(double)` with return type `int`. This
     function prototype in Bionic's libc is not guarded by any preprocessor
     macros [3].

`_LIBCPP_PREFERRED_OVERLOAD` specifies that a given overload is a better match
than an otherwise equally good function declaration. This is implemented in
modern versions of Clang via `__attribute__((__enable_if__))`, and not elsewhere.
See [4] for details. We use `_LIBCPP_PREFERRED_OVERLOAD` to define overloads in
the global namespace that displace the overloads provided by the C
libraries mentioned above.

[1]: https://github.com/bminor/glibc/blob/fe9408087583fd7a6f61bb0dbcf2fd4e83186afa/math/bits/mathcalls.h#L185-L194
[2]: https://github.com/bminor/glibc/blob/fe9408087583fd7a6f61bb0dbcf2fd4e83186afa/math/bits/mathcalls.h#L222-L231
[3]: https://cs.android.com/android/platform/superproject/+/master:bionic/libc/include/math.h;l=322-323;drc=master?hl=fr-BE%22https:%2F%2Fsupport.google.com%2Fmerchants%2Fanswer%2F188494%5C%22%22https:%2F%2Fsupport.google.com%2Fmerchants%2Fanswer%2F188494%5C%22
[4]: https://github.com/llvm/llvm-project/commit/5fd17ab1b093f6b59aabb27f6c2c2278e65c2707
---
 libcxx/include/__math/traits.h                | 20 +++++++++++--------
 .../test/std/numerics/c.math/cmath.pass.cpp   | 20 ++++++++++---------
 .../test/std/numerics/c.math/isinf.pass.cpp   | 12 +++++++++++
 .../test/std/numerics/c.math/isnan.pass.cpp   | 12 +++++++++++
 4 files changed, 47 insertions(+), 17 deletions(-)

diff --git a/libcxx/include/__math/traits.h b/libcxx/include/__math/traits.h
index 27ec52ecef022e9..35c283cc9e21cef 100644
--- a/libcxx/include/__math/traits.h
+++ b/libcxx/include/__math/traits.h
@@ -79,20 +79,22 @@ _LIBCPP_NODISCARD _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI bool isinf
   return false;
 }
 
-#ifdef _LIBCPP_PREFERRED_OVERLOAD
 _LIBCPP_NODISCARD inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI bool isinf(float __x) _NOEXCEPT {
   return __builtin_isinf(__x);
 }
 
-_LIBCPP_NODISCARD inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI _LIBCPP_PREFERRED_OVERLOAD bool
-isinf(double __x) _NOEXCEPT {
+_LIBCPP_NODISCARD inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI
+#ifdef _LIBCPP_PREFERRED_OVERLOAD
+_LIBCPP_PREFERRED_OVERLOAD
+#endif
+    bool
+    isinf(double __x) _NOEXCEPT {
   return __builtin_isinf(__x);
 }
 
 _LIBCPP_NODISCARD inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI bool isinf(long double __x) _NOEXCEPT {
   return __builtin_isinf(__x);
 }
-#endif
 
 // isnan
 
@@ -106,20 +108,22 @@ _LIBCPP_NODISCARD _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI bool isnan
   return false;
 }
 
-#ifdef _LIBCPP_PREFERRED_OVERLOAD
 _LIBCPP_NODISCARD inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI bool isnan(float __x) _NOEXCEPT {
   return __builtin_isnan(__x);
 }
 
-_LIBCPP_NODISCARD inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI _LIBCPP_PREFERRED_OVERLOAD bool
-isnan(double __x) _NOEXCEPT {
+_LIBCPP_NODISCARD inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI
+#ifdef _LIBCPP_PREFERRED_OVERLOAD
+_LIBCPP_PREFERRED_OVERLOAD
+#endif
+    bool
+    isnan(double __x) _NOEXCEPT {
   return __builtin_isnan(__x);
 }
 
 _LIBCPP_NODISCARD inline _LIBCPP_CONSTEXPR_SINCE_CXX23 _LIBCPP_HIDE_FROM_ABI bool isnan(long double __x) _NOEXCEPT {
   return __builtin_isnan(__x);
 }
-#endif
 
 // isnormal
 
diff --git a/libcxx/test/std/numerics/c.math/cmath.pass.cpp b/libcxx/test/std/numerics/c.math/cmath.pass.cpp
index 6028aa5a2110342..34c30fb998f4746 100644
--- a/libcxx/test/std/numerics/c.math/cmath.pass.cpp
+++ b/libcxx/test/std/numerics/c.math/cmath.pass.cpp
@@ -708,15 +708,16 @@ void test_isinf()
     static_assert((std::is_same<decltype(std::isinf((float)0)), bool>::value), "");
 
     typedef decltype(std::isinf((double)0)) DoubleRetType;
-#if !defined(__linux__) || defined(__clang__)
-    static_assert((std::is_same<DoubleRetType, bool>::value), "");
-#else
+#if defined(__GLIBC__) && TEST_STD_VER == 03 && defined(TEST_COMPILER_CLANG)
     // GLIBC < 2.23 defines 'isinf(double)' with a return type of 'int' in
     // all C++ dialects. The test should tolerate this when libc++ can't work
-    // around it.
+    // around it via `_LIBCPP_PREFERRED_OVERLOAD`, which is only available
+    // in modern versions of Clang, and not elsewhere.
     // See: https://sourceware.org/bugzilla/show_bug.cgi?id=19439
     static_assert((std::is_same<DoubleRetType, bool>::value
                 || std::is_same<DoubleRetType, int>::value), "");
+#else
+    static_assert((std::is_same<DoubleRetType, bool>::value), "");
 #endif
 
     static_assert((std::is_same<decltype(std::isinf(0)), bool>::value), "");
@@ -794,15 +795,16 @@ void test_isnan()
     static_assert((std::is_same<decltype(std::isnan((float)0)), bool>::value), "");
 
     typedef decltype(std::isnan((double)0)) DoubleRetType;
-#if !defined(__linux__) || defined(__clang__)
-    static_assert((std::is_same<DoubleRetType, bool>::value), "");
-#else
-    // GLIBC < 2.23 defines 'isinf(double)' with a return type of 'int' in
+#if defined(__GLIBC__) && TEST_STD_VER == 03 && defined(TEST_COMPILER_CLANG)
+    // GLIBC < 2.23 defines 'isnan(double)' with a return type of 'int' in
     // all C++ dialects. The test should tolerate this when libc++ can't work
-    // around it.
+    // around it via `_LIBCPP_PREFERRED_OVERLOAD`, which is only available
+    // in modern versions of Clang, and not elsewhere.
     // See: https://sourceware.org/bugzilla/show_bug.cgi?id=19439
     static_assert((std::is_same<DoubleRetType, bool>::value
                 || std::is_same<DoubleRetType, int>::value), "");
+#else
+    static_assert((std::is_same<DoubleRetType, bool>::value), "");
 #endif
 
     static_assert((std::is_same<decltype(std::isnan(0)), bool>::value), "");
diff --git a/libcxx/test/std/numerics/c.math/isinf.pass.cpp b/libcxx/test/std/numerics/c.math/isinf.pass.cpp
index e935b53187fe6ce..e5169e8056c2ea1 100644
--- a/libcxx/test/std/numerics/c.math/isinf.pass.cpp
+++ b/libcxx/test/std/numerics/c.math/isinf.pass.cpp
@@ -62,9 +62,21 @@ struct TestInt {
   }
 };
 
+template <typename T>
+struct ConvertibleTo {
+  operator T() const { return T(); }
+};
+
 int main(int, char**) {
   types::for_each(types::floating_point_types(), TestFloat());
   types::for_each(types::integral_types(), TestInt());
 
+  // Make sure we can call `std::isinf` with convertible types
+  {
+    assert(!std::isinf(ConvertibleTo<float>()));
+    assert(!std::isinf(ConvertibleTo<double>()));
+    assert(!std::isinf(ConvertibleTo<long double>()));
+  }
+
   return 0;
 }
diff --git a/libcxx/test/std/numerics/c.math/isnan.pass.cpp b/libcxx/test/std/numerics/c.math/isnan.pass.cpp
index fffb12464586292..e4ccab1243e5678 100644
--- a/libcxx/test/std/numerics/c.math/isnan.pass.cpp
+++ b/libcxx/test/std/numerics/c.math/isnan.pass.cpp
@@ -62,9 +62,21 @@ struct TestInt {
   }
 };
 
+template <typename T>
+struct ConvertibleTo {
+  operator T() const { return T(); }
+};
+
 int main(int, char**) {
   types::for_each(types::floating_point_types(), TestFloat());
   types::for_each(types::integral_types(), TestInt());
 
+  // Make sure we can call `std::isnan` with convertible types
+  {
+    assert(!std::isnan(ConvertibleTo<float>()));
+    assert(!std::isnan(ConvertibleTo<double>()));
+    assert(!std::isnan(ConvertibleTo<long double>()));
+  }
+
   return 0;
 }

From c4bf949171a72383d5ba4d2b587d4cc496aacebb Mon Sep 17 00:00:00 2001
From: Hsiangkai Wang <hsiangkai.wang@arm.com>
Date: Fri, 16 Aug 2024 16:22:02 +0100
Subject: [PATCH 182/441] [mlir][linalg] Implement TilingInterface for winograd
 operators (#96184)

In order to support arbitrary size input data of conv2d, implement
TilingInterface for winograd operations. Before converting winograd
operations into nested loops with matrix multiply, tile the input of
conv2d into the supported size first.

Add a transform operation structured.decompose_winograd_op to decompose
winograd operations. Before applying the transform op, use
tile_using_for to tile the input data into supported size. The test case
shows how to tile and decompose winograd operations.
---
 .../mlir/Dialect/Linalg/IR/LinalgOps.td       | 141 ++++++-
 .../Linalg/TransformOps/LinalgTransformOps.td |  37 ++
 .../Dialect/Linalg/Transforms/Transforms.h    |  57 +++
 mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp      | 397 ++++++++++++++++--
 .../TransformOps/LinalgTransformOps.cpp       |  41 ++
 .../Linalg/Transforms/WinogradConv2D.cpp      |  25 +-
 .../transform-tile-and-winograd-rewrite.mlir  | 292 +++++++++++++
 .../Linalg/transform-tile-winograd.mlir       | 380 +++++++++++++++++
 8 files changed, 1330 insertions(+), 40 deletions(-)
 create mode 100644 mlir/test/Dialect/Linalg/transform-tile-and-winograd-rewrite.mlir
 create mode 100644 mlir/test/Dialect/Linalg/transform-tile-winograd.mlir

diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgOps.td b/mlir/include/mlir/Dialect/Linalg/IR/LinalgOps.td
index a9007c8db3078e9..5b6a90f806bedd9 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgOps.td
+++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgOps.td
@@ -154,8 +154,13 @@ def Linalg_SoftmaxOp : Linalg_Op<"softmax",
   let hasVerifier = 1;
 }
 
-def Linalg_WinogradFilterTransformOp :
-    Linalg_Op<"winograd_filter_transform", [AllElementTypesMatch<["filter", "output"]>]> {
+def Linalg_WinogradFilterTransformOp : Linalg_Op<"winograd_filter_transform",
+    [AllElementTypesMatch<["filter", "output"]>,
+     DeclareOpInterfaceMethods<TilingInterface,
+      ["getIterationDomain",
+       "getLoopIteratorTypes",
+       "getResultTilePosition",
+       "getTiledImplementation"]>]> {
   let summary = "Winograd filter transform operator";
   let description = [{
     Winograd Conv2D algorithm will convert linalg Conv2D operator into batched
@@ -190,11 +195,42 @@ def Linalg_WinogradFilterTransformOp :
     `outs` `(` $output `:` type($output) `)`
     `->` type($result)
   }];
+  let extraClassDeclaration = [{
+    ShapedType getFilterOperandType() {
+      return cast<ShapedType>(getFilter().getType());
+    }
+    ShapedType getOutputOperandType() {
+      return cast<ShapedType>(getOutput().getType());
+    }
+    int64_t getFilterOperandRank() {
+      return getFilterOperandType().getRank();
+    }
+    int64_t getOutputOperandRank() {
+      return getOutputOperandType().getRank();
+    }
+    int64_t getFilterFDim() {
+      return 0;
+    }
+    int64_t getFilterHDim() {
+      return 1;
+    }
+    int64_t getFilterWDim() {
+      return 2;
+    }
+    int64_t getFilterCDim() {
+      return 3;
+    }
+  }];
   let hasVerifier = 1;
 }
 
-def Linalg_WinogradInputTransformOp :
-    Linalg_Op<"winograd_input_transform", [AllElementTypesMatch<["input", "output"]>]> {
+def Linalg_WinogradInputTransformOp : Linalg_Op<"winograd_input_transform",
+    [AllElementTypesMatch<["input", "output"]>,
+     DeclareOpInterfaceMethods<TilingInterface,
+      ["getIterationDomain",
+       "getLoopIteratorTypes",
+       "getResultTilePosition",
+       "getTiledImplementation"]>]> {
   let summary = "Winograd input transform operator";
   let description = [{
     Winograd Conv2D algorithm will convert linalg Conv2D operator into batched
@@ -229,11 +265,60 @@ def Linalg_WinogradInputTransformOp :
     `outs` `(` $output `:` type($output) `)`
     `->` type($result)
   }];
+  let extraClassDeclaration = [{
+    ShapedType getInputOperandType() {
+      return cast<ShapedType>(getInput().getType());
+    }
+    ShapedType getOutputOperandType() {
+      return cast<ShapedType>(getOutput().getType());
+    }
+    int64_t getInputOperandRank() {
+      return getInputOperandType().getRank();
+    }
+    int64_t getOutputOperandRank() {
+      return getOutputOperandType().getRank();
+    }
+    int64_t getInputNDim() {
+      return 0;
+    }
+    int64_t getInputHDim() {
+      return 1;
+    }
+    int64_t getInputWDim() {
+      return 2;
+    }
+    int64_t getInputCDim() {
+      return 3;
+    }
+    int64_t getOutputAlphaHDim() {
+      return 0;
+    }
+    int64_t getOutputAlphaWDim() {
+      return 1;
+    }
+    int64_t getOutputTileHDim() {
+      return 2;
+    }
+    int64_t getOutputTileWDim() {
+      return 3;
+    }
+    int64_t getOutputNDim() {
+      return 4;
+    }
+    int64_t getOutputCDim() {
+      return 5;
+    }
+  }];
   let hasVerifier = 1;
 }
 
-def Linalg_WinogradOutputTransformOp :
-    Linalg_Op<"winograd_output_transform", [AllElementTypesMatch<["value", "output"]>]> {
+def Linalg_WinogradOutputTransformOp : Linalg_Op<"winograd_output_transform",
+    [AllElementTypesMatch<["value", "output"]>,
+     DeclareOpInterfaceMethods<TilingInterface,
+      ["getIterationDomain",
+       "getLoopIteratorTypes",
+       "getResultTilePosition",
+       "getTiledImplementation"]>]> {
   let summary = "Winograd output transform operator";
   let description = [{
     Winograd Conv2D algorithm will convert linalg Conv2D operator into batched
@@ -268,6 +353,50 @@ def Linalg_WinogradOutputTransformOp :
     `outs` `(` $output `:` type($output) `)`
     `->` type($result)
   }];
+  let extraClassDeclaration = [{
+    ShapedType getValueOperandType() {
+      return cast<ShapedType>(getValue().getType());
+    }
+    ShapedType getOutputOperandType() {
+      return cast<ShapedType>(getOutput().getType());
+    }
+    int64_t getValueOperandRank() {
+      return getValueOperandType().getRank();
+    }
+    int64_t getOutputOperandRank() {
+      return getOutputOperandType().getRank();
+    }
+    int64_t getValueAlphaHDim() {
+      return 0;
+    }
+    int64_t getValueAlphaWDim() {
+      return 1;
+    }
+    int64_t getValueTileHDim() {
+      return 2;
+    }
+    int64_t getValueTileWDim() {
+      return 3;
+    }
+    int64_t getValueNDim() {
+      return 4;
+    }
+    int64_t getValueFDim() {
+      return 5;
+    }
+    int64_t getOutputNDim() {
+      return 0;
+    }
+    int64_t getOutputHDim() {
+      return 1;
+    }
+    int64_t getOutputWDim() {
+      return 2;
+    }
+    int64_t getOutputFDim() {
+      return 3;
+    }
+  }];
   let hasVerifier = 1;
 }
 
diff --git a/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td b/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td
index ecc86999006db64..106f0d79d9792d6 100644
--- a/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td
+++ b/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td
@@ -2697,4 +2697,41 @@ def WinogradConv2DOp : Op<Transform_Dialect,
   }];
 }
 
+def DecomposeWinogradOp : Op<Transform_Dialect,
+    "structured.decompose_winograd_op",
+    [FunctionalStyleTransformOpTrait, MemoryEffectsOpInterface,
+     TransformOpInterface, TransformEachOpTrait,
+     ReportTrackingListenerFailuresOpTrait]> {
+  let description = [{
+    Decompose winograd operations. It will convert filter, input and output
+    transform operations into a combination of scf, tensor, and linalg
+    equivalent operations. Before applying this transform operations, users
+    need to tile winograd transform operations into supported sizes.
+
+    #### Return modes:
+
+    This operation fails if `target` is unsupported. Otherwise, the operation
+    succeeds and returns a handle of the sequence that replaces the original
+    operations.
+  }];
+
+  let arguments = (ins TransformHandleTypeInterface:$target);
+  let results = (outs TransformHandleTypeInterface:$transformed);
+
+  let assemblyFormat =
+    "$target attr-dict `:` functional-type($target, results)";
+
+  let builders = [
+    OpBuilder<(ins "Value":$target)>
+  ];
+
+  let extraClassDeclaration = [{
+    ::mlir::DiagnosedSilenceableFailure applyToOne(
+        ::mlir::transform::TransformRewriter &rewriter,
+        ::mlir::Operation *target,
+        ::mlir::transform::ApplyToEachResultList &results,
+        ::mlir::transform::TransformState &state);
+  }];
+}
+
 #endif // LINALG_TRANSFORM_OPS
diff --git a/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h b/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
index 477ef7bfafb181f..861e14d22d96251 100644
--- a/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
+++ b/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
@@ -1316,6 +1316,63 @@ FailureOr<Operation *> winogradConv2D(RewriterBase &rewriter,
                                       linalg::Conv2DNhwcFhwcOp op, int64_t m,
                                       int64_t r);
 
+/// Rewrite linalg.winograd_filter_transform. The data layout of the filter is
+/// FHWC. The transformation matrix is 2-dimension. We need to extract H x W
+/// from FHWC first. We generate 2 levels of loops to iterate on F and C. After
+/// the rewriting, we get
+///
+/// scf.for %f = lo_f to hi_f step 1
+///   scf.for %c = lo_c to hi_c step 1
+///     %extracted = extract filter<h x w> from filter<f x h x w x c>
+///     %ret = linalg.matmul G, %extracted
+///     %ret = linalg.matmul %ret, GT
+///     %inserted = insert %ret into filter<h x w x c x f>
+FailureOr<Operation *>
+decomposeWinogradFilterTransformOp(RewriterBase &rewriter,
+                                   linalg::WinogradFilterTransformOp op);
+
+/// Rewrite linalg.winograd_input_transform. The data layout of the input is
+/// NHWC. The transformation matrix is 2-dimension. We need to extract H x W
+/// from NHWC first. We generate 4 levels of loops to iterate on N, C, tileH,
+/// and tileW. After the rewriting, we get
+///
+/// scf.for %h = 0 to tileH step 1
+///   scf.for %w = 0 to tileW step 1
+///     scf.for %n = 0 to N step 1
+///       scf.for %c = 0 to C step 1
+///         %extracted = extract %extracted<alphaH x alphaW> from
+///                              %input<N x H x W x C>
+///                              at [%n, (%h x m), (%w x m), %c]
+///         %ret = linalg.matmul BT, %extracted
+///         %ret = linalg.matmul %ret, B
+///         %inserted = insert %ret<alphaH x alphaW> into
+///                            %output<alphaH x alphaW x tileH x tileW x N x C>
+///                            at [0, 0, %h, %w, %n, %c]
+FailureOr<Operation *>
+decomposeWinogradInputTransformOp(RewriterBase &rewriter,
+                                  linalg::WinogradInputTransformOp op);
+
+/// Rewrite linalg.winograd_output_transform. The data layout of the output is
+/// HWNF. The transformation matrix is 2-dimension. We need to extract H x W
+/// from HWNF first. We generate 4 levels of loops to iterate on N, F, tileH,
+/// and tileW. After the transformation, we get
+///
+/// scf.for %h = 0 to tileH step 1
+///   scf.for %w = 0 to tileW step 1
+///     scf.for %n = 0 to N step 1
+///       scf.for %f = 0 to F step 1
+///         %extracted = extract %extracted<alphaH x alphaW> from
+///                              %input<alphaH x alphaW x tileH x tileW x N x F>
+///                              at [0, 0, %h, %w, %n, %f]
+///         %ret = linalg.matmul AT, %extracted
+///         %ret = linalg.matmul %ret, A
+///         %inserted = insert %ret<alphaH x alphaW> into
+///                            output<N x H x W x F>
+///                            at [%n, (%h x m), (%w x m), %f]
+FailureOr<Operation *>
+decomposeWinogradOutputTransformOp(RewriterBase &rewriter,
+                                   linalg::WinogradOutputTransformOp op);
+
 //===----------------------------------------------------------------------===//
 // Rewrite patterns wrapping transformations.
 // TODO: every single such pattern should be a close to noop wrapper around a
diff --git a/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp b/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
index a101552e419bc87..775ed8f37344ed3 100644
--- a/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
+++ b/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
@@ -2855,8 +2855,8 @@ FailureOr<SmallVector<Value>> SoftmaxOp::decomposeOperation(OpBuilder &b) {
 LogicalResult WinogradFilterTransformOp::verify() {
   auto filterType = cast<ShapedType>(getFilter().getType());
   ArrayRef<int64_t> filterShape = filterType.getShape();
-  int64_t filterH = filterShape[1];
-  int64_t filterW = filterShape[2];
+  int64_t filterH = filterShape[getFilterHDim()];
+  int64_t filterW = filterShape[getFilterWDim()];
   int64_t r = getR();
   int64_t m = getM();
 
@@ -2870,8 +2870,8 @@ LogicalResult WinogradFilterTransformOp::verify() {
   SmallVector<int64_t> expectedOutputShape;
   expectedOutputShape.push_back(filterH == r ? m + r - 1 : 1);
   expectedOutputShape.push_back(filterW == r ? m + r - 1 : 1);
-  expectedOutputShape.push_back(filterShape[3]);
-  expectedOutputShape.push_back(filterShape[0]);
+  expectedOutputShape.push_back(filterShape[getFilterCDim()]);
+  expectedOutputShape.push_back(filterShape[getFilterFDim()]);
 
   auto outputType = cast<ShapedType>(getOutput().getType());
   ArrayRef<int64_t> outputShape = outputType.getShape();
@@ -2881,6 +2881,103 @@ LogicalResult WinogradFilterTransformOp::verify() {
   return success();
 }
 
+SmallVector<Range>
+WinogradFilterTransformOp::getIterationDomain(OpBuilder &builder) {
+  Location loc = getLoc();
+  IntegerAttr zeroAttr = builder.getIndexAttr(0);
+  IntegerAttr oneAttr = builder.getIndexAttr(1);
+  Value filter = getFilter();
+  int64_t filterRank = getFilterOperandRank();
+  SmallVector<Range> loopBounds(filterRank);
+  for (unsigned dim = 0; dim < filterRank; ++dim) {
+    loopBounds[dim].offset = zeroAttr;
+    loopBounds[dim].size = getDimValue(builder, loc, filter, dim);
+    loopBounds[dim].stride = oneAttr;
+  }
+  return loopBounds;
+}
+
+SmallVector<utils::IteratorType>
+WinogradFilterTransformOp::getLoopIteratorTypes() {
+  int64_t filterRank = getFilterOperandRank();
+  SmallVector<utils::IteratorType> iteratorTypes(filterRank,
+                                                 utils::IteratorType::parallel);
+  return iteratorTypes;
+}
+
+LogicalResult WinogradFilterTransformOp::getResultTilePosition(
+    OpBuilder &builder, unsigned resultNumber, ArrayRef<OpFoldResult> offsets,
+    ArrayRef<OpFoldResult> sizes, SmallVector<OpFoldResult> &resultOffsets,
+    SmallVector<OpFoldResult> &resultSizes) {
+  IntegerAttr zeroAttr = builder.getI64IntegerAttr(0);
+  ShapedType filterType = getFilterOperandType();
+  ArrayRef<int64_t> filterShape = filterType.getShape();
+  int64_t filterH = filterShape[getFilterHDim()];
+  int64_t filterW = filterShape[getFilterWDim()];
+  int64_t m = getM();
+  int64_t r = getR();
+  int64_t alpha = m + r - 1;
+  int64_t alphaH = filterH != 1 ? alpha : 1;
+  int64_t alphaW = filterW != 1 ? alpha : 1;
+  IntegerAttr alphaHAttr = builder.getI64IntegerAttr(alphaH);
+  IntegerAttr alphaWAttr = builder.getI64IntegerAttr(alphaW);
+
+  resultOffsets.append(
+      {zeroAttr, zeroAttr, offsets[getFilterCDim()], offsets[getFilterFDim()]});
+  resultSizes.append(
+      {alphaHAttr, alphaWAttr, sizes[getFilterCDim()], sizes[getFilterFDim()]});
+
+  return success();
+}
+
+/// Implement tiling for winograd_filter_transform
+/// The input of winograd_filter_transform is (F, KH, KW, C).
+/// The output of winograd_filter_transform is (alphaH, alphaW, C, F)
+/// Users can specify the tile sizes of F and C.
+/// `offsets` are the values for the offsets of F, KH, KW, C for one tile.
+/// `sizes` are the values for the sizes of F, KH, KW, C for one tile.
+FailureOr<TilingResult> WinogradFilterTransformOp::getTiledImplementation(
+    OpBuilder &builder, ArrayRef<OpFoldResult> offsets,
+    ArrayRef<OpFoldResult> sizes) {
+  IntegerAttr oneAttr = builder.getI64IntegerAttr(1);
+  IntegerAttr zeroAttr = builder.getI64IntegerAttr(0);
+  ShapedType filterType = getFilterOperandType();
+  ArrayRef<int64_t> filterShape = filterType.getShape();
+  int64_t filterH = filterShape[getFilterHDim()];
+  int64_t filterW = filterShape[getFilterWDim()];
+  IntegerAttr filterHAttr = builder.getI64IntegerAttr(filterH);
+  IntegerAttr filterWAttr = builder.getI64IntegerAttr(filterW);
+  SmallVector<Value> tiledOperands;
+  SmallVector<OpFoldResult> sliceOffsets, sliceSizes;
+
+  sliceOffsets.append(
+      {offsets[getFilterFDim()], zeroAttr, zeroAttr, offsets[getFilterCDim()]});
+  sliceSizes.append({sizes[getFilterFDim()], filterHAttr, filterWAttr,
+                     sizes[getFilterCDim()]});
+  int64_t filterRank = getFilterOperandRank();
+  SmallVector<OpFoldResult> filterStrides(filterRank, oneAttr);
+  Location loc = getLoc();
+  tiledOperands.emplace_back(builder.create<tensor::ExtractSliceOp>(
+      loc, getFilter(), sliceOffsets, sliceSizes, filterStrides));
+
+  SmallVector<OpFoldResult> resultOffsets, resultSizes;
+  if (failed(getResultTilePosition(builder, 1, offsets, sizes, resultOffsets,
+                                   resultSizes)))
+    return failure();
+
+  int64_t outputRank = getOutputOperandRank();
+  SmallVector<OpFoldResult> outputStrides(outputRank, oneAttr);
+  tiledOperands.emplace_back(builder.create<tensor::ExtractSliceOp>(
+      loc, getOutput(), resultOffsets, resultSizes, outputStrides));
+
+  SmallVector<Type> resultTypes;
+  resultTypes.push_back(tiledOperands[1].getType());
+  Operation *tiledOp =
+      mlir::clone(builder, getOperation(), resultTypes, tiledOperands);
+
+  return TilingResult{{tiledOp}, SmallVector<Value>(tiledOp->getResults())};
+}
+
 //===----------------------------------------------------------------------===//
 // WinogradInputTransformOp
 //===----------------------------------------------------------------------===//
@@ -2888,8 +2985,8 @@ LogicalResult WinogradFilterTransformOp::verify() {
 LogicalResult WinogradInputTransformOp::verify() {
   auto inputType = cast<ShapedType>(getInput().getType());
   ArrayRef<int64_t> inputShape = inputType.getShape();
-  int64_t inputH = inputShape[1];
-  int64_t inputW = inputShape[2];
+  int64_t inputH = inputShape[getInputHDim()];
+  int64_t inputW = inputShape[getInputWDim()];
   int m = getM();
   int r = getR();
   int64_t tileSize = m + r - 1;
@@ -2898,21 +2995,23 @@ LogicalResult WinogradInputTransformOp::verify() {
 
   SmallVector<int64_t> expectedOutputShape(6, inputH);
   if (ShapedType::isDynamic(inputH)) {
-    expectedOutputShape[0] = tileSize;
-    expectedOutputShape[2] = ShapedType::kDynamic;
+    expectedOutputShape[getOutputAlphaHDim()] = tileSize;
+    expectedOutputShape[getOutputTileHDim()] = ShapedType::kDynamic;
   } else {
-    expectedOutputShape[0] = leftTransform ? tileSize : 1;
-    expectedOutputShape[2] = leftTransform ? (inputH - (r - 1)) / m : 1;
+    expectedOutputShape[getOutputAlphaHDim()] = leftTransform ? tileSize : 1;
+    expectedOutputShape[getOutputTileHDim()] =
+        leftTransform ? (inputH - (r - 1)) / m : 1;
   }
   if (ShapedType::isDynamic(inputW)) {
-    expectedOutputShape[1] = tileSize;
-    expectedOutputShape[3] = ShapedType::kDynamic;
+    expectedOutputShape[getOutputAlphaWDim()] = tileSize;
+    expectedOutputShape[getOutputTileWDim()] = ShapedType::kDynamic;
   } else {
-    expectedOutputShape[1] = rightTransform ? tileSize : 1;
-    expectedOutputShape[3] = rightTransform ? (inputW - (r - 1)) / m : 1;
+    expectedOutputShape[getOutputAlphaWDim()] = rightTransform ? tileSize : 1;
+    expectedOutputShape[getOutputTileWDim()] =
+        rightTransform ? (inputW - (r - 1)) / m : 1;
   }
-  expectedOutputShape[4] = inputShape[0];
-  expectedOutputShape[5] = inputShape[3];
+  expectedOutputShape[getOutputNDim()] = inputShape[getInputNDim()];
+  expectedOutputShape[getOutputCDim()] = inputShape[getInputCDim()];
 
   auto outputType = cast<ShapedType>(getOutput().getType());
   ArrayRef<int64_t> outputShape = outputType.getShape();
@@ -2922,6 +3021,130 @@ LogicalResult WinogradInputTransformOp::verify() {
   return success();
 }
 
+SmallVector<Range>
+WinogradInputTransformOp::getIterationDomain(OpBuilder &builder) {
+  Location loc = getLoc();
+  IntegerAttr zeroAttr = builder.getIndexAttr(0);
+  IntegerAttr oneAttr = builder.getIndexAttr(1);
+  Value output = getOutput();
+  int64_t outputRank = getOutputOperandRank();
+  SmallVector<Range> loopBounds(outputRank);
+  for (unsigned dim = 0; dim < outputRank; ++dim) {
+    loopBounds[dim].offset = zeroAttr;
+    // alphaH, alphaW, tileH, tileW, N, C
+    loopBounds[dim].size = getDimValue(builder, loc, output, dim);
+    loopBounds[dim].stride = oneAttr;
+  }
+  return loopBounds;
+}
+
+SmallVector<utils::IteratorType>
+WinogradInputTransformOp::getLoopIteratorTypes() {
+  int64_t outputRank = getOutputOperandRank();
+  SmallVector<utils::IteratorType> iteratorTypes(outputRank,
+                                                 utils::IteratorType::parallel);
+  return iteratorTypes;
+}
+
+LogicalResult WinogradInputTransformOp::getResultTilePosition(
+    OpBuilder &builder, unsigned resultNumber, ArrayRef<OpFoldResult> offsets,
+    ArrayRef<OpFoldResult> sizes, SmallVector<OpFoldResult> &resultOffsets,
+    SmallVector<OpFoldResult> &resultSizes) {
+  IntegerAttr zeroAttr = builder.getI64IntegerAttr(0);
+  ShapedType inputType = getInputOperandType();
+  ArrayRef<int64_t> inputShape = inputType.getShape();
+  int64_t inputH = inputShape[getInputHDim()];
+  int64_t inputW = inputShape[getInputWDim()];
+  int64_t m = getM();
+  int64_t r = getR();
+  int64_t alpha = m + r - 1;
+  int64_t alphaH = inputH != 1 ? alpha : 1;
+  int64_t alphaW = inputW != 1 ? alpha : 1;
+  IntegerAttr alphaHAttr = builder.getI64IntegerAttr(alphaH);
+  IntegerAttr alphaWAttr = builder.getI64IntegerAttr(alphaW);
+
+  resultOffsets.append({zeroAttr, zeroAttr, offsets[getOutputTileHDim()],
+                        offsets[getOutputTileWDim()], offsets[getOutputNDim()],
+                        offsets[getOutputCDim()]});
+  resultSizes.append({alphaHAttr, alphaWAttr, sizes[getOutputTileHDim()],
+                      sizes[getOutputTileWDim()], sizes[getOutputNDim()],
+                      sizes[getOutputCDim()]});
+
+  return success();
+}
+
+/// Implement tiling for winograd_input_transform
+/// The input of winograd_input_transform is (N, H, W, C).
+/// The output of winograd_input_transform is (alphaH, alphaW, tileH, tileW, N,
+/// C) Users can specify the tile sizes of tileH, tileW, N, and C. `offsets` are
+/// the values for the offsets of tileH, tileW, N, C for one tile. `sizes` are
+/// the values for the sizes of tileH, tileW, N, C for one tile.
+FailureOr<TilingResult>
+WinogradInputTransformOp::getTiledImplementation(OpBuilder &builder,
+                                                 ArrayRef<OpFoldResult> offsets,
+                                                 ArrayRef<OpFoldResult> sizes) {
+  IntegerAttr oneAttr = builder.getI64IntegerAttr(1);
+  IntegerAttr zeroAttr = builder.getI64IntegerAttr(0);
+  ShapedType inputType = getInputOperandType();
+  ArrayRef<int64_t> inputShape = inputType.getShape();
+  int64_t inputH = inputShape[getInputHDim()];
+  int64_t inputW = inputShape[getInputWDim()];
+  int64_t m = getM();
+  int64_t r = getR();
+
+  Location loc = getLoc();
+  MLIRContext *context = builder.getContext();
+  auto offsetAffineMap =
+      AffineMap::get(1, 0, {builder.getAffineDimExpr(0) * m}, context);
+  Value mappedOffsetH = affine::makeComposedAffineApply(
+      builder, loc, offsetAffineMap, offsets[getOutputTileHDim()]);
+  Value mappedOffsetW = affine::makeComposedAffineApply(
+      builder, loc, offsetAffineMap, offsets[getOutputTileWDim()]);
+  auto sizeAffineMap = AffineMap::get(
+      1, 0, {builder.getAffineDimExpr(0) * m + (r - 1)}, context);
+  Value mappedSizeH = affine::makeComposedAffineApply(
+      builder, loc, sizeAffineMap, sizes[getOutputTileHDim()]);
+  Value mappedSizeW = affine::makeComposedAffineApply(
+      builder, loc, sizeAffineMap, sizes[getOutputTileWDim()]);
+
+  SmallVector<Value> tiledOperands;
+  SmallVector<OpFoldResult> sliceOffsets, sliceSizes;
+
+  OpFoldResult offsetH =
+      inputH != 1 ? OpFoldResult(mappedOffsetH) : OpFoldResult(zeroAttr);
+  OpFoldResult offsetW =
+      inputW != 1 ? OpFoldResult(mappedOffsetW) : OpFoldResult(zeroAttr);
+  sliceOffsets.append(
+      {offsets[getOutputNDim()], offsetH, offsetW, offsets[getOutputCDim()]});
+  OpFoldResult sizeH =
+      inputH != 1 ? OpFoldResult(mappedSizeH) : OpFoldResult(oneAttr);
+  OpFoldResult sizeW =
+      inputW != 1 ? OpFoldResult(mappedSizeW) : OpFoldResult(oneAttr);
+  sliceSizes.append(
+      {sizes[getOutputNDim()], sizeH, sizeW, sizes[getOutputCDim()]});
+  int64_t inputRank = getInputOperandRank();
+  SmallVector<OpFoldResult> inputStrides(inputRank, oneAttr);
+  tiledOperands.emplace_back(builder.create<tensor::ExtractSliceOp>(
+      loc, getInput(), sliceOffsets, sliceSizes, inputStrides));
+
+  SmallVector<OpFoldResult> resultOffsets, resultSizes;
+  if (failed(getResultTilePosition(builder, 1, offsets, sizes, resultOffsets,
+                                   resultSizes)))
+    return failure();
+
+  int64_t outputRank = getOutputOperandRank();
+  SmallVector<OpFoldResult> outputStrides(outputRank, oneAttr);
+  tiledOperands.emplace_back(builder.create<tensor::ExtractSliceOp>(
+      loc, getOutput(), resultOffsets, resultSizes, outputStrides));
+
+  SmallVector<Type> resultTypes;
+  resultTypes.push_back(tiledOperands[1].getType());
+  Operation *tiledOp =
+      mlir::clone(builder, getOperation(), resultTypes, tiledOperands);
+
+  return TilingResult{{tiledOp}, SmallVector<Value>(tiledOp->getResults())};
+}
+
 //===----------------------------------------------------------------------===//
 // WinogradOutputTransformOp
 //===----------------------------------------------------------------------===//
@@ -2929,32 +3152,34 @@ LogicalResult WinogradInputTransformOp::verify() {
 LogicalResult WinogradOutputTransformOp::verify() {
   auto valueType = cast<ShapedType>(getValue().getType());
   ArrayRef<int64_t> valueShape = valueType.getShape();
-  int64_t valueH = valueShape[0];
-  int64_t valueW = valueShape[1];
-  int64_t valueTileH = valueShape[2];
-  int64_t valueTileW = valueShape[3];
+  int64_t valueH = valueShape[getValueAlphaHDim()];
+  int64_t valueW = valueShape[getValueAlphaWDim()];
+  int64_t valueTileH = valueShape[getValueTileHDim()];
+  int64_t valueTileW = valueShape[getValueTileWDim()];
   int m = getM();
   int r = getR();
   bool leftTransform = valueH != 1;
   bool rightTransform = valueW != 1;
 
-  SmallVector<int64_t> expectedOutputShape(4, valueH);
+  int64_t outputRank = getOutputOperandRank();
+  SmallVector<int64_t> expectedOutputShape(outputRank, valueH);
   if (ShapedType::isDynamic(valueH) || ShapedType::isDynamic(valueTileH)) {
-    expectedOutputShape[1] = ShapedType::kDynamic;
+    expectedOutputShape[getOutputHDim()] = ShapedType::kDynamic;
   } else {
     if (valueH != (leftTransform ? m + r - 1 : 1))
       return emitOpError("expect input height equals to input tile size");
-    expectedOutputShape[1] = (leftTransform ? m : 1) * valueTileH;
+    expectedOutputShape[getOutputHDim()] = (leftTransform ? m : 1) * valueTileH;
   }
   if (ShapedType::isDynamic(valueW) || ShapedType::isDynamic(valueTileW)) {
-    expectedOutputShape[2] = ShapedType::kDynamic;
+    expectedOutputShape[getOutputWDim()] = ShapedType::kDynamic;
   } else {
     if (valueW != (rightTransform ? m + r - 1 : 1))
       return emitOpError("expect input width equals to input tile size");
-    expectedOutputShape[2] = (rightTransform ? m : 1) * valueTileW;
+    expectedOutputShape[getOutputWDim()] =
+        (rightTransform ? m : 1) * valueTileW;
   }
-  expectedOutputShape[0] = valueShape[4];
-  expectedOutputShape[3] = valueShape[5];
+  expectedOutputShape[getOutputNDim()] = valueShape[getValueNDim()];
+  expectedOutputShape[getOutputFDim()] = valueShape[getValueFDim()];
 
   auto outputType = cast<ShapedType>(getOutput().getType());
   ArrayRef<int64_t> outputShape = outputType.getShape();
@@ -2964,6 +3189,124 @@ LogicalResult WinogradOutputTransformOp::verify() {
   return success();
 }
 
+SmallVector<Range>
+WinogradOutputTransformOp::getIterationDomain(OpBuilder &builder) {
+  Location loc = getLoc();
+  IntegerAttr zeroAttr = builder.getIndexAttr(0);
+  IntegerAttr oneAttr = builder.getIndexAttr(1);
+  Value value = getValue();
+  int64_t valueRank = getValueOperandRank();
+  SmallVector<Range> loopBounds(valueRank);
+  for (unsigned dim = 0; dim < valueRank; ++dim) {
+    loopBounds[dim].offset = zeroAttr;
+    // alphaH, alphaW, tileH, tileW, N, F
+    loopBounds[dim].size = getDimValue(builder, loc, value, dim);
+    loopBounds[dim].stride = oneAttr;
+  }
+  return loopBounds;
+}
+
+SmallVector<utils::IteratorType>
+WinogradOutputTransformOp::getLoopIteratorTypes() {
+  int64_t valueRank = getValueOperandRank();
+  SmallVector<utils::IteratorType> iteratorTypes(valueRank,
+                                                 utils::IteratorType::parallel);
+  return iteratorTypes;
+}
+
+LogicalResult WinogradOutputTransformOp::getResultTilePosition(
+    OpBuilder &builder, unsigned resultNumber, ArrayRef<OpFoldResult> offsets,
+    ArrayRef<OpFoldResult> sizes, SmallVector<OpFoldResult> &resultOffsets,
+    SmallVector<OpFoldResult> &resultSizes) {
+  int64_t m = getM();
+
+  Location loc = getLoc();
+  MLIRContext *context = builder.getContext();
+  auto affineMap =
+      AffineMap::get(1, 0, {builder.getAffineDimExpr(0) * m}, context);
+
+  Value mappedOffsetH = affine::makeComposedAffineApply(
+      builder, loc, affineMap, offsets[getValueTileHDim()]);
+  Value mappedOffsetW = affine::makeComposedAffineApply(
+      builder, loc, affineMap, offsets[getValueTileWDim()]);
+  Value mappedSizeH = affine::makeComposedAffineApply(
+      builder, loc, affineMap, sizes[getValueTileHDim()]);
+  Value mappedSizeW = affine::makeComposedAffineApply(
+      builder, loc, affineMap, sizes[getValueTileWDim()]);
+
+  ShapedType valueType = getValueOperandType();
+  ArrayRef<int64_t> valueShape = valueType.getShape();
+  int64_t valueH = valueShape[0];
+  int64_t valueW = valueShape[1];
+  IntegerAttr oneAttr = builder.getI64IntegerAttr(1);
+  IntegerAttr zeroAttr = builder.getI64IntegerAttr(0);
+  OpFoldResult offsetH =
+      valueH != 1 ? OpFoldResult(mappedOffsetH) : OpFoldResult(zeroAttr);
+  OpFoldResult offsetW =
+      valueW != 1 ? OpFoldResult(mappedOffsetW) : OpFoldResult(zeroAttr);
+  OpFoldResult sizeH =
+      valueH != 1 ? OpFoldResult(mappedSizeH) : OpFoldResult(oneAttr);
+  OpFoldResult sizeW =
+      valueW != 1 ? OpFoldResult(mappedSizeW) : OpFoldResult(oneAttr);
+
+  resultOffsets.append(
+      {offsets[getValueNDim()], offsetH, offsetW, offsets[getValueFDim()]});
+  resultSizes.append(
+      {sizes[getValueNDim()], sizeH, sizeW, sizes[getValueFDim()]});
+  return success();
+}
+
+/// Implement tiling for winograd_output_transform
+/// The input of winograd_output_transform is (alphaH, alphaW, tileH, tileW, N,
+/// F). The output of winograd_output_transform is (N, H, W, F) Users can
+/// specify the tile sizes of tileH, tileW, N, and F. `offsets` are the values
+/// for the offsets of tileH, tileW, N, F for one tile. `sizes` are the values
+/// for the sizes of tileH, tileW, N, F for one tile.
+FailureOr<TilingResult> WinogradOutputTransformOp::getTiledImplementation(
+    OpBuilder &builder, ArrayRef<OpFoldResult> offsets,
+    ArrayRef<OpFoldResult> sizes) {
+  IntegerAttr oneAttr = builder.getI64IntegerAttr(1);
+  IntegerAttr zeroAttr = builder.getI64IntegerAttr(0);
+  Location loc = getLoc();
+  SmallVector<Value> tiledOperands;
+  SmallVector<OpFoldResult> sliceOffsets, sliceSizes;
+
+  ShapedType valueType = getValueOperandType();
+  ArrayRef<int64_t> valueShape = valueType.getShape();
+  int64_t alphaH = valueShape[getValueAlphaHDim()];
+  int64_t alphaW = valueShape[getValueAlphaWDim()];
+  IntegerAttr alphaHAttr = builder.getI64IntegerAttr(alphaH);
+  IntegerAttr alphaWAttr = builder.getI64IntegerAttr(alphaW);
+
+  sliceOffsets.append({zeroAttr, zeroAttr, offsets[getValueTileHDim()],
+                       offsets[getValueTileWDim()], offsets[getValueNDim()],
+                       offsets[getValueFDim()]});
+  sliceSizes.append({alphaHAttr, alphaWAttr, sizes[getValueTileHDim()],
+                     sizes[getValueTileWDim()], sizes[getValueNDim()],
+                     sizes[getValueFDim()]});
+  int64_t valueRank = getValueOperandRank();
+  SmallVector<OpFoldResult> sliceStrides(valueRank, oneAttr);
+  tiledOperands.emplace_back(builder.create<tensor::ExtractSliceOp>(
+      loc, getValue(), sliceOffsets, sliceSizes, sliceStrides));
+
+  SmallVector<OpFoldResult> resultOffsets, resultSizes;
+  if (failed(getResultTilePosition(builder, 1, offsets, sizes, resultOffsets,
+                                   resultSizes)))
+    return failure();
+
+  int64_t outputRank = getOutputOperandRank();
+  SmallVector<OpFoldResult> strides(outputRank, oneAttr);
+  tiledOperands.emplace_back(builder.create<tensor::ExtractSliceOp>(
+      loc, getOutput(), resultOffsets, resultSizes, strides));
+
+  SmallVector<Type> resultTypes;
+  resultTypes.push_back(tiledOperands[1].getType());
+  Operation *tiledOp =
+      mlir::clone(builder, getOperation(), resultTypes, tiledOperands);
+
+  return TilingResult{{tiledOp}, SmallVector<Value>(tiledOp->getResults())};
+}
+
 //===----------------------------------------------------------------------===//
 // LinalgDialect
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
index 48b3abbeee7010c..fbf4e29024f7c20 100644
--- a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
+++ b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
@@ -3851,6 +3851,47 @@ DiagnosedSilenceableFailure transform::WinogradConv2DOp::applyToOne(
   return DiagnosedSilenceableFailure::success();
 }
 
+DiagnosedSilenceableFailure transform::DecomposeWinogradOp::applyToOne(
+    transform::TransformRewriter &rewriter, Operation *target,
+    transform::ApplyToEachResultList &results,
+    transform::TransformState &state) {
+  rewriter.setInsertionPoint(target);
+  FailureOr<Operation *> maybeTransformed = failure();
+  bool supported =
+      TypeSwitch<Operation *, bool>(target)
+          .Case([&](linalg::WinogradFilterTransformOp op) {
+            maybeTransformed = decomposeWinogradFilterTransformOp(rewriter, op);
+            return true;
+          })
+          .Case([&](linalg::WinogradInputTransformOp op) {
+            maybeTransformed = decomposeWinogradInputTransformOp(rewriter, op);
+            return true;
+          })
+          .Case([&](linalg::WinogradOutputTransformOp op) {
+            maybeTransformed = decomposeWinogradOutputTransformOp(rewriter, op);
+            return true;
+          })
+          .Default([&](Operation *op) { return false; });
+
+  if (!supported) {
+    DiagnosedSilenceableFailure diag =
+        emitSilenceableError()
+        << "this operation is not supported to decompose into other operations";
+    diag.attachNote(target->getLoc()) << "target op";
+    return diag;
+  }
+
+  if (supported && failed(maybeTransformed)) {
+    DiagnosedSilenceableFailure diag =
+        emitSilenceableError() << "decompose Winograd operations failed";
+    diag.attachNote(target->getLoc()) << "target op";
+    return diag;
+  }
+
+  results.push_back(*maybeTransformed);
+  return DiagnosedSilenceableFailure::success();
+}
+
 #include "mlir/Dialect/Linalg/TransformOps/LinalgTransformOpsEnums.cpp.inc"
 
 #define GET_OP_CLASSES
diff --git a/mlir/lib/Dialect/Linalg/Transforms/WinogradConv2D.cpp b/mlir/lib/Dialect/Linalg/Transforms/WinogradConv2D.cpp
index c6c770e2781ff07..b65b18699a15aa3 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/WinogradConv2D.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/WinogradConv2D.cpp
@@ -490,8 +490,6 @@ Value inputTransform(RewriterBase &rewriter, Location loc, Value input,
   Type elementType = inputType.getElementType();
   auto inputShape = inputType.getShape(); // N, H, W, C
   int64_t inputN = inputShape[0];
-  int64_t inputH = inputShape[1];
-  int64_t inputW = inputShape[2];
   int64_t inputC = inputShape[3];
   auto valueType = cast<ShapedType>(retValue.getType());
   auto valueShape = valueType.getShape(); // alphaH, alphaW, HTile, WTile, N, C
@@ -500,11 +498,6 @@ Value inputTransform(RewriterBase &rewriter, Location loc, Value input,
   int64_t alphaH = leftTransform ? m + r - 1 : 1;
   int64_t alphaW = rightTransform ? m + r - 1 : 1;
 
-  if ((inputH != (tileH * m) + (r - 1)) && inputH != 1)
-    return Value();
-  if ((inputW != (tileW * m) + (r - 1)) && inputW != 1)
-    return Value();
-
   auto buildBody = [&](OpBuilder &builder, Location loc, ValueRange ivs,
                        ValueRange args) -> scf::ValueVector {
     Value tileHIter = ivs[0];
@@ -1169,6 +1162,24 @@ FailureOr<Operation *> winogradConv2D(RewriterBase &rewriter,
   return winogradConv2DHelper(rewriter, op, m, r);
 }
 
+FailureOr<Operation *>
+decomposeWinogradFilterTransformOp(RewriterBase &rewriter,
+                                   linalg::WinogradFilterTransformOp op) {
+  return decomposeWinogradFilterTransformHelper(rewriter, op);
+}
+
+FailureOr<Operation *>
+decomposeWinogradInputTransformOp(RewriterBase &rewriter,
+                                  linalg::WinogradInputTransformOp op) {
+  return decomposeWinogradInputTransformHelper(rewriter, op);
+}
+
+FailureOr<Operation *>
+decomposeWinogradOutputTransformOp(RewriterBase &rewriter,
+                                   linalg::WinogradOutputTransformOp op) {
+  return decomposeWinogradOutputTransformHelper(rewriter, op);
+}
+
 void populateWinogradConv2DPatterns(RewritePatternSet &patterns, int64_t m,
                                     int64_t r) {
   MLIRContext *context = patterns.getContext();
diff --git a/mlir/test/Dialect/Linalg/transform-tile-and-winograd-rewrite.mlir b/mlir/test/Dialect/Linalg/transform-tile-and-winograd-rewrite.mlir
new file mode 100644
index 000000000000000..6bb3fb1423edc62
--- /dev/null
+++ b/mlir/test/Dialect/Linalg/transform-tile-and-winograd-rewrite.mlir
@@ -0,0 +1,292 @@
+// RUN: mlir-opt %s -transform-interpreter -canonicalize --split-input-file | FileCheck %s
+
+func.func @conv2d(%arg0: tensor<2x10x10x5xf32>, %arg1: tensor<2x3x3x5xf32>, %arg2: tensor<2x8x8x2xf32>) -> tensor<2x8x8x2xf32> {
+  %0 = tensor.empty() : tensor<6x6x5x2xf32>
+  %1 = linalg.winograd_filter_transform m(4) r(3) ins(%arg1 : tensor<2x3x3x5xf32>) outs(%0 : tensor<6x6x5x2xf32>) -> tensor<6x6x5x2xf32>
+  %2 = tensor.empty() : tensor<6x6x2x2x2x5xf32>
+  %3 = linalg.winograd_input_transform m(4) r(3) ins(%arg0 : tensor<2x10x10x5xf32>) outs(%2 : tensor<6x6x2x2x2x5xf32>) -> tensor<6x6x2x2x2x5xf32>
+  %collapsed = tensor.collapse_shape %1 [[0, 1], [2], [3]] : tensor<6x6x5x2xf32> into tensor<36x5x2xf32>
+  %collapsed_0 = tensor.collapse_shape %3 [[0, 1], [2, 3, 4], [5]] : tensor<6x6x2x2x2x5xf32> into tensor<36x8x5xf32>
+  %4 = tensor.empty() : tensor<36x8x2xf32>
+  %5 = linalg.batch_matmul ins(%collapsed_0, %collapsed : tensor<36x8x5xf32>, tensor<36x5x2xf32>) outs(%4 : tensor<36x8x2xf32>) -> tensor<36x8x2xf32>
+  %expanded = tensor.expand_shape %5 [[0, 1], [2, 3, 4], [5]] output_shape [6, 6, 2, 2, 2, 2] : tensor<36x8x2xf32> into tensor<6x6x2x2x2x2xf32>
+  %6 = linalg.winograd_output_transform m(4) r(3) ins(%expanded : tensor<6x6x2x2x2x2xf32>) outs(%arg2 : tensor<2x8x8x2xf32>) -> tensor<2x8x8x2xf32>
+  return %6 : tensor<2x8x8x2xf32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["linalg.winograd_filter_transform"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %2 = transform.structured.match ops{["linalg.winograd_input_transform"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %3, %loop3:2 = transform.structured.tile_using_for %2 tile_sizes [0, 0, 1, 1, 0, 0] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
+    %4 = transform.structured.match ops{["linalg.winograd_output_transform"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %5, %loop5:2 = transform.structured.tile_using_for %4 tile_sizes [0, 0, 1, 1, 0, 0] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
+    %7 = transform.structured.decompose_winograd_op %0 : (!transform.any_op) -> (!transform.any_op)
+    %8 = transform.structured.match ops{["linalg.winograd_input_transform"]} in %3 : (!transform.any_op) -> !transform.any_op
+    %9 = transform.structured.decompose_winograd_op %8 : (!transform.any_op) -> (!transform.any_op)
+    %10 = transform.structured.match ops{["linalg.winograd_output_transform"]} in %5 : (!transform.any_op) -> !transform.any_op
+    %11 = transform.structured.decompose_winograd_op %10 : (!transform.any_op) -> (!transform.any_op)
+    transform.yield
+  }
+}
+
+// CHECK: #[[$MAP0:.+]] = affine_map<(d0) -> (d0 * 4)>
+// CHECK: #[[$MAP1:.+]] = affine_map<(d0, d1) -> ()>
+// CHECK: #[[$MAP2:.+]] = affine_map<(d0, d1) -> (d0, d1)>
+// CHECK-LABEL: func.func @conv2d
+// CHECK-SAME:  (%[[ARG0:.*]]: tensor<2x10x10x5xf32>, %[[ARG1:.*]]: tensor<2x3x3x5xf32>, %[[ARG2:.*]]: tensor<2x8x8x2xf32>) -> tensor<2x8x8x2xf32> {
+// CHECK:  %[[CST:.*]] = arith.constant 1.024000e+03 : f32
+// CHECK:  %[[C1:.*]] = arith.constant 1 : index
+// CHECK:  %[[C5:.*]] = arith.constant 5 : index
+// CHECK:  %[[C2:.*]] = arith.constant 2 : index
+// CHECK:  %[[C0:.*]] = arith.constant 0 : index
+// CHECK:  %[[S0:.*]] = tensor.empty()
+// CHECK:  %[[S1:.*]] = scf.for %[[ARG3:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG4:.*]] = %[[S0]])
+// CHECK:    %[[S9:.*]] = scf.for %[[ARG5:.*]] = %[[C0]] to %[[C5]] step %[[C1]] iter_args(%[[ARG6:.*]] = %[[ARG4]])
+// CHECK:      %[[EXTRACTED_SLICE:.*]] = tensor.extract_slice %[[ARG1]][%[[ARG3]], 0, 0, %[[ARG5]]] [1, 3, 3, 1] [1, 1, 1, 1]
+// CHECK:      %[[S11:.*]] = linalg.matmul
+// CHECK:      %[[S13:.*]] = linalg.matmul
+// CHECK:      %[[INSERTED_SLICE:.*]] = tensor.insert_slice %[[S13]] into %[[ARG6]][0, 0, %[[ARG5]], %[[ARG3]]] [6, 6, 1, 1] [1, 1, 1, 1]
+// CHECK:      scf.yield %[[INSERTED_SLICE]]
+// CHECK:    scf.yield %[[S9]]
+// CHECK:  %[[S2:.*]] = tensor.empty() : tensor<6x6x2x2x2x5xf32>
+// CHECK:  %[[S3:.*]] = tensor.empty() : tensor<6x6x2x2x2x5xf32>
+// CHECK:  %[[S4:.*]] = scf.for %[[ARG3:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG4:.*]] = %[[S3]])
+// CHECK:    %[[S9:.*]] = scf.for %[[ARG5:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG6:.*]] = %[[ARG4]])
+// CHECK:      %[[S10:.*]] = affine.apply #[[$MAP0]](%[[ARG3]])
+// CHECK:      %[[S11:.*]] = affine.apply #[[$MAP0]](%[[ARG5]])
+// CHECK:      %[[EXTRACTED_SLICE:.*]] = tensor.extract_slice %[[ARG0]][0, %[[S10]], %[[S11]], 0] [2, 6, 6, 5] [1, 1, 1, 1]
+// CHECK:      %[[EXTRACTED_SLICE_7:.*]] = tensor.extract_slice %[[S2]][0, 0, %[[ARG3]], %[[ARG5]], 0, 0] [6, 6, 1, 1, 2, 5] [1, 1, 1, 1, 1, 1]
+// CHECK:      %[[S12:.*]] = scf.for %[[ARG7:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG8:.*]] = %[[EXTRACTED_SLICE_7]])
+// CHECK:        %[[S13:.*]] = scf.for %[[ARG9:.*]] = %[[C0]] to %[[C5]] step %[[C1]] iter_args(%[[ARG10:.*]] = %[[ARG8]])
+// CHECK:          %[[EXTRACTED_SLICE_8:.*]] = tensor.extract_slice %[[EXTRACTED_SLICE]][%[[ARG7]], 0, 0, %[[ARG9]]] [1, 6, 6, 1] [1, 1, 1, 1]
+// CHECK:          %[[S15:.*]] = linalg.matmul
+// CHECK:          %[[S17:.*]] = linalg.matmul
+// CHECK:          %[[INSERTED_SLICE_9:.*]] = tensor.insert_slice %[[S17]] into %[[ARG10]][0, 0, 0, 0, %[[ARG7]], %[[ARG9]]] [6, 6, 1, 1, 1, 1] [1, 1, 1, 1, 1, 1]
+// CHECK:          scf.yield %[[INSERTED_SLICE_9]]
+// CHECK:        scf.yield %[[S13]]
+// CHECK:      %[[INSERTED_SLICE:.*]] = tensor.insert_slice %[[S12]] into %[[ARG6]][0, 0, %[[ARG3]], %[[ARG5]], 0, 0] [6, 6, 1, 1, 2, 5] [1, 1, 1, 1, 1, 1]
+// CHECK:      scf.yield %[[INSERTED_SLICE]]
+// CHECK:    scf.yield %[[S9]]
+// CHECK:  %[[COLLAPSED:.*]] = tensor.collapse_shape %[[S1]] {{\[}}[0, 1], [2], [3]]
+// CHECK:  %[[COLLAPSED_6:.*]] = tensor.collapse_shape %[[S4]] {{\[}}[0, 1], [2, 3, 4], [5]]
+// CHECK:  %[[S6:.*]] = linalg.batch_matmul
+// CHECK:  %[[EXPANDED:.*]] = tensor.expand_shape %[[S6]] {{\[}}[0, 1], [2, 3, 4], [5]] output_shape [6, 6, 2, 2, 2, 2]
+// CHECK:  %[[S7:.*]] = tensor.empty() : tensor<2x8x8x2xf32>
+// CHECK:  %[[S8:.*]] = scf.for %[[ARG3:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG4:.*]] = %[[S7]])
+// CHECK:    %[[S9:.*]] = scf.for %[[ARG5:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG6:.*]] = %[[ARG4]])
+// CHECK:      %[[EXTRACTED_SLICE:.*]] = tensor.extract_slice %[[EXPANDED]][0, 0, %[[ARG3]], %[[ARG5]], 0, 0] [6, 6, 1, 1, 2, 2] [1, 1, 1, 1, 1, 1]
+// CHECK:      %[[S10:.*]] = affine.apply #[[$MAP0]](%[[ARG3]])
+// CHECK:      %[[S11:.*]] = affine.apply #[[$MAP0]](%[[ARG5]])
+// CHECK:      %[[EXTRACTED_SLICE_7:.*]] = tensor.extract_slice %[[ARG2]][0, %[[S10]], %[[S11]], 0] [2, 4, 4, 2] [1, 1, 1, 1]
+// CHECK:      %[[S12:.*]] = scf.for %[[ARG7:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG8:.*]] = %[[EXTRACTED_SLICE_7]])
+// CHECK:        %[[S15:.*]] = scf.for %[[ARG9:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG10:.*]] = %[[ARG8]])
+// CHECK:          %[[EXTRACTED_SLICE_8:.*]] = tensor.extract_slice %[[EXTRACTED_SLICE]][0, 0, 0, 0, %[[ARG7]], %[[ARG9]]] [6, 6, 1, 1, 1, 1] [1, 1, 1, 1, 1, 1]
+// CHECK:          %[[S17:.*]] = linalg.matmul
+// CHECK:          %[[S19:.*]] = linalg.matmul
+// CHECK:          %[[S20:.*]] = tensor.empty()
+// CHECK:          %[[S21:.*]] = linalg.generic {indexing_maps = [#[[$MAP1]], #[[$MAP2]]], iterator_types = ["parallel", "parallel"]} ins(%[[CST]] : f32) outs(%[[S20]] : tensor<4x4xf32>) {
+// CHECK:          ^bb0(%[[IN:.*]]: f32, %[[OUT:.*]]: f32):
+// CHECK:            linalg.yield %[[IN]] : f32
+// CHECK:          } -> tensor<4x4xf32>
+// CHECK:          %[[S22:.*]] = linalg.mul ins(%[[S21]], %[[S19]] : tensor<4x4xf32>, tensor<4x4xf32>) outs(%[[S20]] : tensor<4x4xf32>) -> tensor<4x4xf32>
+// CHECK:          %[[INSERTED_SLICE_9:.*]] = tensor.insert_slice %[[S22]] into %[[ARG10]][%[[ARG7]], 0, 0, %[[ARG9]]] [1, 4, 4, 1] [1, 1, 1, 1]
+// CHECK:          scf.yield %[[INSERTED_SLICE_9]]
+// CHECK:        scf.yield %[[S15]]
+// CHECK:      %[[S13:.*]] = affine.apply #[[$MAP0]](%[[ARG3]])
+// CHECK:      %[[S14:.*]] = affine.apply #[[$MAP0]](%[[ARG5]])
+// CHECK:      %[[INSERTED_SLICE:.*]] = tensor.insert_slice %[[S12]] into %[[ARG6]][0, %[[S13]], %[[S14]], 0] [2, 4, 4, 2] [1, 1, 1, 1]
+// CHECK:      scf.yield %[[INSERTED_SLICE]]
+// CHECK:    scf.yield %[[S9]]
+
+// -----
+
+func.func @conv2d_unaligned(%arg0: tensor<2x11x11x5xf32>, %arg1: tensor<2x3x3x5xf32>, %arg2: tensor<2x9x9x2xf32>) -> tensor<2x9x9x2xf32> {
+  %cst = arith.constant 0.000000e+00 : f32
+  %0 = tensor.empty() : tensor<6x6x5x2xf32>
+  %1 = linalg.winograd_filter_transform m(4) r(3) ins(%arg1 : tensor<2x3x3x5xf32>) outs(%0 : tensor<6x6x5x2xf32>) -> tensor<6x6x5x2xf32>
+  %padded = tensor.pad %arg0 low[0, 0, 0, 0] high[0, 3, 3, 0] {
+  ^bb0(%arg4: index, %arg5: index, %arg6: index, %arg7: index):
+    tensor.yield %cst : f32
+  } : tensor<2x11x11x5xf32> to tensor<2x14x14x5xf32>
+  %2 = tensor.empty() : tensor<6x6x3x3x2x5xf32>
+  %3 = linalg.winograd_input_transform m(4) r(3) ins(%padded : tensor<2x14x14x5xf32>) outs(%2 : tensor<6x6x3x3x2x5xf32>) -> tensor<6x6x3x3x2x5xf32>
+  %collapsed = tensor.collapse_shape %1 [[0, 1], [2], [3]] : tensor<6x6x5x2xf32> into tensor<36x5x2xf32>
+  %collapsed_0 = tensor.collapse_shape %3 [[0, 1], [2, 3, 4], [5]] : tensor<6x6x3x3x2x5xf32> into tensor<36x18x5xf32>
+  %4 = tensor.empty() : tensor<36x18x2xf32>
+  %5 = linalg.batch_matmul ins(%collapsed_0, %collapsed : tensor<36x18x5xf32>, tensor<36x5x2xf32>) outs(%4 : tensor<36x18x2xf32>) -> tensor<36x18x2xf32>
+  %expanded = tensor.expand_shape %5 [[0, 1], [2, 3, 4], [5]] output_shape [6, 6, 3, 3, 2, 2] : tensor<36x18x2xf32> into tensor<6x6x3x3x2x2xf32>
+  %padded_1 = tensor.pad %arg2 low[0, 0, 0, 0] high[0, 3, 3, 0] {
+  ^bb0(%arg4: index, %arg5: index, %arg6: index, %arg7: index):
+    tensor.yield %cst : f32
+  } : tensor<2x9x9x2xf32> to tensor<2x12x12x2xf32>
+  %6 = linalg.winograd_output_transform m(4) r(3) ins(%expanded : tensor<6x6x3x3x2x2xf32>) outs(%padded_1 : tensor<2x12x12x2xf32>) -> tensor<2x12x12x2xf32>
+  %extracted_slice = tensor.extract_slice %6[0, 0, 0, 0] [2, 9, 9, 2] [1, 1, 1, 1] : tensor<2x12x12x2xf32> to tensor<2x9x9x2xf32>
+  return %extracted_slice : tensor<2x9x9x2xf32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["linalg.winograd_filter_transform"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %2 = transform.structured.match ops{["linalg.winograd_input_transform"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %3, %loop3:2 = transform.structured.tile_using_for %2 tile_sizes [0, 0, 1, 1, 0, 0] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
+    %4 = transform.structured.match ops{["linalg.winograd_output_transform"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %5, %loop5:2 = transform.structured.tile_using_for %4 tile_sizes [0, 0, 1, 1, 0, 0] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
+    %7 = transform.structured.decompose_winograd_op %0 : (!transform.any_op) -> (!transform.any_op)
+    %8 = transform.structured.match ops{["linalg.winograd_input_transform"]} in %3 : (!transform.any_op) -> !transform.any_op
+    %9 = transform.structured.decompose_winograd_op %8 : (!transform.any_op) -> (!transform.any_op)
+    %10 = transform.structured.match ops{["linalg.winograd_output_transform"]} in %5 : (!transform.any_op) -> !transform.any_op
+    %11 = transform.structured.decompose_winograd_op %10 : (!transform.any_op) -> (!transform.any_op)
+    transform.yield
+  }
+}
+
+// CHECK: #[[$MAP0:.+]] = affine_map<(d0) -> (d0 * 4)>
+// CHECK: #[[$MAP1:.+]] = affine_map<(d0, d1) -> ()>
+// CHECK: #[[$MAP2:.+]] = affine_map<(d0, d1) -> (d0, d1)>
+// CHECK-LABEL: func.func @conv2d_unaligned
+// CHECK-SAME:  (%[[ARG0:.*]]: tensor<2x11x11x5xf32>, %[[ARG1:.*]]: tensor<2x3x3x5xf32>, %[[ARG2:.*]]: tensor<2x9x9x2xf32>) -> tensor<2x9x9x2xf32> {
+// CHECK:  %[[CST:.*]] = arith.constant 1.024000e+03 : f32
+// CHECK:  %[[C3:.*]] = arith.constant 3 : index
+// CHECK:  %[[C1:.*]] = arith.constant 1 : index
+// CHECK:  %[[C5:.*]] = arith.constant 5 : index
+// CHECK:  %[[C2:.*]] = arith.constant 2 : index
+// CHECK:  %[[C0:.*]] = arith.constant 0 : index
+// CHECK:  %[[S0:.*]] = tensor.empty()
+// CHECK:  %[[S1:.*]] = scf.for %[[ARG4:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG5:.*]] = %[[S0]])
+// CHECK:    %[[S9:.*]] = scf.for %[[ARG6:.*]] = %[[C0]] to %[[C5]] step %[[C1]] iter_args(%[[ARG7:.*]] = %[[ARG5]])
+// CHECK:      %[[EXTRACTED_SLICE_9:.*]] = tensor.extract_slice %[[ARG1]][%[[ARG4]], 0, 0, %[[ARG6]]] [1, 3, 3, 1] [1, 1, 1, 1]
+// CHECK:      %[[S11:.*]] = linalg.matmul
+// CHECK:      %[[S13:.*]] = linalg.matmul
+// CHECK:      %[[INSERTED_SLICE:.*]] = tensor.insert_slice %[[S13]] into %[[ARG7]][0, 0, %[[ARG6]], %[[ARG4]]] [6, 6, 1, 1] [1, 1, 1, 1]
+// CHECK:      scf.yield %[[INSERTED_SLICE]] : tensor<6x6x5x2xf32>
+// CHECK:    scf.yield %[[S9]] : tensor<6x6x5x2xf32>
+// CHECK:  %[[PADDED:.*]] = tensor.pad %[[ARG0]] low[0, 0, 0, 0] high[0, 3, 3, 0]
+// CHECK:  %[[S2:.*]] = tensor.empty() : tensor<6x6x3x3x2x5xf32>
+// CHECK:  %[[S3:.*]] = tensor.empty() : tensor<6x6x3x3x2x5xf32>
+// CHECK:  %[[S4:.*]] = scf.for %[[ARG4:.*]] = %[[C0]] to %[[C3]] step %[[C1]] iter_args(%[[ARG5:.*]] = %[[S3]])
+// CHECK:    %[[S9:.*]] = scf.for %[[ARG6:.*]] = %[[C0]] to %[[C3]] step %[[C1]] iter_args(%[[ARG7:.*]] = %[[ARG5]])
+// CHECK:      %[[S10:.*]] = affine.apply #[[$MAP0]](%[[ARG4]])
+// CHECK:      %[[S11:.*]] = affine.apply #[[$MAP0]](%[[ARG6]])
+// CHECK:      %[[EXTRACTED_SLICE_9:.*]] = tensor.extract_slice %[[PADDED]][0, %[[S10]], %[[S11]], 0] [2, 6, 6, 5] [1, 1, 1, 1]
+// CHECK:      %[[EXTRACTED_SLICE_10:.*]] = tensor.extract_slice %[[S2]][0, 0, %[[ARG4]], %[[ARG6]], 0, 0] [6, 6, 1, 1, 2, 5] [1, 1, 1, 1, 1, 1]
+// CHECK:      %[[S12:.*]] = scf.for %[[ARG8:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG9:.*]] = %[[EXTRACTED_SLICE_10]])
+// CHECK:        %[[S13:.*]] = scf.for %[[ARG10:.*]] = %[[C0]] to %[[C5]] step %[[C1]] iter_args(%[[ARG11:.*]] = %[[ARG9]])
+// CHECK:          %[[EXTRACTED_SLICE_11:.*]] = tensor.extract_slice %[[EXTRACTED_SLICE_9]][%[[ARG8]], 0, 0, %[[ARG10]]] [1, 6, 6, 1] [1, 1, 1, 1]
+// CHECK:          %[[S15:.*]] = linalg.matmul
+// CHECK:          %[[S17:.*]] = linalg.matmul
+// CHECK:          %[[INSERTED_SLICE_12:.*]] = tensor.insert_slice %[[S17]] into %[[ARG11]][0, 0, 0, 0, %[[ARG8]], %[[ARG10]]] [6, 6, 1, 1, 1, 1] [1, 1, 1, 1, 1, 1]
+// CHECK:          scf.yield %[[INSERTED_SLICE_12]] : tensor<6x6x1x1x2x5xf32>
+// CHECK:        scf.yield %[[S13]] : tensor<6x6x1x1x2x5xf32>
+// CHECK:      %[[INSERTED_SLICE:.*]] = tensor.insert_slice %[[S12]] into %[[ARG7]][0, 0, %[[ARG4]], %[[ARG6]], 0, 0] [6, 6, 1, 1, 2, 5] [1, 1, 1, 1, 1, 1]
+// CHECK:      scf.yield %[[INSERTED_SLICE]]
+// CHECK:    scf.yield %[[S9]]
+// CHECK:  %[[COLLAPSED:.*]] = tensor.collapse_shape %[[S1]] {{\[}}[0, 1], [2], [3]]
+// CHECK:  %[[COLLAPSED_7:.*]] = tensor.collapse_shape %[[S4]] {{\[}}[0, 1], [2, 3, 4], [5]]
+// CHECK:  %[[S6:.*]] = linalg.batch_matmul
+// CHECK:  %[[EXPANDED:.*]] = tensor.expand_shape %[[S6]] {{\[}}[0, 1], [2, 3, 4], [5]] output_shape [6, 6, 3, 3, 2, 2]
+// CHECK:  %[[PADDED_8:.*]] = tensor.pad %[[ARG2]] low[0, 0, 0, 0] high[0, 3, 3, 0]
+// CHECK:  %[[S7:.*]] = tensor.empty() : tensor<2x12x12x2xf32>
+// CHECK:  %[[S8:.*]] = scf.for %[[ARG4:.*]] = %[[C0]] to %[[C3]] step %[[C1]] iter_args(%[[ARG5:.*]] = %[[S7]])
+// CHECK:    %[[S9:.*]] = scf.for %[[ARG6:.*]] = %[[C0]] to %[[C3]] step %[[C1]] iter_args(%[[ARG7:.*]] = %[[ARG5]])
+// CHECK:      %[[EXTRACTED_SLICE_9:.*]] = tensor.extract_slice %[[EXPANDED]][0, 0, %[[ARG4]], %[[ARG6]], 0, 0] [6, 6, 1, 1, 2, 2] [1, 1, 1, 1, 1, 1]
+// CHECK:      %[[S10:.*]] = affine.apply #[[$MAP0]](%[[ARG4]])
+// CHECK:      %[[S11:.*]] = affine.apply #[[$MAP0]](%[[ARG6]])
+// CHECK:      %[[EXTRACTED_SLICE_10:.*]] = tensor.extract_slice %[[PADDED_8]][0, %[[S10]], %[[S11]], 0] [2, 4, 4, 2] [1, 1, 1, 1]
+// CHECK:      %[[S12:.*]] = scf.for %[[ARG8:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG9:.*]] = %[[EXTRACTED_SLICE_10]])
+// CHECK:        %[[S15:.*]] = scf.for %[[ARG10:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG11:.*]] = %[[ARG9]])
+// CHECK:          %[[EXTRACTED_SLICE_11:.*]] = tensor.extract_slice %[[EXTRACTED_SLICE_9]][0, 0, 0, 0, %[[ARG8]], %[[ARG10]]] [6, 6, 1, 1, 1, 1] [1, 1, 1, 1, 1, 1]
+// CHECK:          %[[S17:.*]] = linalg.matmul
+// CHECK:          %[[S19:.*]] = linalg.matmul
+// CHECK:          %[[S20:.*]] = tensor.empty() : tensor<4x4xf32>
+// CHECK:          %[[S21:.*]] = linalg.generic {indexing_maps = [#[[$MAP1]], #[[$MAP2]]], iterator_types = ["parallel", "parallel"]} ins(%[[CST]] : f32) outs(%[[S20]] : tensor<4x4xf32>) {
+// CHECK:          ^bb0(%[[IN:.*]]: f32, %[[OUT:.*]]: f32):
+// CHECK:            linalg.yield %[[IN]] : f32
+// CHECK:          } -> tensor<4x4xf32>
+// CHECK:          %[[S22:.*]] = linalg.mul ins(%[[S21]], %[[S19]] : tensor<4x4xf32>, tensor<4x4xf32>) outs(%[[S20]] : tensor<4x4xf32>) -> tensor<4x4xf32>
+// CHECK:          %[[INSERTED_SLICE_12:.*]] = tensor.insert_slice %[[S22]] into %[[ARG11]][%[[ARG8]], 0, 0, %[[ARG10]]] [1, 4, 4, 1] [1, 1, 1, 1]
+// CHECK:          scf.yield %[[INSERTED_SLICE_12]]
+// CHECK:        scf.yield %[[S15]] : tensor<2x4x4x2xf32>
+// CHECK:      %[[S13:.*]] = affine.apply #[[$MAP0]](%[[ARG4]])
+// CHECK:      %[[S14:.*]] = affine.apply #[[$MAP0]](%[[ARG6]])
+// CHECK:      %[[INSERTED_SLICE:.*]] = tensor.insert_slice %[[S12]] into %[[ARG7]][0, %[[S13]], %[[S14]], 0] [2, 4, 4, 2] [1, 1, 1, 1]
+// CHECK:      scf.yield %[[INSERTED_SLICE]]
+// CHECK:    scf.yield %[[S9]]
+// CHECK:  %[[EXTRACTED_SLICE:.*]] = tensor.extract_slice %[[S8]][0, 0, 0, 0] [2, 9, 9, 2] [1, 1, 1, 1]
+// CHECK:  return %[[EXTRACTED_SLICE]]
+
+// -----
+
+func.func @conv2d_mx1_rx1(%arg0: tensor<2x6x1x5xf32>, %arg1: tensor<2x3x1x5xf32>, %arg2: tensor<2x4x1x2xf32>) -> tensor<2x4x1x2xf32> {
+  %0 = tensor.empty() : tensor<6x1x5x2xf32>
+  %1 = linalg.winograd_filter_transform m(4) r(3) ins(%arg1 : tensor<2x3x1x5xf32>) outs(%0 : tensor<6x1x5x2xf32>) -> tensor<6x1x5x2xf32>
+  %2 = tensor.empty() : tensor<6x1x1x1x2x5xf32>
+  %3 = linalg.winograd_input_transform m(4) r(3) ins(%arg0 : tensor<2x6x1x5xf32>) outs(%2 : tensor<6x1x1x1x2x5xf32>) -> tensor<6x1x1x1x2x5xf32>
+  %collapsed = tensor.collapse_shape %1 [[0, 1], [2], [3]] : tensor<6x1x5x2xf32> into tensor<6x5x2xf32>
+  %collapsed_0 = tensor.collapse_shape %3 [[0, 1], [2, 3, 4], [5]] : tensor<6x1x1x1x2x5xf32> into tensor<6x2x5xf32>
+  %4 = tensor.empty() : tensor<6x2x2xf32>
+  %5 = linalg.batch_matmul ins(%collapsed_0, %collapsed : tensor<6x2x5xf32>, tensor<6x5x2xf32>) outs(%4 : tensor<6x2x2xf32>) -> tensor<6x2x2xf32>
+  %expanded = tensor.expand_shape %5 [[0, 1], [2, 3, 4], [5]] output_shape [6, 1, 1, 1, 2, 2] : tensor<6x2x2xf32> into tensor<6x1x1x1x2x2xf32>
+  %6 = linalg.winograd_output_transform m(4) r(3) ins(%expanded : tensor<6x1x1x1x2x2xf32>) outs(%arg2 : tensor<2x4x1x2xf32>) -> tensor<2x4x1x2xf32>
+  return %6 : tensor<2x4x1x2xf32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["linalg.winograd_filter_transform"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %2 = transform.structured.match ops{["linalg.winograd_input_transform"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %3, %loop3:2 = transform.structured.tile_using_for %2 tile_sizes [0, 0, 1, 1, 0, 0] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
+    %4 = transform.structured.match ops{["linalg.winograd_output_transform"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %5, %loop5:2 = transform.structured.tile_using_for %4 tile_sizes [0, 0, 1, 1, 0, 0] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
+    %7 = transform.structured.decompose_winograd_op %0 : (!transform.any_op) -> (!transform.any_op)
+    %8 = transform.structured.match ops{["linalg.winograd_input_transform"]} in %3 : (!transform.any_op) -> !transform.any_op
+    %9 = transform.structured.decompose_winograd_op %8 : (!transform.any_op) -> (!transform.any_op)
+    %10 = transform.structured.match ops{["linalg.winograd_output_transform"]} in %5 : (!transform.any_op) -> !transform.any_op
+    %11 = transform.structured.decompose_winograd_op %10 : (!transform.any_op) -> (!transform.any_op)
+    transform.yield
+  }
+}
+
+// CHECK: #[[$MAP0:.+]] = affine_map<(d0, d1) -> ()>
+// CHECK: #[[$MAP1:.+]] = affine_map<(d0, d1) -> (d0, d1)>
+// CHECK-LABEL: func.func @conv2d_mx1_rx1
+// CHECK-SAME:  (%[[ARG0:.*]]: tensor<2x6x1x5xf32>, %[[ARG1:.*]]: tensor<2x3x1x5xf32>, %[[ARG2:.*]]: tensor<2x4x1x2xf32>) -> tensor<2x4x1x2xf32> {
+// CHECK:   %[[CST:.*]] = arith.constant 3.200000e+01 : f32
+// CHECK:   %[[C1:.*]] = arith.constant 1 : index
+// CHECK:   %[[C5:.*]] = arith.constant 5 : index
+// CHECK:   %[[C2:.*]] = arith.constant 2 : index
+// CHECK:   %[[C0:.*]] = arith.constant 0 : index
+// CHECK:   %[[S0:.*]] = tensor.empty() : tensor<6x1x5x2xf32>
+// CHECK:   %[[S1:.*]] = scf.for %[[ARG3:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG4:.*]] = %[[S0]])
+// CHECK:     %[[S7:.*]] = scf.for %[[ARG5:.*]] = %[[C0]] to %[[C5]] step %[[C1]] iter_args(%[[ARG6:.*]] = %[[ARG4]])
+// CHECK:       %[[EXTRACTED_SLICE:.*]] = tensor.extract_slice %[[ARG1]][%[[ARG3]], 0, 0, %[[ARG5]]] [1, 3, 1, 1] [1, 1, 1, 1]
+// CHECK:       %[[S9:.*]] = linalg.matmul
+// CHECK:       %[[INSERTED_SLICE:.*]] = tensor.insert_slice %[[S9]] into %[[ARG6]][0, 0, %[[ARG5]], %[[ARG3]]] [6, 1, 1, 1] [1, 1, 1, 1]
+// CHECK:       scf.yield %[[INSERTED_SLICE]]
+// CHECK:     scf.yield %[[S7]]
+// CHECK:   %[[S2:.*]] = tensor.empty() : tensor<6x1x1x1x2x5xf32>
+// CHECK:   %[[S3:.*]] = scf.for %[[ARG3:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG4:.*]] = %[[S2]])
+// CHECK:     %[[S7:.*]] = scf.for %[[ARG5:.*]] = %[[C0]] to %[[C5]] step %[[C1]] iter_args(%[[ARG6:.*]] = %[[ARG4]])
+// CHECK:       %[[EXTRACTED_SLICE:.*]] = tensor.extract_slice %[[ARG0]][%[[ARG3]], 0, 0, %[[ARG5]]] [1, 6, 1, 1] [1, 1, 1, 1]
+// CHECK:       %[[S9:.*]] = linalg.matmul
+// CHECK:       %[[INSERTED_SLICE:.*]] = tensor.insert_slice %[[S9]] into %[[ARG6]][0, 0, 0, 0, %[[ARG3]], %[[ARG5]]] [6, 1, 1, 1, 1, 1] [1, 1, 1, 1, 1, 1]
+// CHECK:       scf.yield %[[INSERTED_SLICE]]
+// CHECK:     scf.yield %[[S7]]
+// CHECK:   %[[COLLAPSED:.*]] = tensor.collapse_shape %[[S1]] {{\[}}[0, 1], [2], [3]]
+// CHECK:   %[[COLLAPSED_3:.*]] = tensor.collapse_shape %[[S3]] {{\[}}[0, 1], [2, 3, 4], [5]]
+// CHECK:   %[[S5:.*]] = linalg.batch_matmul
+// CHECK:   %[[EXPANDED:.*]] = tensor.expand_shape %[[S5]] {{\[}}[0, 1], [2, 3, 4], [5]] output_shape [6, 1, 1, 1, 2, 2]
+// CHECK:   %[[S6:.*]] = scf.for %[[ARG3:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG4:.*]] = %[[ARG2]])
+// CHECK:     %[[S7:.*]] = scf.for %[[ARG5:.*]] = %[[C0]] to %[[C2]] step %[[C1]] iter_args(%[[ARG6:.*]] = %[[ARG4]])
+// CHECK:       %[[EXTRACTED_SLICE:.*]] = tensor.extract_slice %[[EXPANDED]][0, 0, 0, 0, %[[ARG3]], %[[ARG5]]] [6, 1, 1, 1, 1, 1] [1, 1, 1, 1, 1, 1]
+// CHECK:       %[[S9:.*]] = linalg.matmul
+// CHECK:       %[[S10:.*]] = tensor.empty() : tensor<4x1xf32>
+// CHECK:       %[[S11:.*]] = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel"]} ins(%[[CST]] : f32) outs(%[[S10]] : tensor<4x1xf32>) {
+// CHECK:       ^bb0(%[[IN:.*]]: f32, %[[OUT:.*]]: f32):
+// CHECK:         linalg.yield %[[IN]] : f32
+// CHECK:       } -> tensor<4x1xf32>
+// CHECK:       %[[S12:.*]] = linalg.mul ins(%[[S11]], %[[S9]] : tensor<4x1xf32>, tensor<4x1xf32>) outs(%[[S10]] : tensor<4x1xf32>) -> tensor<4x1xf32>
+// CHECK:       %[[INSERTED_SLICE:.*]] = tensor.insert_slice %[[S12]] into %[[ARG6]][%[[ARG3]], 0, 0, %[[ARG5]]] [1, 4, 1, 1] [1, 1, 1, 1]
+// CHECK:       scf.yield %[[INSERTED_SLICE]]
+// CHECK:     scf.yield %[[S7]]
+// CHECK:   return %[[S6]]
diff --git a/mlir/test/Dialect/Linalg/transform-tile-winograd.mlir b/mlir/test/Dialect/Linalg/transform-tile-winograd.mlir
new file mode 100644
index 000000000000000..21522a2083b463e
--- /dev/null
+++ b/mlir/test/Dialect/Linalg/transform-tile-winograd.mlir
@@ -0,0 +1,380 @@
+// RUN: mlir-opt %s -transform-interpreter --split-input-file | FileCheck %s
+
+func.func @tile_winograd_filter(%arg0: tensor<2x3x3x5xf32>, %arg1: tensor<6x6x5x2xf32>) -> tensor<6x6x5x2xf32> {
+  %0 = linalg.winograd_filter_transform m(4) r(3) ins(%arg0 : tensor<2x3x3x5xf32>) outs(%arg1 : tensor<6x6x5x2xf32>) -> tensor<6x6x5x2xf32>
+  return %0 : tensor<6x6x5x2xf32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["linalg.winograd_filter_transform"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %1, %loop1:2 = transform.structured.tile_using_for %0 tile_sizes [1, 0, 0, 1] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
+    transform.yield
+  }
+}
+
+// CHECK-LABEL: func.func @tile_winograd_filter(
+// CHECK-SAME:  %[[ARG0:.*]]: tensor<2x3x3x5xf32>, %[[ARG1:.*]]: tensor<6x6x5x2xf32>) -> tensor<6x6x5x2xf32> {
+// CHECK-DAG:  %[[C0:.*]] = arith.constant 0 : index
+// CHECK-DAG:  %[[C0_0:.*]] = arith.constant 0 : index
+// CHECK-DAG:  %[[C2:.*]] = arith.constant 2 : index
+// CHECK-DAG:  %[[C5:.*]] = arith.constant 5 : index
+// CHECK-DAG:  %[[C1:.*]] = arith.constant 1 : index
+// CHECK-DAG:  %[[C1_1:.*]] = arith.constant 1 : index
+// CHECK:  %[[S1:.*]] = scf.for %[[ARG2:.*]] = %[[C0]] to %[[C2]] step %[[C1]]
+// CHECK:    %[[S2:.*]] = scf.for %[[ARG4:.*]] = %[[C0_0]] to %[[C5]] step %[[C1_1]]
+// CHECK:      %[[EXTRACTED_SLICE:.*]] = tensor.extract_slice %[[ARG0]][%[[ARG2]], 0, 0, %[[ARG4]]] [1, 3, 3, 1] [1, 1, 1, 1] : tensor<2x3x3x5xf32> to tensor<1x3x3x1xf32>
+// CHECK:      %[[EXTRACTED_SLICE_2:.*]] = tensor.extract_slice %[[ARG1]][0, 0, %[[ARG4]], %[[ARG2]]] [6, 6, 1, 1] [1, 1, 1, 1] : tensor<6x6x5x2xf32> to tensor<6x6x1x1xf32>
+// CHECK:      %[[S3:.*]] = linalg.winograd_filter_transform m(4) r(3) ins(%[[EXTRACTED_SLICE]] : tensor<1x3x3x1xf32>) outs(%[[EXTRACTED_SLICE_2]] : tensor<6x6x1x1xf32>) -> tensor<6x6x1x1xf32>
+
+// -----
+
+func.func @tile_winograd_filter(%arg0: tensor<2x3x3x5xf32>, %arg1: tensor<6x6x5x2xf32>) -> tensor<6x6x5x2xf32> {
+  %0 = linalg.winograd_filter_transform m(4) r(3) ins(%arg0 : tensor<2x3x3x5xf32>) outs(%arg1 : tensor<6x6x5x2xf32>) -> tensor<6x6x5x2xf32>
+  return %0 : tensor<6x6x5x2xf32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["linalg.winograd_filter_transform"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %1, %loop1:2 = transform.structured.tile_using_for %0 tile_sizes [1, 0, 0, 2] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
+    transform.yield
+  }
+}
+
+// CHECK: #[[$MAP0:.+]] = affine_map<(d0) -> (-d0 + 5, 2)>
+// CHECK-LABEL: func.func @tile_winograd_filter(
+// CHECK-SAME:  %[[ARG0:.*]]: tensor<2x3x3x5xf32>, %[[ARG1:.*]]: tensor<6x6x5x2xf32>) -> tensor<6x6x5x2xf32> {
+// CHECK-DAG:   %[[C0:.*]] = arith.constant 0 : index
+// CHECK-DAG:   %[[C0_0:.*]] = arith.constant 0 : index
+// CHECK-DAG:   %[[C2:.*]] = arith.constant 2 : index
+// CHECK-DAG:   %[[C5:.*]] = arith.constant 5 : index
+// CHECK-DAG:   %[[C1:.*]] = arith.constant 1 : index
+// CHECK-DAG:   %[[C2_1:.*]] = arith.constant 2 : index
+// CHECK:   %[[S1:.*]] = scf.for %[[ARG2:.*]] = %[[C0]] to %[[C2]] step %[[C1]]
+// CHECK:     %[[S2:.*]] = scf.for %[[ARG4:.*]] = %[[C0_0]] to %[[C5]] step %[[C2_1]]
+// CHECK:       %[[C5_2:.*]] = arith.constant 5 : index
+// CHECK:       %[[S3:.*]] = affine.min #[[$MAP0]](%[[ARG4]])
+// CHECK:       %[[EXTRACTED_SLICE:.*]] = tensor.extract_slice %[[ARG0]][%[[ARG2]], 0, 0, %[[ARG4]]] [1, 3, 3, %[[S3]]] [1, 1, 1, 1] : tensor<2x3x3x5xf32> to tensor<1x3x3x?xf32>
+// CHECK:       %[[EXTRACTED_SLICE_3:.*]] = tensor.extract_slice %[[ARG1]][0, 0, %[[ARG4]], %[[ARG2]]] [6, 6, %[[S3]], 1] [1, 1, 1, 1] : tensor<6x6x5x2xf32> to tensor<6x6x?x1xf32>
+// CHECK:       %[[S4:.*]] = linalg.winograd_filter_transform m(4) r(3) ins(%[[EXTRACTED_SLICE]] : tensor<1x3x3x?xf32>) outs(%[[EXTRACTED_SLICE_3]] : tensor<6x6x?x1xf32>) -> tensor<6x6x?x1xf32>
+
+// -----
+
+func.func @tile_winograd_filter(%arg0: tensor<2x3x1x5xf32>, %arg1: tensor<6x1x5x2xf32>) -> tensor<6x1x5x2xf32> {
+  %0 = linalg.winograd_filter_transform m(4) r(3) ins(%arg0 : tensor<2x3x1x5xf32>) outs(%arg1 : tensor<6x1x5x2xf32>) -> tensor<6x1x5x2xf32>
+  return %0 : tensor<6x1x5x2xf32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["linalg.winograd_filter_transform"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %1, %loop1:2 = transform.structured.tile_using_for %0 tile_sizes [1, 0, 0, 1] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
+    transform.yield
+  }
+}
+
+// CHECK-LABEL: func.func @tile_winograd_filter(
+// CHECK-SAME:  %[[ARG0:.*]]: tensor<2x3x1x5xf32>, %[[ARG1:.*]]: tensor<6x1x5x2xf32>) -> tensor<6x1x5x2xf32> {
+// CHECK-DAG:   %[[C0:.*]] = arith.constant 0 : index
+// CHECK-DAG:   %[[C0_0:.*]] = arith.constant 0 : index
+// CHECK-DAG:   %[[C2:.*]] = arith.constant 2 : index
+// CHECK-DAG:   %[[C5:.*]] = arith.constant 5 : index
+// CHECK-DAG:   %[[C1:.*]] = arith.constant 1 : index
+// CHECK-DAG:   %[[C1_1:.*]] = arith.constant 1 : index
+// CHECK:   %[[S1:.*]] = scf.for %[[ARG2:.*]] = %[[C0]] to %[[C2]] step %[[C1]]
+// CHECK:     %[[S2:.*]] = scf.for %[[ARG4:.*]] = %[[C0_0]] to %[[C5]] step %[[C1_1]]
+// CHECK:       %[[EXTRACTED_SLICE:.*]] = tensor.extract_slice %[[ARG0]][%[[ARG2]], 0, 0, %[[ARG4]]] [1, 3, 1, 1] [1, 1, 1, 1] : tensor<2x3x1x5xf32> to tensor<1x3x1x1xf32>
+// CHECK:       %[[EXTRACTED_SLICE_2:.*]] = tensor.extract_slice %[[ARG1]][0, 0, %[[ARG4]], %[[ARG2]]] [6, 1, 1, 1] [1, 1, 1, 1] : tensor<6x1x5x2xf32> to tensor<6x1x1x1xf32>
+// CHECK:       %[[S3:.*]] = linalg.winograd_filter_transform m(4) r(3) ins(%[[EXTRACTED_SLICE]] : tensor<1x3x1x1xf32>) outs(%[[EXTRACTED_SLICE_2]] : tensor<6x1x1x1xf32>) -> tensor<6x1x1x1xf32>
+
+// -----
+
+func.func @tile_winograd_input(%arg0: tensor<2x10x10x5xf32>, %arg1: tensor<6x6x2x2x2x5xf32>) -> tensor<6x6x2x2x2x5xf32> {
+  %0 = linalg.winograd_input_transform m(4) r(3) ins(%arg0 : tensor<2x10x10x5xf32>) outs(%arg1 : tensor<6x6x2x2x2x5xf32>) -> tensor<6x6x2x2x2x5xf32>
+  return %0 : tensor<6x6x2x2x2x5xf32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["linalg.winograd_input_transform"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %1, %loop3:2 = transform.structured.tile_using_for %0 tile_sizes [0, 0, 1, 1, 0, 0] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
+    transform.yield
+  }
+}
+
+// CHECK: #[[$MAP0:.+]] = affine_map<(d0) -> (d0 * 4)>
+// CHECK: #[[$MAP1:.+]] = affine_map<() -> (6)>
+// CHECK-LABEL: func.func @tile_winograd_input(
+// CHECK-SAME:  %[[ARG0:.*]]: tensor<2x10x10x5xf32>, %[[ARG1:.*]]: tensor<6x6x2x2x2x5xf32>) -> tensor<6x6x2x2x2x5xf32> {
+// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index
+// CHECK-DAG: %[[C0_0:.*]] = arith.constant 0 : index
+// CHECK-DAG: %[[C2:.*]] = arith.constant 2 : index
+// CHECK-DAG: %[[C2_1:.*]] = arith.constant 2 : index
+// CHECK-DAG: %[[C1:.*]] = arith.constant 1 : index
+// CHECK-DAG: %[[C1_2:.*]] = arith.constant 1 : index
+// CHECK: %[[S1:.*]] = scf.for %[[ARG2:.*]] = %[[C0]] to %[[C2]] step %[[C1]]
+// CHECK: %[[S2:.*]] = scf.for %[[ARG4:.*]] = %[[C0_0]] to %[[C2_1]] step %[[C1_2]]
+// CHECK:   %[[S3:.*]] = affine.apply #[[$MAP0]](%[[ARG2]])
+// CHECK:   %[[S4:.*]] = affine.apply #[[$MAP0]](%[[ARG4]])
+// CHECK:   %[[S5:.*]] = affine.apply #[[$MAP1]]()
+// CHECK:   %[[S6:.*]] = affine.apply #[[$MAP1]]()
+// CHECK:   %[[EXTRACTED_SLICE:.*]] = tensor.extract_slice %[[ARG0]][0, %[[S3]], %[[S4]], 0] [2, %[[S5]], %[[S6]], 5] [1, 1, 1, 1] : tensor<2x10x10x5xf32> to tensor<2x?x?x5xf32>
+// CHECK:   %[[EXTRACTED_SLICE_5:.*]] = tensor.extract_slice %[[ARG1]][0, 0, %[[ARG2]], %[[ARG4]], 0, 0] [6, 6, 1, 1, 2, 5] [1, 1, 1, 1, 1, 1] : tensor<6x6x2x2x2x5xf32> to tensor<6x6x1x1x2x5xf32>
+// CHECK:   %[[S7:.*]] = linalg.winograd_input_transform m(4) r(3) ins(%[[EXTRACTED_SLICE]] : tensor<2x?x?x5xf32>) outs(%[[EXTRACTED_SLICE_5]] : tensor<6x6x1x1x2x5xf32>) -> tensor<6x6x1x1x2x5xf32>
+
+// -----
+
+func.func @tile_winograd_input(%arg0: tensor<2x10x10x5xf32>, %arg1: tensor<6x6x2x2x2x5xf32>) -> tensor<6x6x2x2x2x5xf32> {
+  %0 = linalg.winograd_input_transform m(4) r(3) ins(%arg0 : tensor<2x10x10x5xf32>) outs(%arg1 : tensor<6x6x2x2x2x5xf32>) -> tensor<6x6x2x2x2x5xf32>
+  return %0 : tensor<6x6x2x2x2x5xf32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["linalg.winograd_input_transform"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %1, %loop3:4 = transform.structured.tile_using_for %0 tile_sizes [0, 0, 1, 1, 1, 1] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
+    transform.yield
+  }
+}
+
+// CHECK: #[[$MAP0:.+]] = affine_map<(d0) -> (d0 * 4)>
+// CHECK: #[[$MAP1:.+]] = affine_map<() -> (6)>
+// CHECK-LABEL: func.func @tile_winograd_input(
+// CHECK-SAME:  %[[ARG0:.*]]: tensor<2x10x10x5xf32>, %[[ARG1:.*]]: tensor<6x6x2x2x2x5xf32>) -> tensor<6x6x2x2x2x5xf32> {
+// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index
+// CHECK-DAG: %[[C0_0:.*]] = arith.constant 0 : index
+// CHECK-DAG: %[[C0_3:.*]] = arith.constant 0 : index
+// CHECK-DAG: %[[C0_6:.*]] = arith.constant 0 : index
+// CHECK-DAG: %[[C2:.*]] = arith.constant 2 : index
+// CHECK-DAG: %[[C2_1:.*]] = arith.constant 2 : index
+// CHECK-DAG: %[[C2_4:.*]] = arith.constant 2 : index
+// CHECK-DAG: %[[C5:.*]] = arith.constant 5 : index
+// CHECK-DAG: %[[C1:.*]] = arith.constant 1 : index
+// CHECK-DAG: %[[C1_2:.*]] = arith.constant 1 : index
+// CHECK-DAG: %[[C1_5:.*]] = arith.constant 1 : index
+// CHECK-DAG: %[[C1_7:.*]] = arith.constant 1 : index
+// CHECK: %[[S1:.*]] = scf.for %[[ARG2:.*]] = %[[C0]] to %[[C2]] step %[[C1]]
+// CHECK:   %[[S2:.*]] = scf.for %[[ARG4:.*]] = %[[C0_0]] to %[[C2_1]] step %[[C1_2]]
+// CHECK:     %[[S3:.*]] = scf.for %[[ARG6:.*]] = %[[C0_3]] to %[[C2_4]] step %[[C1_5]]
+// CHECK:       %[[S4:.*]] = scf.for %[[ARG8:.*]] = %[[C0_6]] to %[[C5]] step %[[C1_7]]
+// CHECK:         %[[S5:.*]] = affine.apply #[[$MAP0]](%[[ARG2]])
+// CHECK:         %[[S6:.*]] = affine.apply #[[$MAP0]](%[[ARG4]])
+// CHECK:         %[[S7:.*]] = affine.apply #[[$MAP1]]()
+// CHECK:         %[[S8:.*]] = affine.apply #[[$MAP1]]()
+// CHECK:         %[[EXTRACTED_SLICE:.*]] = tensor.extract_slice %[[ARG0]][%[[ARG6]], %[[S5]], %[[S6]], %[[ARG8]]] [1, %[[S7]], %[[S8]], 1] [1, 1, 1, 1] : tensor<2x10x10x5xf32> to tensor<1x?x?x1xf32>
+// CHECK:         %[[EXTRACTED_SLICE_10:.*]] = tensor.extract_slice %[[ARG1]][0, 0, %[[ARG2]], %[[ARG4]], %[[ARG6]], %[[ARG8]]] [6, 6, 1, 1, 1, 1] [1, 1, 1, 1, 1, 1] : tensor<6x6x2x2x2x5xf32> to tensor<6x6x1x1x1x1xf32>
+// CHECK:         %[[S9:.*]] = linalg.winograd_input_transform m(4) r(3) ins(%[[EXTRACTED_SLICE]] : tensor<1x?x?x1xf32>) outs(%[[EXTRACTED_SLICE_10]] : tensor<6x6x1x1x1x1xf32>) -> tensor<6x6x1x1x1x1xf32>
+
+// -----
+
+func.func @tile_winograd_input(%arg0: tensor<2x10x10x5xf32>, %arg1: tensor<6x6x2x2x2x5xf32>) -> tensor<6x6x2x2x2x5xf32> {
+  %0 = linalg.winograd_input_transform m(4) r(3) ins(%arg0 : tensor<2x10x10x5xf32>) outs(%arg1 : tensor<6x6x2x2x2x5xf32>) -> tensor<6x6x2x2x2x5xf32>
+  return %0 : tensor<6x6x2x2x2x5xf32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["linalg.winograd_input_transform"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %1, %loop3:4 = transform.structured.tile_using_for %0 tile_sizes [0, 0, 2, 2, 2, 2] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
+    transform.yield
+  }
+}
+
+// CHECK: #[[$MAP0:.+]] = affine_map<(d0) -> (-d0 + 5, 2)>
+// CHECK: #[[$MAP1:.+]] = affine_map<(d0) -> (d0 * 4)>
+// CHECK: #[[$MAP2:.+]] = affine_map<() -> (10)>
+// CHECK-LABEL: func.func @tile_winograd_input(
+// CHECK-SAME:  %[[ARG0:.*]]: tensor<2x10x10x5xf32>, %[[ARG1:.*]]: tensor<6x6x2x2x2x5xf32>) -> tensor<6x6x2x2x2x5xf32> {
+// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index
+// CHECK-DAG: %[[C0_1:.*]] = arith.constant 0 : index
+// CHECK-DAG: %[[C0_4:.*]] = arith.constant 0 : index
+// CHECK-DAG: %[[C0_7:.*]] = arith.constant 0 : index
+// CHECK-DAG: %[[C2:.*]] = arith.constant 2 : index
+// CHECK-DAG: %[[C2_2:.*]] = arith.constant 2 : index
+// CHECK-DAG: %[[C2_5:.*]] = arith.constant 2 : index
+// CHECK-DAG: %[[C5:.*]] = arith.constant 5 : index
+// CHECK-DAG: %[[C2_0:.*]] = arith.constant 2 : index
+// CHECK-DAG: %[[C2_3:.*]] = arith.constant 2 : index
+// CHECK-DAG: %[[C2_6:.*]] = arith.constant 2 : index
+// CHECK-DAG: %[[C2_8:.*]] = arith.constant 2 : index
+// CHECK: %[[S1:.*]] = scf.for %[[ARG2:.*]] = %[[C0]] to %[[C2]] step %[[C2_0]]
+// CHECK:   %[[S2:.*]] = scf.for %[[ARG4:.*]] = %[[C0_1]] to %[[C2_2]] step %[[C2_3]]
+// CHECK:     %[[S3:.*]] = scf.for %[[ARG6:.*]] = %[[C0_4]] to %[[C2_5]] step %[[C2_6]]
+// CHECK:       %[[S4:.*]] = scf.for %[[ARG8:.*]] = %[[C0_7]] to %[[C5]] step %[[C2_8]]
+// CHECK:         %[[S5:.*]] = affine.min #[[$MAP0]](%[[ARG8]])
+// CHECK:         %[[S6:.*]] = affine.apply #[[$MAP1]](%[[ARG2]])
+// CHECK:         %[[S7:.*]] = affine.apply #[[$MAP1]](%[[ARG4]])
+// CHECK:         %[[S8:.*]] = affine.apply #[[$MAP2]]()
+// CHECK:         %[[S9:.*]] = affine.apply #[[$MAP2]]()
+// CHECK:         %[[EXTRACTED_SLICE:.*]] = tensor.extract_slice %[[ARG0]][%[[ARG6]], %[[S6]], %[[S7]], %[[ARG8]]] [2, %[[S8]], %[[S9]], %[[S5]]] [1, 1, 1, 1] : tensor<2x10x10x5xf32> to tensor<2x?x?x?xf32>
+// CHECK:         %[[EXTRACTED_SLICE_12:.*]] = tensor.extract_slice %[[ARG1]][0, 0, %[[ARG2]], %[[ARG4]], %[[ARG6]], %[[ARG8]]] [6, 6, 2, 2, 2, %[[S5]]] [1, 1, 1, 1, 1, 1] : tensor<6x6x2x2x2x5xf32> to tensor<6x6x2x2x2x?xf32>
+// CHECK:         %[[S10:.*]] = linalg.winograd_input_transform m(4) r(3) ins(%[[EXTRACTED_SLICE]] : tensor<2x?x?x?xf32>) outs(%[[EXTRACTED_SLICE_12]] : tensor<6x6x2x2x2x?xf32>) -> tensor<6x6x2x2x2x?xf32>
+
+// -----
+
+func.func @tile_winograd_input(%arg0: tensor<2x1x10x5xf32>, %arg1: tensor<1x6x1x2x2x5xf32>) -> tensor<1x6x1x2x2x5xf32> {
+  %0 = linalg.winograd_input_transform m(4) r(3) ins(%arg0 : tensor<2x1x10x5xf32>) outs(%arg1 : tensor<1x6x1x2x2x5xf32>) -> tensor<1x6x1x2x2x5xf32>
+  return %0 : tensor<1x6x1x2x2x5xf32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["linalg.winograd_input_transform"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %1, %loop3:4 = transform.structured.tile_using_for %0 tile_sizes [0, 0, 1, 1, 1, 1] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
+    transform.yield
+  }
+}
+
+// CHECK: #[[$MAP0:.+]] = affine_map<(d0) -> (d0 * 4)>
+// CHECK: #[[$MAP1:.+]] = affine_map<() -> (6)>
+// CHECK-LABEL: func.func @tile_winograd_input(
+// CHECK-SAME:  %[[ARG0:.*]]: tensor<2x1x10x5xf32>, %[[ARG1:.*]]: tensor<1x6x1x2x2x5xf32>) -> tensor<1x6x1x2x2x5xf32> {
+// CHECK-DAG:   %[[C0:.*]] = arith.constant 0 : index
+// CHECK-DAG:   %[[C0_1:.*]] = arith.constant 0 : index
+// CHECK-DAG:   %[[C0_3:.*]] = arith.constant 0 : index
+// CHECK-DAG:   %[[C0_6:.*]] = arith.constant 0 : index
+// CHECK-DAG:   %[[C1:.*]] = arith.constant 1 : index
+// CHECK-DAG:   %[[C2:.*]] = arith.constant 2 : index
+// CHECK-DAG:   %[[C2_4:.*]] = arith.constant 2 : index
+// CHECK-DAG:   %[[C5:.*]] = arith.constant 5 : index
+// CHECK-DAG:   %[[C1_0:.*]] = arith.constant 1 : index
+// CHECK-DAG:   %[[C1_2:.*]] = arith.constant 1 : index
+// CHECK-DAG:   %[[C1_5:.*]] = arith.constant 1 : index
+// CHECK-DAG:   %[[C1_7:.*]] = arith.constant 1 : index
+// CHECK:   %[[S1:.*]] = scf.for %[[ARG2:.*]] = %[[C0]] to %[[C1]] step %[[C1_0]]
+// CHECK:     %[[S2:.*]] = scf.for %[[ARG4:.*]] = %[[C0_1]] to %[[C2]] step %[[C1_2]]
+// CHECK:       %[[S3:.*]] = scf.for %[[ARG6:.*]] = %[[C0_3]] to %[[C2_4]] step %[[C1_5]]
+// CHECK:         %[[S4:.*]] = scf.for %[[ARG8:.*]] = %[[C0_6]] to %[[C5]] step %[[C1_7]]
+// CHECK:           %[[S5:.*]] = affine.apply #[[$MAP0]](%[[ARG2]])
+// CHECK:           %[[S6:.*]] = affine.apply #[[$MAP0]](%[[ARG4]])
+// CHECK:           %[[S7:.*]] = affine.apply #[[$MAP1]]()
+// CHECK:           %[[S8:.*]] = affine.apply #[[$MAP1]]()
+// CHECK:           %[[EXTRACTED_SLICE:.*]] = tensor.extract_slice %[[ARG0]][%[[ARG6]], 0, %[[S6]], %[[ARG8]]] [1, 1, %[[S8]], 1] [1, 1, 1, 1] : tensor<2x1x10x5xf32> to tensor<1x1x?x1xf32>
+// CHECK:           %[[EXTRACTED_SLICE_10:.*]] = tensor.extract_slice %[[ARG1]][0, 0, %[[ARG2]], %[[ARG4]], %[[ARG6]], %[[ARG8]]] [1, 6, 1, 1, 1, 1] [1, 1, 1, 1, 1, 1] : tensor<1x6x1x2x2x5xf32> to tensor<1x6x1x1x1x1xf32>
+// CHECK:           %[[S9:.*]] = linalg.winograd_input_transform m(4) r(3) ins(%[[EXTRACTED_SLICE]] : tensor<1x1x?x1xf32>) outs(%[[EXTRACTED_SLICE_10]] : tensor<1x6x1x1x1x1xf32>) -> tensor<1x6x1x1x1x1xf32>
+
+// -----
+
+func.func @tile_winograd_output(%arg0 : tensor<6x6x2x2x2x2xf32>, %arg1: tensor<2x8x8x2xf32>) -> tensor<2x8x8x2xf32> {
+  %0 = linalg.winograd_output_transform m(4) r(3) ins(%arg0 : tensor<6x6x2x2x2x2xf32>) outs(%arg1 : tensor<2x8x8x2xf32>) -> tensor<2x8x8x2xf32>
+  return %0 : tensor<2x8x8x2xf32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["linalg.winograd_output_transform"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %1, %loop1:2 = transform.structured.tile_using_for %0 tile_sizes [0, 0, 1, 1, 0, 0] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
+    transform.yield
+  }
+}
+
+// CHECK: #[[$MAP0:.+]] = affine_map<(d0) -> (d0 * 4)>
+// CHECK: #[[$MAP1:.+]] = affine_map<() -> (4)>
+// CHECK-LABEL: func.func @tile_winograd_output(
+// CHECK-SAME:  %[[ARG0:.*]]: tensor<6x6x2x2x2x2xf32>, %[[ARG1:.*]]: tensor<2x8x8x2xf32>) -> tensor<2x8x8x2xf32> {
+// CHECK-DAG:   %[[C0:.*]] = arith.constant 0 : index
+// CHECK-DAG:   %[[C0_0:.*]] = arith.constant 0 : index
+// CHECK-DAG:   %[[C2:.*]] = arith.constant 2 : index
+// CHECK-DAG:   %[[C2_1:.*]] = arith.constant 2 : index
+// CHECK-DAG:   %[[C1:.*]] = arith.constant 1 : index
+// CHECK-DAG:   %[[C1_2:.*]] = arith.constant 1 : index
+// CHECK:   %[[S1:.*]] = scf.for %[[ARG2:.*]] = %[[C0]] to %[[C2]] step %[[C1]]
+// CHECK:     %[[S2:.*]] = scf.for %[[ARG4:.*]] = %[[C0_0]] to %[[C2_1]] step %[[C1_2]]
+// CHECK:       %[[EXTRACTED_SLICE:.*]] = tensor.extract_slice %[[ARG0]][0, 0, %[[ARG2]], %[[ARG4]], 0, 0] [6, 6, 1, 1, 2, 2] [1, 1, 1, 1, 1, 1] : tensor<6x6x2x2x2x2xf32> to tensor<6x6x1x1x2x2xf32>
+// CHECK:       %[[S3:.*]] = affine.apply #[[$MAP0]](%[[ARG2]])
+// CHECK:       %[[S4:.*]] = affine.apply #[[$MAP0]](%[[ARG4]])
+// CHECK:       %[[S5:.*]] = affine.apply #[[$MAP1]]()
+// CHECK:       %[[S6:.*]] = affine.apply #[[$MAP1]]()
+// CHECK:       %[[EXTRACTED_SLICE_5:.*]] = tensor.extract_slice %[[ARG1]][0, %[[S3]], %[[S4]], 0] [2, %[[S5]], %[[S6]], 2] [1, 1, 1, 1] : tensor<2x8x8x2xf32> to tensor<2x?x?x2xf32>
+
+// -----
+
+func.func @tile_winograd_output(%arg0 : tensor<6x6x2x2x3x5xf32>, %arg1: tensor<3x8x8x5xf32>) -> tensor<3x8x8x5xf32> {
+  %0 = linalg.winograd_output_transform m(4) r(3) ins(%arg0 : tensor<6x6x2x2x3x5xf32>) outs(%arg1 : tensor<3x8x8x5xf32>) -> tensor<3x8x8x5xf32>
+  return %0 : tensor<3x8x8x5xf32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["linalg.winograd_output_transform"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %1, %loop1:4 = transform.structured.tile_using_for %0 tile_sizes [0, 0, 2, 2, 2, 2] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
+    transform.yield
+  }
+}
+
+// CHECK: #[[$MAP0:.+]] = affine_map<(d0) -> (-d0 + 3, 2)>
+// CHECK: #[[$MAP1:.+]] = affine_map<(d0) -> (-d0 + 5, 2)>
+// CHECK: #[[$MAP2:.+]] = affine_map<(d0) -> (d0 * 4)>
+// CHECK: #[[$MAP3:.+]] = affine_map<() -> (8)>
+// CHECK-LABEL:  func.func @tile_winograd_output(
+// CHECK-SAME:   %[[ARG0:.*]]: tensor<6x6x2x2x3x5xf32>, %[[ARG1:.*]]: tensor<3x8x8x5xf32>) -> tensor<3x8x8x5xf32> {
+// CHECK-DAG:    %[[C0:.*]] = arith.constant 0 : index
+// CHECK-DAG:    %[[C0_1:.*]] = arith.constant 0 : index
+// CHECK-DAG:    %[[C0_4:.*]] = arith.constant 0 : index
+// CHECK-DAG:    %[[C0_6:.*]] = arith.constant 0 : index
+// CHECK-DAG:    %[[C2:.*]] = arith.constant 2 : index
+// CHECK-DAG:    %[[C2_2:.*]] = arith.constant 2 : index
+// CHECK-DAG:    %[[C3:.*]] = arith.constant 3 : index
+// CHECK-DAG:    %[[C5:.*]] = arith.constant 5 : index
+// CHECK-DAG:    %[[C2_0:.*]] = arith.constant 2 : index
+// CHECK-DAG:    %[[C2_3:.*]] = arith.constant 2 : index
+// CHECK-DAG:    %[[C2_5:.*]] = arith.constant 2 : index
+// CHECK-DAG:    %[[C2_7:.*]] = arith.constant 2 : index
+// CHECK:    %[[S1:.*]] = scf.for %[[ARG2:.*]] = %[[C0]] to %[[C2]] step %[[C2_0]]
+// CHECK:      %[[S2:.*]] = scf.for %[[ARG4:.*]] = %[[C0_1]] to %[[C2_2]] step %[[C2_3]]
+// CHECK:        %[[S3:.*]] = scf.for %[[ARG6:.*]] = %[[C0_4]] to %[[C3]] step %[[C2_5]]
+// CHECK:          %[[S4:.*]] = scf.for %[[ARG8:.*]] = %[[C0_6]] to %[[C5]] step %[[C2_7]]
+// CHECK:            %[[C3_8:.*]] = arith.constant 3 : index
+// CHECK:            %[[S5:.*]] = affine.min #[[$MAP0]](%[[ARG6]])
+// CHECK:            %[[C5_9:.*]] = arith.constant 5 : index
+// CHECK:            %[[S6:.*]] = affine.min #[[$MAP1]](%[[ARG8]])
+// CHECK:            %[[EXTRACTED_SLICE:.*]] = tensor.extract_slice %[[ARG0]][0, 0, %[[ARG2]], %[[ARG4]], %[[ARG6]], %[[ARG8]]] [6, 6, 2, 2, %[[S5]], %[[S6]]] [1, 1, 1, 1, 1, 1] : tensor<6x6x2x2x3x5xf32> to tensor<6x6x2x2x?x?xf32>
+// CHECK:            %[[S7:.*]] = affine.apply #[[$MAP2]](%[[ARG2]])
+// CHECK:            %[[S8:.*]] = affine.apply #[[$MAP2]](%[[ARG4]])
+// CHECK:            %[[S9:.*]] = affine.apply #[[$MAP3]]()
+// CHECK:            %[[S10:.*]] = affine.apply #[[$MAP3]]()
+// CHECK:            %[[EXTRACTED_SLICE_12:.*]] = tensor.extract_slice %[[ARG1]][%[[ARG6]], %[[S7]], %[[S8]], %[[ARG8]]] [%[[S5]], %[[S9]], %[[S10]], %[[S6]]] [1, 1, 1, 1] : tensor<3x8x8x5xf32> to tensor<?x?x?x?xf32>
+
+// -----
+
+func.func @tile_winograd_output(%arg0 : tensor<6x1x2x1x3x5xf32>, %arg1: tensor<3x8x1x5xf32>) -> tensor<3x8x1x5xf32> {
+  %0 = linalg.winograd_output_transform m(4) r(3) ins(%arg0 : tensor<6x1x2x1x3x5xf32>) outs(%arg1 : tensor<3x8x1x5xf32>) -> tensor<3x8x1x5xf32>
+  return %0 : tensor<3x8x1x5xf32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["linalg.winograd_output_transform"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %1, %loop1:4 = transform.structured.tile_using_for %0 tile_sizes [0, 0, 1, 1, 1, 1] : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op, !transform.any_op)
+    transform.yield
+  }
+}
+
+// CHECK: #[[$MAP0:.+]] = affine_map<(d0) -> (d0 * 4)>
+// CHECK: #[[$MAP1:.+]] = affine_map<() -> (4)>
+// CHECK-LABEL: func.func @tile_winograd_output(
+// CHECK-SAME:  %[[ARG0:.*]]: tensor<6x1x2x1x3x5xf32>, %[[ARG1:.*]]: tensor<3x8x1x5xf32>) -> tensor<3x8x1x5xf32> {
+// CHECK-DAG:   %[[C0:.*]] = arith.constant 0 : index
+// CHECK-DAG:   %[[C0_0:.*]] = arith.constant 0 : index
+// CHECK-DAG:   %[[C0_3:.*]] = arith.constant 0 : index
+// CHECK-DAG:   %[[C0_5:.*]] = arith.constant 0 : index
+// CHECK-DAG:   %[[C2:.*]] = arith.constant 2 : index
+// CHECK-DAG:   %[[C1_1:.*]] = arith.constant 1 : index
+// CHECK-DAG:   %[[C3:.*]] = arith.constant 3 : index
+// CHECK-DAG:   %[[C5:.*]] = arith.constant 5 : index
+// CHECK-DAG:   %[[C1:.*]] = arith.constant 1 : index
+// CHECK-DAG:   %[[C1_2:.*]] = arith.constant 1 : index
+// CHECK-DAG:   %[[C1_4:.*]] = arith.constant 1 : index
+// CHECK-DAG:   %[[C1_6:.*]] = arith.constant 1 : index
+// CHECK:   %[[S1:.*]] = scf.for %[[ARG2:.*]] = %[[C0]] to %[[C2]] step %[[C1]]
+// CHECK:     %[[S2:.*]] = scf.for %[[ARG4:.*]] = %[[C0_0]] to %[[C1_1]] step %[[C1_2]]
+// CHECK:       %[[S3:.*]] = scf.for %[[ARG6:.*]] = %[[C0_3]] to %[[C3]] step %[[C1_4]]
+// CHECK:         %[[S4:.*]] = scf.for %[[ARG8:.*]] = %[[C0_5]] to %[[C5]] step %[[C1_6]]
+// CHECK:           %[[EXTRACTED_SLICE:.*]] = tensor.extract_slice %[[ARG0]][0, 0, %[[ARG2]], %[[ARG4]], %[[ARG6]], %[[ARG8]]] [6, 1, 1, 1, 1, 1] [1, 1, 1, 1, 1, 1] : tensor<6x1x2x1x3x5xf32> to tensor<6x1x1x1x1x1xf32>
+// CHECK:           %[[S5:.*]] = affine.apply #[[$MAP0]](%[[ARG2]])
+// CHECK:           %[[S6:.*]] = affine.apply #[[$MAP0]](%[[ARG4]])
+// CHECK:           %[[S7:.*]] = affine.apply #[[$MAP1]]()
+// CHECK:           %[[S8:.*]] = affine.apply #[[$MAP1]]()
+// CHECK:           %[[EXTRACTED_SLICE_9:.*]] = tensor.extract_slice %[[ARG1]][%[[ARG6]], %[[S5]], 0, %[[ARG8]]] [1, %[[S7]], 1, 1] [1, 1, 1, 1] : tensor<3x8x1x5xf32> to tensor<1x?x1x1xf32>
+// CHECK:           %[[S9:.*]] = linalg.winograd_output_transform m(4) r(3) ins(%[[EXTRACTED_SLICE]] : tensor<6x1x1x1x1x1xf32>) outs(%[[EXTRACTED_SLICE_9]] : tensor<1x?x1x1xf32>) -> tensor<1x?x1x1xf32>

From e25f6b0fa6b4f00131ae40e02d6ab89c39638f98 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Fri, 16 Aug 2024 08:33:52 -0700
Subject: [PATCH 183/441] [Bazel] Port AST/ByteCode #104552

---
 .../llvm-project-overlay/clang/BUILD.bazel     | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/utils/bazel/llvm-project-overlay/clang/BUILD.bazel b/utils/bazel/llvm-project-overlay/clang/BUILD.bazel
index 5aa6b0142b44e5e..945c3d64a991e6d 100644
--- a/utils/bazel/llvm-project-overlay/clang/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/clang/BUILD.bazel
@@ -852,14 +852,14 @@ gentbl(
 )
 
 gentbl(
-    name = "ast_interp_opcodes_gen",
+    name = "ast_bytecode_opcodes_gen",
     tbl_outs = [(
         "-gen-clang-opcodes",
-        "lib/AST/Interp/Opcodes.inc",
+        "lib/AST/ByteCode/Opcodes.inc",
     )],
     tblgen = ":clang-tblgen",
-    td_file = "lib/AST/Interp/Opcodes.td",
-    td_srcs = ["lib/AST/Interp/Opcodes.td"],
+    td_file = "lib/AST/ByteCode/Opcodes.td",
+    td_srcs = ["lib/AST/ByteCode/Opcodes.td"],
 )
 
 gentbl(
@@ -922,11 +922,11 @@ cc_library(
     srcs = glob([
         "lib/AST/*.cpp",
         "lib/AST/*.h",
-        "lib/AST/Interp/*.cpp",
-        "lib/AST/Interp/*.h",
+        "lib/AST/ByteCode/*.cpp",
+        "lib/AST/ByteCode/*.h",
     ]) + [
         "lib/AST/AttrDocTable.inc",
-        "lib/AST/Interp/Opcodes.inc",
+        "lib/AST/ByteCode/Opcodes.inc",
     ],
     hdrs = glob([
         "include/clang/AST/*.h",
@@ -942,7 +942,7 @@ cc_library(
         # this library because it contains files that collide easily with system
         # headers such as `CXXABI.h`.
         "-I$(GENDIR)/$(WORKSPACE_ROOT)/clang/lib/AST",
-        "-I$(GENDIR)/$(WORKSPACE_ROOT)/clang/lib/AST/Interp",
+        "-I$(GENDIR)/$(WORKSPACE_ROOT)/clang/lib/AST/ByteCode",
     ],
     textual_hdrs = [
         "include/clang/AST/AttrImpl.inc",
@@ -972,8 +972,8 @@ cc_library(
         ":ast_comment_html_tags_gen",
         ":ast_comment_html_tags_properties_gen",
         ":ast_comment_nodes_gen",
+        ":ast_bytecode_opcodes_gen",
         ":ast_decl_nodes_gen",
-        ":ast_interp_opcodes_gen",
         ":ast_properties_base_gen",
         ":ast_stmt_data_collectors_gen",
         ":ast_stmt_nodes_gen",

From 7aa4726f44587791ed4f41a13c68e8f499ed34f0 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Fri, 16 Aug 2024 08:35:43 -0700
Subject: [PATCH 184/441] [CodeGen] Use range-based for loops (NFC) (#104536)

---
 llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp | 4 ++--
 llvm/lib/CodeGen/TypePromotion.cpp           | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 219c60eab04f5ac..8ceb729f0c156e9 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -1463,8 +1463,8 @@ static uint64_t getOffsetFromIndices(const User &U, const DataLayout &DL) {
     for (auto Idx : IVI->indices())
       Indices.push_back(ConstantInt::get(Int32Ty, Idx));
   } else {
-    for (unsigned i = 1; i < U.getNumOperands(); ++i)
-      Indices.push_back(U.getOperand(i));
+    for (Value *Op : drop_begin(U.operands()))
+      Indices.push_back(Op);
   }
 
   return 8 * static_cast<uint64_t>(
diff --git a/llvm/lib/CodeGen/TypePromotion.cpp b/llvm/lib/CodeGen/TypePromotion.cpp
index 11aa3abe9068537..b29c46b0540cdce 100644
--- a/llvm/lib/CodeGen/TypePromotion.cpp
+++ b/llvm/lib/CodeGen/TypePromotion.cpp
@@ -665,8 +665,8 @@ void IRPromoter::Mutate() {
     } else if (auto *Switch = dyn_cast<SwitchInst>(I))
       TruncTysMap[I].push_back(Switch->getCondition()->getType());
     else {
-      for (unsigned i = 0; i < I->getNumOperands(); ++i)
-        TruncTysMap[I].push_back(I->getOperand(i)->getType());
+      for (const Value *Op : I->operands())
+        TruncTysMap[I].push_back(Op->getType());
     }
   }
   for (auto *V : Visited) {

From 66878ff69293c4008707261592d448d5896239e7 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Fri, 16 Aug 2024 08:35:59 -0700
Subject: [PATCH 185/441] [Hexagon] Use range-based for loops (NFC) (#104538)

---
 llvm/lib/Target/Hexagon/HexagonCopyHoisting.cpp   | 4 +---
 llvm/lib/Target/Hexagon/HexagonGenMemAbsolute.cpp | 8 +++-----
 2 files changed, 4 insertions(+), 8 deletions(-)

diff --git a/llvm/lib/Target/Hexagon/HexagonCopyHoisting.cpp b/llvm/lib/Target/Hexagon/HexagonCopyHoisting.cpp
index a2230289ae69c7d..59c882bf37afaa7 100644
--- a/llvm/lib/Target/Hexagon/HexagonCopyHoisting.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonCopyHoisting.cpp
@@ -249,9 +249,7 @@ void HexagonCopyHoisting::moveCopyInstr(MachineBasicBlock *DestBB,
   DestBB->splice(FirstTI, MI->getParent(), MI);
 
   addMItoCopyList(MI);
-  for (auto I = std::next(DestBB->succ_begin()), E = DestBB->succ_end(); I != E;
-       ++I) {
-    MachineBasicBlock *SuccBB = *I;
+  for (MachineBasicBlock *SuccBB : drop_begin(DestBB->successors())) {
     auto &BBCopyInst = CopyMIList[SuccBB->getNumber()];
     MachineInstr *SuccMI = BBCopyInst[Key];
     SuccMI->eraseFromParent();
diff --git a/llvm/lib/Target/Hexagon/HexagonGenMemAbsolute.cpp b/llvm/lib/Target/Hexagon/HexagonGenMemAbsolute.cpp
index 651ccc2db9ba24c..92574a16c4f9318 100644
--- a/llvm/lib/Target/Hexagon/HexagonGenMemAbsolute.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonGenMemAbsolute.cpp
@@ -86,11 +86,9 @@ bool HexagonGenMemAbsolute::runOnMachineFunction(MachineFunction &Fn) {
       getAnalysis<MachineDominatorTreeWrapperPass>().getDomTree();
 
   // Loop over all of the basic blocks
-  for (MachineFunction::iterator MBBb = Fn.begin(), MBBe = Fn.end();
-       MBBb != MBBe; ++MBBb) {
-    MachineBasicBlock *MBB = &*MBBb;
+  for (MachineBasicBlock &MBB : Fn) {
     // Traverse the basic block
-    for (MachineBasicBlock::iterator MII = MBB->begin(); MII != MBB->end();
+    for (MachineBasicBlock::iterator MII = MBB.begin(); MII != MBB.end();
          ++MII) {
       MachineInstr *MI = &*MII;
       int Opc = MI->getOpcode();
@@ -205,7 +203,7 @@ bool HexagonGenMemAbsolute::runOnMachineFunction(MachineFunction &Fn) {
 
       LLVM_DEBUG(dbgs() << "Replaced with " << *MIB << "\n");
       // Erase the instructions that got replaced.
-      MII = MBB->erase(MI);
+      MII = MBB.erase(MI);
       --MII;
       NextMI->getParent()->erase(NextMI);
     }

From d8c9d583930befff3e0d78a5ab785d33a8e95f0f Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Fri, 16 Aug 2024 08:42:36 -0700
Subject: [PATCH 186/441] [RISCV] Don't support TRUNCATE_SSAT_U. (#104468)

RISC-V doesn't have an instruction for this. We were treating it the
same as TRUNCATE_USAT_U.
---
 llvm/lib/Target/RISCV/RISCVISelLowering.cpp   | 15 ++--
 .../RISCV/rvv/fixed-vectors-trunc-sat-clip.ll | 32 ++++++---
 .../CodeGen/RISCV/rvv/fpclamptosat_vec.ll     | 68 ++++++++++++-------
 .../RISCV/rvv/trunc-sat-clip-sdnode.ll        | 32 ++++++---
 .../RISCV/rvv/trunc-select-to-max-usat.ll     | 57 +++++++++++-----
 5 files changed, 136 insertions(+), 68 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 434f22bb9c8df3f..da937960f1ea76a 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -784,9 +784,9 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
 
       // Integer VTs are lowered as a series of "RISCVISD::TRUNCATE_VECTOR_VL"
       // nodes which truncate by one power of two at a time.
-      setOperationAction({ISD::TRUNCATE, ISD::TRUNCATE_SSAT_S,
-                          ISD::TRUNCATE_SSAT_U, ISD::TRUNCATE_USAT_U},
-                         VT, Custom);
+      setOperationAction(
+          {ISD::TRUNCATE, ISD::TRUNCATE_SSAT_S, ISD::TRUNCATE_USAT_U}, VT,
+          Custom);
 
       // Custom-lower insert/extract operations to simplify patterns.
       setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT}, VT,
@@ -1102,9 +1102,9 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
 
         setOperationAction(ISD::SELECT, VT, Custom);
 
-        setOperationAction({ISD::TRUNCATE, ISD::TRUNCATE_SSAT_S,
-                            ISD::TRUNCATE_SSAT_U, ISD::TRUNCATE_USAT_U},
-                           VT, Custom);
+        setOperationAction(
+            {ISD::TRUNCATE, ISD::TRUNCATE_SSAT_S, ISD::TRUNCATE_USAT_U}, VT,
+            Custom);
 
         setOperationAction(ISD::BITCAST, VT, Custom);
 
@@ -6232,7 +6232,6 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
   }
   case ISD::TRUNCATE:
   case ISD::TRUNCATE_SSAT_S:
-  case ISD::TRUNCATE_SSAT_U:
   case ISD::TRUNCATE_USAT_U:
     // Only custom-lower vector truncates
     if (!Op.getSimpleValueType().isVector())
@@ -8119,7 +8118,7 @@ SDValue RISCVTargetLowering::lowerVectorTruncLike(SDValue Op,
   unsigned NewOpc;
   if (Opc == ISD::TRUNCATE_SSAT_S)
     NewOpc = RISCVISD::TRUNCATE_VECTOR_VL_SSAT;
-  else if (Opc == ISD::TRUNCATE_SSAT_U || Opc == ISD::TRUNCATE_USAT_U)
+  else if (Opc == ISD::TRUNCATE_USAT_U)
     NewOpc = RISCVISD::TRUNCATE_VECTOR_VL_USAT;
   else
     NewOpc = RISCVISD::TRUNCATE_VECTOR_VL;
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-trunc-sat-clip.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-trunc-sat-clip.ll
index e2f540e991fd0b1..4e367bb0d70cd1c 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-trunc-sat-clip.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-trunc-sat-clip.ll
@@ -101,8 +101,10 @@ define void @trunc_sat_u8u16_notopt(ptr %x, ptr %y) {
 define void @trunc_sat_u8u16_maxmin(ptr %x, ptr %y) {
 ; CHECK-LABEL: trunc_sat_u8u16_maxmin:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
+; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
 ; CHECK-NEXT:    vle16.v v8, (a0)
+; CHECK-NEXT:    vmax.vx v8, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v8, v8, 0
 ; CHECK-NEXT:    vse8.v v8, (a1)
 ; CHECK-NEXT:    ret
@@ -117,8 +119,10 @@ define void @trunc_sat_u8u16_maxmin(ptr %x, ptr %y) {
 define void @trunc_sat_u8u16_minmax(ptr %x, ptr %y) {
 ; CHECK-LABEL: trunc_sat_u8u16_minmax:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
+; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
 ; CHECK-NEXT:    vle16.v v8, (a0)
+; CHECK-NEXT:    vmax.vx v8, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v8, v8, 0
 ; CHECK-NEXT:    vse8.v v8, (a1)
 ; CHECK-NEXT:    ret
@@ -352,8 +356,10 @@ define void @trunc_sat_u32u64_min(ptr %x, ptr %y) {
 define void @trunc_sat_u32u64_maxmin(ptr %x, ptr %y) {
 ; CHECK-LABEL: trunc_sat_u32u64_maxmin:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; CHECK-NEXT:    vle64.v v8, (a0)
+; CHECK-NEXT:    vmax.vx v8, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v10, v8, 0
 ; CHECK-NEXT:    vse32.v v10, (a1)
 ; CHECK-NEXT:    ret
@@ -368,8 +374,10 @@ define void @trunc_sat_u32u64_maxmin(ptr %x, ptr %y) {
 define void @trunc_sat_u32u64_minmax(ptr %x, ptr %y) {
 ; CHECK-LABEL: trunc_sat_u32u64_minmax:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; CHECK-NEXT:    vle64.v v8, (a0)
+; CHECK-NEXT:    vmax.vx v8, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v10, v8, 0
 ; CHECK-NEXT:    vse32.v v10, (a1)
 ; CHECK-NEXT:    ret
@@ -437,8 +445,10 @@ define void @trunc_sat_u8u32_min(ptr %x, ptr %y) {
 define void @trunc_sat_u8u32_maxmin(ptr %x, ptr %y) {
 ; CHECK-LABEL: trunc_sat_u8u32_maxmin:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; CHECK-NEXT:    vle32.v v8, (a0)
+; CHECK-NEXT:    vmax.vx v8, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v8, v8, 0
 ; CHECK-NEXT:    vsetvli zero, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v8, v8, 0
@@ -455,8 +465,10 @@ define void @trunc_sat_u8u32_maxmin(ptr %x, ptr %y) {
 define void @trunc_sat_u8u32_minmax(ptr %x, ptr %y) {
 ; CHECK-LABEL: trunc_sat_u8u32_minmax:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; CHECK-NEXT:    vle32.v v8, (a0)
+; CHECK-NEXT:    vmax.vx v8, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v8, v8, 0
 ; CHECK-NEXT:    vsetvli zero, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v8, v8, 0
@@ -532,8 +544,10 @@ define void @trunc_sat_u8u64_min(ptr %x, ptr %y) {
 define void @trunc_sat_u8u64_maxmin(ptr %x, ptr %y) {
 ; CHECK-LABEL: trunc_sat_u8u64_maxmin:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; CHECK-NEXT:    vle64.v v8, (a0)
+; CHECK-NEXT:    vmax.vx v8, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v10, v8, 0
 ; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v8, v10, 0
@@ -552,8 +566,10 @@ define void @trunc_sat_u8u64_maxmin(ptr %x, ptr %y) {
 define void @trunc_sat_u8u64_minmax(ptr %x, ptr %y) {
 ; CHECK-LABEL: trunc_sat_u8u64_minmax:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
 ; CHECK-NEXT:    vle64.v v8, (a0)
+; CHECK-NEXT:    vmax.vx v8, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v10, v8, 0
 ; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v8, v10, 0
diff --git a/llvm/test/CodeGen/RISCV/rvv/fpclamptosat_vec.ll b/llvm/test/CodeGen/RISCV/rvv/fpclamptosat_vec.ll
index 102ca143dfc737e..1395dc914bb4021 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fpclamptosat_vec.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fpclamptosat_vec.ll
@@ -113,6 +113,7 @@ define <2 x i32> @ustest_f64i32(<2 x double> %x) {
 ; CHECK-V:       # %bb.0: # %entry
 ; CHECK-V-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; CHECK-V-NEXT:    vfcvt.rtz.x.f.v v8, v8
+; CHECK-V-NEXT:    vmax.vx v8, v8, zero
 ; CHECK-V-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; CHECK-V-NEXT:    vnclipu.wi v8, v8, 0
 ; CHECK-V-NEXT:    ret
@@ -303,6 +304,9 @@ define <4 x i32> @ustest_f32i32(<4 x float> %x) {
 ; CHECK-V:       # %bb.0: # %entry
 ; CHECK-V-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; CHECK-V-NEXT:    vfwcvt.rtz.x.f.v v10, v8
+; CHECK-V-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
+; CHECK-V-NEXT:    vmax.vx v10, v10, zero
+; CHECK-V-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; CHECK-V-NEXT:    vnclipu.wi v8, v10, 0
 ; CHECK-V-NEXT:    ret
 entry:
@@ -797,16 +801,17 @@ define <4 x i32> @ustest_f16i32(<4 x half> %x) {
 ; CHECK-V-NEXT:    call __extendhfsf2
 ; CHECK-V-NEXT:    fcvt.l.s a0, fa0, rtz
 ; CHECK-V-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
-; CHECK-V-NEXT:    vmv.s.x v10, a0
+; CHECK-V-NEXT:    vmv.s.x v8, a0
 ; CHECK-V-NEXT:    addi a0, sp, 16
-; CHECK-V-NEXT:    vl1r.v v8, (a0) # Unknown-size Folded Reload
-; CHECK-V-NEXT:    vslideup.vi v10, v8, 1
+; CHECK-V-NEXT:    vl1r.v v9, (a0) # Unknown-size Folded Reload
+; CHECK-V-NEXT:    vslideup.vi v8, v9, 1
 ; CHECK-V-NEXT:    csrr a0, vlenb
 ; CHECK-V-NEXT:    add a0, sp, a0
 ; CHECK-V-NEXT:    addi a0, a0, 16
-; CHECK-V-NEXT:    vl2r.v v8, (a0) # Unknown-size Folded Reload
+; CHECK-V-NEXT:    vl2r.v v10, (a0) # Unknown-size Folded Reload
 ; CHECK-V-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
-; CHECK-V-NEXT:    vslideup.vi v10, v8, 2
+; CHECK-V-NEXT:    vslideup.vi v8, v10, 2
+; CHECK-V-NEXT:    vmax.vx v10, v8, zero
 ; CHECK-V-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; CHECK-V-NEXT:    vnclipu.wi v8, v10, 0
 ; CHECK-V-NEXT:    csrr a0, vlenb
@@ -939,8 +944,9 @@ define <2 x i16> @ustest_f64i16(<2 x double> %x) {
 ; CHECK-V:       # %bb.0: # %entry
 ; CHECK-V-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
 ; CHECK-V-NEXT:    vfncvt.rtz.x.f.w v9, v8
+; CHECK-V-NEXT:    vmax.vx v8, v9, zero
 ; CHECK-V-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
-; CHECK-V-NEXT:    vnclipu.wi v8, v9, 0
+; CHECK-V-NEXT:    vnclipu.wi v8, v8, 0
 ; CHECK-V-NEXT:    ret
 entry:
   %conv = fptosi <2 x double> %x to <2 x i32>
@@ -1133,6 +1139,7 @@ define <4 x i16> @ustest_f32i16(<4 x float> %x) {
 ; CHECK-V:       # %bb.0: # %entry
 ; CHECK-V-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; CHECK-V-NEXT:    vfcvt.rtz.x.f.v v8, v8
+; CHECK-V-NEXT:    vmax.vx v8, v8, zero
 ; CHECK-V-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; CHECK-V-NEXT:    vnclipu.wi v8, v8, 0
 ; CHECK-V-NEXT:    ret
@@ -2107,23 +2114,24 @@ define <8 x i16> @ustest_f16i16(<8 x half> %x) {
 ; CHECK-V-NEXT:    call __extendhfsf2
 ; CHECK-V-NEXT:    fcvt.l.s a0, fa0, rtz
 ; CHECK-V-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
-; CHECK-V-NEXT:    vmv.s.x v10, a0
+; CHECK-V-NEXT:    vmv.s.x v8, a0
 ; CHECK-V-NEXT:    addi a0, sp, 16
-; CHECK-V-NEXT:    vl1r.v v8, (a0) # Unknown-size Folded Reload
-; CHECK-V-NEXT:    vslideup.vi v10, v8, 1
+; CHECK-V-NEXT:    vl1r.v v9, (a0) # Unknown-size Folded Reload
+; CHECK-V-NEXT:    vslideup.vi v8, v9, 1
 ; CHECK-V-NEXT:    csrr a0, vlenb
 ; CHECK-V-NEXT:    add a0, sp, a0
 ; CHECK-V-NEXT:    addi a0, a0, 16
-; CHECK-V-NEXT:    vl1r.v v8, (a0) # Unknown-size Folded Reload
+; CHECK-V-NEXT:    vl1r.v v9, (a0) # Unknown-size Folded Reload
 ; CHECK-V-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; CHECK-V-NEXT:    vslideup.vi v10, v8, 2
+; CHECK-V-NEXT:    vslideup.vi v8, v9, 2
 ; CHECK-V-NEXT:    csrr a0, vlenb
 ; CHECK-V-NEXT:    slli a0, a0, 1
 ; CHECK-V-NEXT:    add a0, sp, a0
 ; CHECK-V-NEXT:    addi a0, a0, 16
-; CHECK-V-NEXT:    vl2r.v v8, (a0) # Unknown-size Folded Reload
+; CHECK-V-NEXT:    vl2r.v v10, (a0) # Unknown-size Folded Reload
 ; CHECK-V-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; CHECK-V-NEXT:    vslideup.vi v10, v8, 4
+; CHECK-V-NEXT:    vslideup.vi v8, v10, 4
+; CHECK-V-NEXT:    vmax.vx v10, v8, zero
 ; CHECK-V-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; CHECK-V-NEXT:    vnclipu.wi v8, v10, 0
 ; CHECK-V-NEXT:    csrr a0, vlenb
@@ -3465,6 +3473,7 @@ define <2 x i32> @ustest_f64i32_mm(<2 x double> %x) {
 ; CHECK-V:       # %bb.0: # %entry
 ; CHECK-V-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
 ; CHECK-V-NEXT:    vfcvt.rtz.x.f.v v8, v8
+; CHECK-V-NEXT:    vmax.vx v8, v8, zero
 ; CHECK-V-NEXT:    vsetvli zero, zero, e32, mf2, ta, ma
 ; CHECK-V-NEXT:    vnclipu.wi v8, v8, 0
 ; CHECK-V-NEXT:    ret
@@ -3650,6 +3659,9 @@ define <4 x i32> @ustest_f32i32_mm(<4 x float> %x) {
 ; CHECK-V:       # %bb.0: # %entry
 ; CHECK-V-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; CHECK-V-NEXT:    vfwcvt.rtz.x.f.v v10, v8
+; CHECK-V-NEXT:    vsetvli zero, zero, e64, m2, ta, ma
+; CHECK-V-NEXT:    vmax.vx v10, v10, zero
+; CHECK-V-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; CHECK-V-NEXT:    vnclipu.wi v8, v10, 0
 ; CHECK-V-NEXT:    ret
 entry:
@@ -4139,16 +4151,17 @@ define <4 x i32> @ustest_f16i32_mm(<4 x half> %x) {
 ; CHECK-V-NEXT:    call __extendhfsf2
 ; CHECK-V-NEXT:    fcvt.l.s a0, fa0, rtz
 ; CHECK-V-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
-; CHECK-V-NEXT:    vmv.s.x v10, a0
+; CHECK-V-NEXT:    vmv.s.x v8, a0
 ; CHECK-V-NEXT:    addi a0, sp, 16
-; CHECK-V-NEXT:    vl1r.v v8, (a0) # Unknown-size Folded Reload
-; CHECK-V-NEXT:    vslideup.vi v10, v8, 1
+; CHECK-V-NEXT:    vl1r.v v9, (a0) # Unknown-size Folded Reload
+; CHECK-V-NEXT:    vslideup.vi v8, v9, 1
 ; CHECK-V-NEXT:    csrr a0, vlenb
 ; CHECK-V-NEXT:    add a0, sp, a0
 ; CHECK-V-NEXT:    addi a0, a0, 16
-; CHECK-V-NEXT:    vl2r.v v8, (a0) # Unknown-size Folded Reload
+; CHECK-V-NEXT:    vl2r.v v10, (a0) # Unknown-size Folded Reload
 ; CHECK-V-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
-; CHECK-V-NEXT:    vslideup.vi v10, v8, 2
+; CHECK-V-NEXT:    vslideup.vi v8, v10, 2
+; CHECK-V-NEXT:    vmax.vx v10, v8, zero
 ; CHECK-V-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; CHECK-V-NEXT:    vnclipu.wi v8, v10, 0
 ; CHECK-V-NEXT:    csrr a0, vlenb
@@ -4276,8 +4289,9 @@ define <2 x i16> @ustest_f64i16_mm(<2 x double> %x) {
 ; CHECK-V:       # %bb.0: # %entry
 ; CHECK-V-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
 ; CHECK-V-NEXT:    vfncvt.rtz.x.f.w v9, v8
+; CHECK-V-NEXT:    vmax.vx v8, v9, zero
 ; CHECK-V-NEXT:    vsetvli zero, zero, e16, mf4, ta, ma
-; CHECK-V-NEXT:    vnclipu.wi v8, v9, 0
+; CHECK-V-NEXT:    vnclipu.wi v8, v8, 0
 ; CHECK-V-NEXT:    ret
 entry:
   %conv = fptosi <2 x double> %x to <2 x i32>
@@ -4465,6 +4479,7 @@ define <4 x i16> @ustest_f32i16_mm(<4 x float> %x) {
 ; CHECK-V:       # %bb.0: # %entry
 ; CHECK-V-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
 ; CHECK-V-NEXT:    vfcvt.rtz.x.f.v v8, v8
+; CHECK-V-NEXT:    vmax.vx v8, v8, zero
 ; CHECK-V-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; CHECK-V-NEXT:    vnclipu.wi v8, v8, 0
 ; CHECK-V-NEXT:    ret
@@ -5434,23 +5449,24 @@ define <8 x i16> @ustest_f16i16_mm(<8 x half> %x) {
 ; CHECK-V-NEXT:    call __extendhfsf2
 ; CHECK-V-NEXT:    fcvt.l.s a0, fa0, rtz
 ; CHECK-V-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
-; CHECK-V-NEXT:    vmv.s.x v10, a0
+; CHECK-V-NEXT:    vmv.s.x v8, a0
 ; CHECK-V-NEXT:    addi a0, sp, 16
-; CHECK-V-NEXT:    vl1r.v v8, (a0) # Unknown-size Folded Reload
-; CHECK-V-NEXT:    vslideup.vi v10, v8, 1
+; CHECK-V-NEXT:    vl1r.v v9, (a0) # Unknown-size Folded Reload
+; CHECK-V-NEXT:    vslideup.vi v8, v9, 1
 ; CHECK-V-NEXT:    csrr a0, vlenb
 ; CHECK-V-NEXT:    add a0, sp, a0
 ; CHECK-V-NEXT:    addi a0, a0, 16
-; CHECK-V-NEXT:    vl1r.v v8, (a0) # Unknown-size Folded Reload
+; CHECK-V-NEXT:    vl1r.v v9, (a0) # Unknown-size Folded Reload
 ; CHECK-V-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; CHECK-V-NEXT:    vslideup.vi v10, v8, 2
+; CHECK-V-NEXT:    vslideup.vi v8, v9, 2
 ; CHECK-V-NEXT:    csrr a0, vlenb
 ; CHECK-V-NEXT:    slli a0, a0, 1
 ; CHECK-V-NEXT:    add a0, sp, a0
 ; CHECK-V-NEXT:    addi a0, a0, 16
-; CHECK-V-NEXT:    vl2r.v v8, (a0) # Unknown-size Folded Reload
+; CHECK-V-NEXT:    vl2r.v v10, (a0) # Unknown-size Folded Reload
 ; CHECK-V-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
-; CHECK-V-NEXT:    vslideup.vi v10, v8, 4
+; CHECK-V-NEXT:    vslideup.vi v8, v10, 4
+; CHECK-V-NEXT:    vmax.vx v10, v8, zero
 ; CHECK-V-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; CHECK-V-NEXT:    vnclipu.wi v8, v10, 0
 ; CHECK-V-NEXT:    csrr a0, vlenb
diff --git a/llvm/test/CodeGen/RISCV/rvv/trunc-sat-clip-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/trunc-sat-clip-sdnode.ll
index f43faadc532f260..01a90d8a33b6ec2 100644
--- a/llvm/test/CodeGen/RISCV/rvv/trunc-sat-clip-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/trunc-sat-clip-sdnode.ll
@@ -102,7 +102,9 @@ define void @trunc_sat_u8u16_maxmin(ptr %x, ptr %y) {
 ; CHECK-LABEL: trunc_sat_u8u16_maxmin:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vl1re16.v v8, (a0)
-; CHECK-NEXT:    vsetvli a0, zero, e8, mf2, ta, ma
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vmax.vx v8, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v8, v8, 0
 ; CHECK-NEXT:    vse8.v v8, (a1)
 ; CHECK-NEXT:    ret
@@ -118,7 +120,9 @@ define void @trunc_sat_u8u16_minmax(ptr %x, ptr %y) {
 ; CHECK-LABEL: trunc_sat_u8u16_minmax:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vl1re16.v v8, (a0)
-; CHECK-NEXT:    vsetvli a0, zero, e8, mf2, ta, ma
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vmax.vx v8, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v8, v8, 0
 ; CHECK-NEXT:    vse8.v v8, (a1)
 ; CHECK-NEXT:    ret
@@ -353,7 +357,9 @@ define void @trunc_sat_u32u64_maxmin(ptr %x, ptr %y) {
 ; CHECK-LABEL: trunc_sat_u32u64_maxmin:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vl4re64.v v8, (a0)
-; CHECK-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vsetvli a0, zero, e64, m4, ta, ma
+; CHECK-NEXT:    vmax.vx v8, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v12, v8, 0
 ; CHECK-NEXT:    vs2r.v v12, (a1)
 ; CHECK-NEXT:    ret
@@ -369,7 +375,9 @@ define void @trunc_sat_u32u64_minmax(ptr %x, ptr %y) {
 ; CHECK-LABEL: trunc_sat_u32u64_minmax:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vl4re64.v v8, (a0)
-; CHECK-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vsetvli a0, zero, e64, m4, ta, ma
+; CHECK-NEXT:    vmax.vx v8, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v12, v8, 0
 ; CHECK-NEXT:    vs2r.v v12, (a1)
 ; CHECK-NEXT:    ret
@@ -438,7 +446,9 @@ define void @trunc_sat_u8u32_maxmin(ptr %x, ptr %y) {
 ; CHECK-LABEL: trunc_sat_u8u32_maxmin:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vl2re32.v v8, (a0)
-; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vmax.vx v8, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v10, v8, 0
 ; CHECK-NEXT:    vsetvli zero, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v8, v10, 0
@@ -456,7 +466,9 @@ define void @trunc_sat_u8u32_minmax(ptr %x, ptr %y) {
 ; CHECK-LABEL: trunc_sat_u8u32_minmax:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vl2re32.v v8, (a0)
-; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vmax.vx v8, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v10, v8, 0
 ; CHECK-NEXT:    vsetvli zero, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v8, v10, 0
@@ -533,7 +545,9 @@ define void @trunc_sat_u8u64_maxmin(ptr %x, ptr %y) {
 ; CHECK-LABEL: trunc_sat_u8u64_maxmin:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vl4re64.v v8, (a0)
-; CHECK-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vsetvli a0, zero, e64, m4, ta, ma
+; CHECK-NEXT:    vmax.vx v8, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v12, v8, 0
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v8, v12, 0
@@ -553,7 +567,9 @@ define void @trunc_sat_u8u64_minmax(ptr %x, ptr %y) {
 ; CHECK-LABEL: trunc_sat_u8u64_minmax:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vl4re64.v v8, (a0)
-; CHECK-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vsetvli a0, zero, e64, m4, ta, ma
+; CHECK-NEXT:    vmax.vx v8, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v12, v8, 0
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v8, v12, 0
diff --git a/llvm/test/CodeGen/RISCV/rvv/trunc-select-to-max-usat.ll b/llvm/test/CodeGen/RISCV/rvv/trunc-select-to-max-usat.ll
index 9c558d23e17e08e..d2f73826e4e9e8f 100644
--- a/llvm/test/CodeGen/RISCV/rvv/trunc-select-to-max-usat.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/trunc-select-to-max-usat.ll
@@ -5,7 +5,9 @@
 define <4 x i8> @test_v4i16_v4i8(<4 x i16> %x) {
 ; CHECK-LABEL: test_v4i16_v4i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
+; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; CHECK-NEXT:    vmax.vx v8, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v8, v8, 0
 ; CHECK-NEXT:    ret
   %a = icmp sgt <4 x i16> %x, zeroinitializer
@@ -19,7 +21,9 @@ define <4 x i8> @test_v4i16_v4i8(<4 x i16> %x) {
 define <4 x i8> @test_v4i32_v4i8(<4 x i32> %x) {
 ; CHECK-LABEL: test_v4i32_v4i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vmax.vx v8, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v8, v8, 0
 ; CHECK-NEXT:    vsetvli zero, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v8, v8, 0
@@ -35,7 +39,9 @@ define <4 x i8> @test_v4i32_v4i8(<4 x i32> %x) {
 define <4 x i8> @test_v4i64_v4i8(<4 x i64> %x) {
 ; CHECK-LABEL: test_v4i64_v4i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; CHECK-NEXT:    vmax.vx v8, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v10, v8, 0
 ; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v8, v10, 0
@@ -53,7 +59,9 @@ define <4 x i8> @test_v4i64_v4i8(<4 x i64> %x) {
 define <4 x i16> @test_v4i32_v4i16(<4 x i32> %x) {
 ; CHECK-LABEL: test_v4i32_v4i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vmax.vx v8, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v8, v8, 0
 ; CHECK-NEXT:    ret
   %a = icmp sgt <4 x i32> %x, zeroinitializer
@@ -67,7 +75,9 @@ define <4 x i16> @test_v4i32_v4i16(<4 x i32> %x) {
 define <4 x i16> @test_v4i64_v4i16(<4 x i64> %x) {
 ; CHECK-LABEL: test_v4i64_v4i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; CHECK-NEXT:    vmax.vx v8, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v10, v8, 0
 ; CHECK-NEXT:    vsetvli zero, zero, e16, mf2, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v8, v10, 0
@@ -83,9 +93,10 @@ define <4 x i16> @test_v4i64_v4i16(<4 x i64> %x) {
 define <4 x i32> @test_v4i64_v4i32(<4 x i64> %x) {
 ; CHECK-LABEL: test_v4i64_v4i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; CHECK-NEXT:    vnclipu.wi v10, v8, 0
-; CHECK-NEXT:    vmv.v.v v8, v10
+; CHECK-NEXT:    vsetivli zero, 4, e64, m2, ta, ma
+; CHECK-NEXT:    vmax.vx v10, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; CHECK-NEXT:    vnclipu.wi v8, v10, 0
 ; CHECK-NEXT:    ret
   %a = icmp sgt <4 x i64> %x, zeroinitializer
   %b = sext <4 x i1> %a to <4 x i64>
@@ -98,7 +109,9 @@ define <4 x i32> @test_v4i64_v4i32(<4 x i64> %x) {
 define <vscale x 4 x i8> @test_nxv4i16_nxv4i8(<vscale x 4 x i16> %x) {
 ; CHECK-LABEL: test_nxv4i16_nxv4i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e8, mf2, ta, ma
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vmax.vx v8, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v8, v8, 0
 ; CHECK-NEXT:    ret
   %a = icmp sgt <vscale x 4 x i16> %x, zeroinitializer
@@ -112,7 +125,9 @@ define <vscale x 4 x i8> @test_nxv4i16_nxv4i8(<vscale x 4 x i16> %x) {
 define <vscale x 4 x i8> @test_nxv4i32_nxv4i8(<vscale x 4 x i32> %x) {
 ; CHECK-LABEL: test_nxv4i32_nxv4i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vmax.vx v8, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v10, v8, 0
 ; CHECK-NEXT:    vsetvli zero, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v8, v10, 0
@@ -128,7 +143,9 @@ define <vscale x 4 x i8> @test_nxv4i32_nxv4i8(<vscale x 4 x i32> %x) {
 define <vscale x 4 x i8> @test_nxv4i64_nxv4i8(<vscale x 4 x i64> %x) {
 ; CHECK-LABEL: test_nxv4i64_nxv4i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vsetvli a0, zero, e64, m4, ta, ma
+; CHECK-NEXT:    vmax.vx v8, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v12, v8, 0
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v8, v12, 0
@@ -146,9 +163,10 @@ define <vscale x 4 x i8> @test_nxv4i64_nxv4i8(<vscale x 4 x i64> %x) {
 define <vscale x 4 x i16> @test_nxv4i32_nxv4i16(<vscale x 4 x i32> %x) {
 ; CHECK-LABEL: test_nxv4i32_nxv4i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
-; CHECK-NEXT:    vnclipu.wi v10, v8, 0
-; CHECK-NEXT:    vmv.v.v v8, v10
+; CHECK-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vmax.vx v10, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vnclipu.wi v8, v10, 0
 ; CHECK-NEXT:    ret
   %a = icmp sgt <vscale x 4 x i32> %x, zeroinitializer
   %b = sext <vscale x 4 x i1> %a to <vscale x 4 x i32>
@@ -161,7 +179,9 @@ define <vscale x 4 x i16> @test_nxv4i32_nxv4i16(<vscale x 4 x i32> %x) {
 define <vscale x 4 x i16> @test_nxv4i64_nxv4i16(<vscale x 4 x i64> %x) {
 ; CHECK-LABEL: test_nxv4i64_nxv4i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vsetvli a0, zero, e64, m4, ta, ma
+; CHECK-NEXT:    vmax.vx v8, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v12, v8, 0
 ; CHECK-NEXT:    vsetvli zero, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vnclipu.wi v8, v12, 0
@@ -177,9 +197,10 @@ define <vscale x 4 x i16> @test_nxv4i64_nxv4i16(<vscale x 4 x i64> %x) {
 define <vscale x 4 x i32> @test_nxv4i64_nxv4i32(<vscale x 4 x i64> %x) {
 ; CHECK-LABEL: test_nxv4i64_nxv4i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
-; CHECK-NEXT:    vnclipu.wi v12, v8, 0
-; CHECK-NEXT:    vmv.v.v v8, v12
+; CHECK-NEXT:    vsetvli a0, zero, e64, m4, ta, ma
+; CHECK-NEXT:    vmax.vx v12, v8, zero
+; CHECK-NEXT:    vsetvli zero, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vnclipu.wi v8, v12, 0
 ; CHECK-NEXT:    ret
   %a = icmp sgt <vscale x 4 x i64> %x, zeroinitializer
   %b = sext <vscale x 4 x i1> %a to <vscale x 4 x i64>

From 321de07b778a16e88c589ab0af05ad10c906af13 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Fri, 16 Aug 2024 08:42:55 -0700
Subject: [PATCH 187/441] [DAGCombiner] Remove TRUNCATE_(S/U)SAT_(S/U) from an
 assert that isn't tested. NFC (#104466)

---
 llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index b21bea649e956b6..e75db2950579f6c 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -13201,9 +13201,7 @@ SDValue DAGCombiner::matchVSelectOpSizesWithSetCC(SDNode *Cast) {
   unsigned CastOpcode = Cast->getOpcode();
   assert((CastOpcode == ISD::SIGN_EXTEND || CastOpcode == ISD::ZERO_EXTEND ||
           CastOpcode == ISD::TRUNCATE || CastOpcode == ISD::FP_EXTEND ||
-          CastOpcode == ISD::TRUNCATE_SSAT_S ||
-          CastOpcode == ISD::TRUNCATE_SSAT_U ||
-          CastOpcode == ISD::TRUNCATE_USAT_U || CastOpcode == ISD::FP_ROUND) &&
+          CastOpcode == ISD::FP_ROUND) &&
          "Unexpected opcode for vector select narrowing/widening");
 
   // We only do this transform before legal ops because the pattern may be

From 82fe79357f54d4db4c4e2c46bbfbd4345022ae22 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Fri, 16 Aug 2024 08:43:02 -0700
Subject: [PATCH 188/441] [gn] Port AST/ByteCode #104552

---
 .../utils/gn/secondary/clang/lib/AST/BUILD.gn | 52 +++++++++----------
 .../AST/{Interp => ByteCode}/BUILD.gn         |  0
 .../gn/secondary/clang/unittests/BUILD.gn     |  2 +-
 3 files changed, 27 insertions(+), 27 deletions(-)
 rename llvm/utils/gn/secondary/clang/unittests/AST/{Interp => ByteCode}/BUILD.gn (100%)

diff --git a/llvm/utils/gn/secondary/clang/lib/AST/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/AST/BUILD.gn
index 1708af8612bc287..1dd34f2a077bc52 100644
--- a/llvm/utils/gn/secondary/clang/lib/AST/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/lib/AST/BUILD.gn
@@ -3,7 +3,7 @@ import("//clang/utils/TableGen/clang_tablegen.gni")
 clang_tablegen("Opcodes") {
   visibility = [ ":AST" ]
   args = [ "-gen-clang-opcodes" ]
-  td_file = "Interp/Opcodes.td"
+  td_file = "ByteCode/Opcodes.td"
 }
 
 clang_tablegen("AttrDocTable") {
@@ -92,31 +92,31 @@ static_library("AST") {
     "ExternalASTSource.cpp",
     "FormatString.cpp",
     "InheritViz.cpp",
-    "Interp/ByteCodeEmitter.cpp",
-    "Interp/Compiler.cpp",
-    "Interp/Context.cpp",
-    "Interp/Descriptor.cpp",
-    "Interp/Disasm.cpp",
-    "Interp/DynamicAllocator.cpp",
-    "Interp/EvalEmitter.cpp",
-    "Interp/EvaluationResult.cpp",
-    "Interp/Floating.cpp",
-    "Interp/Frame.cpp",
-    "Interp/Function.cpp",
-    "Interp/Interp.cpp",
-    "Interp/InterpBlock.cpp",
-    "Interp/InterpBuiltin.cpp",
-    "Interp/InterpFrame.cpp",
-    "Interp/InterpShared.cpp",
-    "Interp/InterpStack.cpp",
-    "Interp/InterpState.cpp",
-    "Interp/MemberPointer.cpp",
-    "Interp/Pointer.cpp",
-    "Interp/PrimType.cpp",
-    "Interp/Program.cpp",
-    "Interp/Record.cpp",
-    "Interp/Source.cpp",
-    "Interp/State.cpp",
+    "ByteCode/ByteCodeEmitter.cpp",
+    "ByteCode/Compiler.cpp",
+    "ByteCode/Context.cpp",
+    "ByteCode/Descriptor.cpp",
+    "ByteCode/Disasm.cpp",
+    "ByteCode/DynamicAllocator.cpp",
+    "ByteCode/EvalEmitter.cpp",
+    "ByteCode/EvaluationResult.cpp",
+    "ByteCode/Floating.cpp",
+    "ByteCode/Frame.cpp",
+    "ByteCode/Function.cpp",
+    "ByteCode/Interp.cpp",
+    "ByteCode/InterpBlock.cpp",
+    "ByteCode/InterpBuiltin.cpp",
+    "ByteCode/InterpFrame.cpp",
+    "ByteCode/InterpShared.cpp",
+    "ByteCode/InterpStack.cpp",
+    "ByteCode/InterpState.cpp",
+    "ByteCode/MemberPointer.cpp",
+    "ByteCode/Pointer.cpp",
+    "ByteCode/PrimType.cpp",
+    "ByteCode/Program.cpp",
+    "ByteCode/Record.cpp",
+    "ByteCode/Source.cpp",
+    "ByteCode/State.cpp",
     "ItaniumCXXABI.cpp",
     "ItaniumMangle.cpp",
     "JSONNodeDumper.cpp",
diff --git a/llvm/utils/gn/secondary/clang/unittests/AST/Interp/BUILD.gn b/llvm/utils/gn/secondary/clang/unittests/AST/ByteCode/BUILD.gn
similarity index 100%
rename from llvm/utils/gn/secondary/clang/unittests/AST/Interp/BUILD.gn
rename to llvm/utils/gn/secondary/clang/unittests/AST/ByteCode/BUILD.gn
diff --git a/llvm/utils/gn/secondary/clang/unittests/BUILD.gn b/llvm/utils/gn/secondary/clang/unittests/BUILD.gn
index 8b98e3beec284d5..a6a4a5708341f0d 100644
--- a/llvm/utils/gn/secondary/clang/unittests/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/unittests/BUILD.gn
@@ -3,7 +3,7 @@ import("//clang/lib/StaticAnalyzer/Frontend/enable.gni")
 group("unittests") {
   deps = [
     "AST:ASTTests",
-    "AST/Interp:InterpTests",
+    "AST/ByteCode:InterpTests",
     "ASTMatchers:ASTMatchersTests",
     "ASTMatchers/Dynamic:DynamicASTMatchersTests",
     "Basic:BasicTests",

From ce88ca928a29748f5fd179a3ab7f7039b3a311c4 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Fri, 16 Aug 2024 08:43:54 -0700
Subject: [PATCH 189/441] [TableGen] Sign extend constants based on size for
 EmitIntegerMatcher. (#104550)

I'm planning to add a getSignedConstant to SelectionDAG and use it for
EmitInteger in SelectionDAGISel which already uses int64_t.
getSignedConstant will assert that the constant has the correct number
of significant bits for the VT.

This patch ensures that tablegen emits constants in this form.
---
 llvm/utils/TableGen/Common/DAGISelMatcher.h | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/llvm/utils/TableGen/Common/DAGISelMatcher.h b/llvm/utils/TableGen/Common/DAGISelMatcher.h
index f4ababcd4be68ae..81a5e3e050f2724 100644
--- a/llvm/utils/TableGen/Common/DAGISelMatcher.h
+++ b/llvm/utils/TableGen/Common/DAGISelMatcher.h
@@ -836,7 +836,8 @@ class EmitIntegerMatcher : public Matcher {
 
 public:
   EmitIntegerMatcher(int64_t val, MVT::SimpleValueType vt)
-      : Matcher(EmitInteger), Val(val), VT(vt) {}
+      : Matcher(EmitInteger), Val(SignExtend64(val, MVT(vt).getSizeInBits())),
+        VT(vt) {}
 
   int64_t getValue() const { return Val; }
   MVT::SimpleValueType getVT() const { return VT; }

From 6fceb3e86591e9a111671b9b76e11262279d7bad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andrzej=20Warzy=C5=84ski?= <andrzej.warzynski@arm.com>
Date: Fri, 16 Aug 2024 16:48:16 +0100
Subject: [PATCH 190/441] [mlir][vector] Add more tests for ConvertVectorToLLVM
 (4/n) (#103391)

Adds tests with scalable vectors for the Vector-To-LLVM conversion pass.
Covers the following Ops:
  * vector.insertelement
  * vector.insert

I have also renamed some function names from `@insert_element{}` to
`@insertelement{}` - that's to make a clearer distinction between
tests for `vector.insertelement` (tested by `@insertelement{}`) and
`vector.insert` (tested by `@insert_element{}`).
---
 .../VectorToLLVM/vector-to-llvm.mlir          | 99 +++++++++++++++++--
 1 file changed, 93 insertions(+), 6 deletions(-)

diff --git a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir
index 9b61c4493994c2e..0cd7ee6fb424fc0 100644
--- a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir
+++ b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir
@@ -1245,9 +1245,9 @@ func.func @extract_element_with_value_2d(%arg0: vector<1x16xf32>, %arg1: index)
 
 // -----
 
-// CHECK-LABEL: @insert_element_0d
+// CHECK-LABEL: @insertelement_0d
 // CHECK-SAME: %[[A:.*]]: f32,
-func.func @insert_element_0d(%a: f32, %b: vector<f32>) -> vector<f32> {
+func.func @insertelement_0d(%a: f32, %b: vector<f32>) -> vector<f32> {
   // CHECK: %[[B:.*]] =  builtin.unrealized_conversion_cast %{{.*}} :
   // CHECK:   vector<f32> to vector<1xf32>
   // CHECK: %[[C0:.*]] = llvm.mlir.constant(0 : index) : i64
@@ -1258,26 +1258,38 @@ func.func @insert_element_0d(%a: f32, %b: vector<f32>) -> vector<f32> {
 
 // -----
 
-func.func @insert_element(%arg0: f32, %arg1: vector<4xf32>) -> vector<4xf32> {
+func.func @insertelement(%arg0: f32, %arg1: vector<4xf32>) -> vector<4xf32> {
   %0 = arith.constant 3 : i32
   %1 = vector.insertelement %arg0, %arg1[%0 : i32] : vector<4xf32>
   return %1 : vector<4xf32>
 }
-// CHECK-LABEL: @insert_element(
+// CHECK-LABEL: @insertelement(
 // CHECK-SAME: %[[A:.*]]: f32,
 // CHECK-SAME: %[[B:.*]]: vector<4xf32>)
 //       CHECK:   %[[c:.*]] = arith.constant 3 : i32
 //       CHECK:   %[[x:.*]] = llvm.insertelement %[[A]], %[[B]][%[[c]] : i32] : vector<4xf32>
 //       CHECK:   return %[[x]] : vector<4xf32>
 
+func.func @insertelement_scalable(%arg0: f32, %arg1: vector<[4]xf32>) -> vector<[4]xf32> {
+  %0 = arith.constant 3 : i32
+  %1 = vector.insertelement %arg0, %arg1[%0 : i32] : vector<[4]xf32>
+  return %1 : vector<[4]xf32>
+}
+// CHECK-LABEL: @insertelement_scalable(
+// CHECK-SAME: %[[A:.*]]: f32,
+// CHECK-SAME: %[[B:.*]]: vector<[4]xf32>)
+//       CHECK:   %[[c:.*]] = arith.constant 3 : i32
+//       CHECK:   %[[x:.*]] = llvm.insertelement %[[A]], %[[B]][%[[c]] : i32] : vector<[4]xf32>
+//       CHECK:   return %[[x]] : vector<[4]xf32>
+
 // -----
 
-func.func @insert_element_index(%arg0: f32, %arg1: vector<4xf32>) -> vector<4xf32> {
+func.func @insertelement_index(%arg0: f32, %arg1: vector<4xf32>) -> vector<4xf32> {
   %0 = arith.constant 3 : index
   %1 = vector.insertelement %arg0, %arg1[%0 : index] : vector<4xf32>
   return %1 : vector<4xf32>
 }
-// CHECK-LABEL: @insert_element_index(
+// CHECK-LABEL: @insertelement_index(
 //  CHECK-SAME: %[[A:.*]]: f32,
 //  CHECK-SAME: %[[B:.*]]: vector<4xf32>)
 //       CHECK:   %[[c:.*]] = arith.constant 3 : index
@@ -1285,6 +1297,19 @@ func.func @insert_element_index(%arg0: f32, %arg1: vector<4xf32>) -> vector<4xf3
 //       CHECK:   %[[x:.*]] = llvm.insertelement %[[A]], %[[B]][%[[i]] : i64] : vector<4xf32>
 //       CHECK:   return %[[x]] : vector<4xf32>
 
+func.func @insertelement_index_scalable(%arg0: f32, %arg1: vector<[4]xf32>) -> vector<[4]xf32> {
+  %0 = arith.constant 3 : index
+  %1 = vector.insertelement %arg0, %arg1[%0 : index] : vector<[4]xf32>
+  return %1 : vector<[4]xf32>
+}
+// CHECK-LABEL: @insertelement_index_scalable(
+//  CHECK-SAME: %[[A:.*]]: f32,
+//  CHECK-SAME: %[[B:.*]]: vector<[4]xf32>)
+//       CHECK:   %[[c:.*]] = arith.constant 3 : index
+//       CHECK:   %[[i:.*]] = builtin.unrealized_conversion_cast %[[c]] : index to i64
+//       CHECK:   %[[x:.*]] = llvm.insertelement %[[A]], %[[B]][%[[i]] : i64] : vector<[4]xf32>
+//       CHECK:   return %[[x]] : vector<[4]xf32>
+
 // -----
 
 func.func @insert_element_into_vec_1d(%arg0: f32, %arg1: vector<4xf32>) -> vector<4xf32> {
@@ -1296,6 +1321,15 @@ func.func @insert_element_into_vec_1d(%arg0: f32, %arg1: vector<4xf32>) -> vecto
 //       CHECK:   llvm.insertelement {{.*}}, {{.*}}[{{.*}} : i64] : vector<4xf32>
 //       CHECK:   return {{.*}} : vector<4xf32>
 
+func.func @insert_element_into_vec_1d_scalable(%arg0: f32, %arg1: vector<[4]xf32>) -> vector<[4]xf32> {
+  %0 = vector.insert %arg0, %arg1[3] : f32 into vector<[4]xf32>
+  return %0 : vector<[4]xf32>
+}
+// CHECK-LABEL: @insert_element_into_vec_1d_scalable
+//       CHECK:   llvm.mlir.constant(3 : i64) : i64
+//       CHECK:   llvm.insertelement {{.*}}, {{.*}}[{{.*}} : i64] : vector<[4]xf32>
+//       CHECK:   return {{.*}} : vector<[4]xf32>
+
 // -----
 
 func.func @insert_index_element_into_vec_1d(%arg0: index, %arg1: vector<4xindex>) -> vector<4xindex> {
@@ -1312,6 +1346,21 @@ func.func @insert_index_element_into_vec_1d(%arg0: index, %arg1: vector<4xindex>
 //       CHECK:   %[[T5:.*]] = builtin.unrealized_conversion_cast %[[T4]] : vector<4xi64> to vector<4xindex>
 //       CHECK:   return %[[T5]] : vector<4xindex>
 
+
+func.func @insert_index_element_into_vec_1d_scalable(%arg0: index, %arg1: vector<[4]xindex>) -> vector<[4]xindex> {
+  %0 = vector.insert %arg0, %arg1[3] : index into vector<[4]xindex>
+  return %0 : vector<[4]xindex>
+}
+// CHECK-LABEL: @insert_index_element_into_vec_1d_scalable(
+// CHECK-SAME: %[[A:.*]]: index,
+// CHECK-SAME: %[[B:.*]]: vector<[4]xindex>)
+//       CHECK-DAG:   %[[T0:.*]] = builtin.unrealized_conversion_cast %[[A]] : index to i64
+//       CHECK-DAG:   %[[T1:.*]] = builtin.unrealized_conversion_cast %[[B]] : vector<[4]xindex> to vector<[4]xi64>
+//       CHECK:   %[[T3:.*]] = llvm.mlir.constant(3 : i64) : i64
+//       CHECK:   %[[T4:.*]] = llvm.insertelement %[[T0]], %[[T1]][%[[T3]] : i64] : vector<[4]xi64>
+//       CHECK:   %[[T5:.*]] = builtin.unrealized_conversion_cast %[[T4]] : vector<[4]xi64> to vector<[4]xindex>
+//       CHECK:   return %[[T5]] : vector<[4]xindex>
+
 // -----
 
 func.func @insert_vec_2d_into_vec_3d(%arg0: vector<8x16xf32>, %arg1: vector<4x8x16xf32>) -> vector<4x8x16xf32> {
@@ -1322,6 +1371,14 @@ func.func @insert_vec_2d_into_vec_3d(%arg0: vector<8x16xf32>, %arg1: vector<4x8x
 //       CHECK:   llvm.insertvalue {{.*}}, {{.*}}[3] : !llvm.array<4 x array<8 x vector<16xf32>>>
 //       CHECK:   return {{.*}} : vector<4x8x16xf32>
 
+func.func @insert_vec_2d_into_vec_3d_scalable(%arg0: vector<8x[16]xf32>, %arg1: vector<4x8x[16]xf32>) -> vector<4x8x[16]xf32> {
+  %0 = vector.insert %arg0, %arg1[3] : vector<8x[16]xf32> into vector<4x8x[16]xf32>
+  return %0 : vector<4x8x[16]xf32>
+}
+// CHECK-LABEL: @insert_vec_2d_into_vec_3d_scalable
+//       CHECK:   llvm.insertvalue {{.*}}, {{.*}}[3] : !llvm.array<4 x array<8 x vector<[16]xf32>>>
+//       CHECK:   return {{.*}} : vector<4x8x[16]xf32>
+
 // -----
 
 func.func @insert_vec_1d_into_vec_3d(%arg0: vector<16xf32>, %arg1: vector<4x8x16xf32>) -> vector<4x8x16xf32> {
@@ -1332,6 +1389,14 @@ func.func @insert_vec_1d_into_vec_3d(%arg0: vector<16xf32>, %arg1: vector<4x8x16
 //       CHECK:   llvm.insertvalue {{.*}}, {{.*}}[3, 7] : !llvm.array<4 x array<8 x vector<16xf32>>>
 //       CHECK:   return {{.*}} : vector<4x8x16xf32>
 
+func.func @insert_vec_1d_into_vec_3d_scalable(%arg0: vector<[16]xf32>, %arg1: vector<4x8x[16]xf32>) -> vector<4x8x[16]xf32> {
+  %0 = vector.insert %arg0, %arg1[3, 7] : vector<[16]xf32> into vector<4x8x[16]xf32>
+  return %0 : vector<4x8x[16]xf32>
+}
+// CHECK-LABEL: @insert_vec_1d_into_vec_3d_scalable
+//       CHECK:   llvm.insertvalue {{.*}}, {{.*}}[3, 7] : !llvm.array<4 x array<8 x vector<[16]xf32>>>
+//       CHECK:   return {{.*}} : vector<4x8x[16]xf32>
+
 // -----
 
 func.func @insert_element_into_vec_3d(%arg0: f32, %arg1: vector<4x8x16xf32>) -> vector<4x8x16xf32> {
@@ -1345,6 +1410,17 @@ func.func @insert_element_into_vec_3d(%arg0: f32, %arg1: vector<4x8x16xf32>) ->
 //       CHECK:   llvm.insertvalue {{.*}}, {{.*}}[3, 7] : !llvm.array<4 x array<8 x vector<16xf32>>>
 //       CHECK:   return {{.*}} : vector<4x8x16xf32>
 
+func.func @insert_element_into_vec_3d_scalable(%arg0: f32, %arg1: vector<4x8x[16]xf32>) -> vector<4x8x[16]xf32> {
+  %0 = vector.insert %arg0, %arg1[3, 7, 15] : f32 into vector<4x8x[16]xf32>
+  return %0 : vector<4x8x[16]xf32>
+}
+// CHECK-LABEL: @insert_element_into_vec_3d_scalable
+//       CHECK:   llvm.extractvalue {{.*}}[3, 7] : !llvm.array<4 x array<8 x vector<[16]xf32>>>
+//       CHECK:   llvm.mlir.constant(15 : i64) : i64
+//       CHECK:   llvm.insertelement {{.*}}, {{.*}}[{{.*}} : i64] : vector<[16]xf32>
+//       CHECK:   llvm.insertvalue {{.*}}, {{.*}}[3, 7] : !llvm.array<4 x array<8 x vector<[16]xf32>>>
+//       CHECK:   return {{.*}} : vector<4x8x[16]xf32>
+
 // -----
 
 func.func @insert_element_with_value_1d(%arg0: vector<16xf32>, %arg1: f32, %arg2: index)
@@ -1358,6 +1434,17 @@ func.func @insert_element_with_value_1d(%arg0: vector<16xf32>, %arg1: f32, %arg2
 //       CHECK:   %[[UC:.+]] = builtin.unrealized_conversion_cast %[[INDEX]] : index to i64
 //       CHECK:   llvm.insertelement %[[SRC]], %[[DST]][%[[UC]] : i64] : vector<16xf32>
 
+func.func @insert_element_with_value_1d_scalable(%arg0: vector<[16]xf32>, %arg1: f32, %arg2: index)
+                                      -> vector<[16]xf32> {
+  %0 = vector.insert %arg1, %arg0[%arg2]: f32 into vector<[16]xf32>
+  return %0 : vector<[16]xf32>
+}
+
+// CHECK-LABEL: @insert_element_with_value_1d_scalable
+//  CHECK-SAME:   %[[DST:.+]]: vector<[16]xf32>, %[[SRC:.+]]: f32, %[[INDEX:.+]]: index
+//       CHECK:   %[[UC:.+]] = builtin.unrealized_conversion_cast %[[INDEX]] : index to i64
+//       CHECK:   llvm.insertelement %[[SRC]], %[[DST]][%[[UC]] : i64] : vector<[16]xf32>
+
 // -----
 
 func.func @insert_element_with_value_2d(%base: vector<1x16xf32>, %value: f32, %idx: index)

From 4a57e834f7de83d85d994c63647aa957279c354e Mon Sep 17 00:00:00 2001
From: Chris B <chris.bieneman@me.com>
Date: Fri, 16 Aug 2024 10:51:07 -0500
Subject: [PATCH 191/441] [HLSL] Flesh out basic type typedefs (#104479)

We had a few missing typedefs that are supported by DXC. Specifically
1-element vectors, size-explicit 32-bit types and size-explicit floating
point types.

This adds the typedefs and a test file that just verifies the expected
sizes and vector element counts.

I needed to add some of these missing typedefs to address #102964, and
thought instead I should try and be a bit comprehensive and put it in a
separate PR.

Nothing complicated here, just typedefs and static asserts to verify
them.
---
 clang/lib/Headers/hlsl/hlsl_basic_types.h | 48 ++++++++++++++++++++++-
 clang/test/SemaHLSL/Types/typedefs.hlsl   | 34 ++++++++++++++++
 2 files changed, 81 insertions(+), 1 deletion(-)
 create mode 100644 clang/test/SemaHLSL/Types/typedefs.hlsl

diff --git a/clang/lib/Headers/hlsl/hlsl_basic_types.h b/clang/lib/Headers/hlsl/hlsl_basic_types.h
index da6903df65ffed7..eff94e0d7f9500e 100644
--- a/clang/lib/Headers/hlsl/hlsl_basic_types.h
+++ b/clang/lib/Headers/hlsl/hlsl_basic_types.h
@@ -23,52 +23,98 @@ namespace hlsl {
 // 16-bit integer.
 typedef unsigned short uint16_t;
 typedef short int16_t;
+
+// 16-bit floating point.
+typedef half float16_t;
 #endif
 
+// 32-bit integer.
+typedef int int32_t;
+
 // unsigned 32-bit integer.
 typedef unsigned int uint;
+typedef unsigned int uint32_t;
+
+// 32-bit floating point.
+typedef float float32_t;
 
 // 64-bit integer.
 typedef unsigned long uint64_t;
 typedef long int64_t;
 
+// 64-bit floating point
+typedef double float64_t;
+
 // built-in vector data types:
 
 #ifdef __HLSL_ENABLE_16_BIT
+typedef vector<int16_t, 1> int16_t1;
 typedef vector<int16_t, 2> int16_t2;
 typedef vector<int16_t, 3> int16_t3;
 typedef vector<int16_t, 4> int16_t4;
+typedef vector<uint16_t, 1> uint16_t1;
 typedef vector<uint16_t, 2> uint16_t2;
 typedef vector<uint16_t, 3> uint16_t3;
 typedef vector<uint16_t, 4> uint16_t4;
 #endif
+typedef vector<bool, 1> bool1;
 typedef vector<bool, 2> bool2;
 typedef vector<bool, 3> bool3;
 typedef vector<bool, 4> bool4;
+typedef vector<int, 1> int1;
 typedef vector<int, 2> int2;
 typedef vector<int, 3> int3;
 typedef vector<int, 4> int4;
+typedef vector<uint, 1> uint1;
 typedef vector<uint, 2> uint2;
 typedef vector<uint, 3> uint3;
 typedef vector<uint, 4> uint4;
+typedef vector<int32_t, 1> int32_t1;
+typedef vector<int32_t, 2> int32_t2;
+typedef vector<int32_t, 3> int32_t3;
+typedef vector<int32_t, 4> int32_t4;
+typedef vector<uint32_t, 1> uint32_t1;
+typedef vector<uint32_t, 2> uint32_t2;
+typedef vector<uint32_t, 3> uint32_t3;
+typedef vector<uint32_t, 4> uint32_t4;
+typedef vector<int64_t, 1> int64_t1;
 typedef vector<int64_t, 2> int64_t2;
 typedef vector<int64_t, 3> int64_t3;
 typedef vector<int64_t, 4> int64_t4;
+typedef vector<uint64_t, 1> uint64_t1;
 typedef vector<uint64_t, 2> uint64_t2;
 typedef vector<uint64_t, 3> uint64_t3;
 typedef vector<uint64_t, 4> uint64_t4;
 
+typedef vector<half, 1> half1;
 typedef vector<half, 2> half2;
 typedef vector<half, 3> half3;
 typedef vector<half, 4> half4;
-
+typedef vector<float, 1> float1;
 typedef vector<float, 2> float2;
 typedef vector<float, 3> float3;
 typedef vector<float, 4> float4;
+typedef vector<double, 1> double1;
 typedef vector<double, 2> double2;
 typedef vector<double, 3> double3;
 typedef vector<double, 4> double4;
 
+#ifdef __HLSL_ENABLE_16_BIT
+typedef vector<float16_t, 1> float16_t1;
+typedef vector<float16_t, 2> float16_t2;
+typedef vector<float16_t, 3> float16_t3;
+typedef vector<float16_t, 4> float16_t4;
+#endif
+
+typedef vector<float32_t, 1> float32_t1;
+typedef vector<float32_t, 2> float32_t2;
+typedef vector<float32_t, 3> float32_t3;
+typedef vector<float32_t, 4> float32_t4;
+typedef vector<float64_t, 1> float64_t1;
+typedef vector<float64_t, 2> float64_t2;
+typedef vector<float64_t, 3> float64_t3;
+typedef vector<float64_t, 4> float64_t4;
+
 } // namespace hlsl
 
 #endif //_HLSL_HLSL_BASIC_TYPES_H_
diff --git a/clang/test/SemaHLSL/Types/typedefs.hlsl b/clang/test/SemaHLSL/Types/typedefs.hlsl
new file mode 100644
index 000000000000000..fd72b1ae8a47ffe
--- /dev/null
+++ b/clang/test/SemaHLSL/Types/typedefs.hlsl
@@ -0,0 +1,34 @@
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.4-library -finclude-default-header -verify -fnative-half-type %s
+// RUN: %clang_cc1 -triple spirv-linux-vulkan-library -finclude-default-header -verify -fnative-half-type %s
+
+// expected-no-diagnostics
+#define SizeCheck(Ty, SizeInBits)                                              \
+  _Static_assert(sizeof(Ty) == SizeInBits / 8, #Ty " is " #SizeInBits "-bit"); \
+  _Static_assert(sizeof(Ty##1) == (SizeInBits * 1) / 8, #Ty "1 is 1x" #SizeInBits "-bit"); \
+  _Static_assert(__builtin_vectorelements(Ty##1) == 1, #Ty "1 is has 1 " #SizeInBits "-bit element"); \
+  _Static_assert(sizeof(Ty##2) == (SizeInBits * 2) / 8, #Ty "2 is 2x" #SizeInBits "-bit"); \
+  _Static_assert(__builtin_vectorelements(Ty##2) == 2, #Ty "2 is has 2 " #SizeInBits "-bit element"); \
+  _Static_assert(__builtin_vectorelements(Ty##3) == 3, #Ty "3 is has 3 " #SizeInBits "-bit element"); \
+  _Static_assert(sizeof(Ty##4) == (SizeInBits * 4) / 8, #Ty "4 is 4x" #SizeInBits "-bit"); \
+  _Static_assert(__builtin_vectorelements(Ty##4) == 4, #Ty "4 is has 4 " #SizeInBits "-bit element");
+
+// FIXME: https://github.com/llvm/llvm-project/issues/104503 - 3 element vectors
+// should be the size of 3 elements not padded to 4.
+// _Static_assert(sizeof(Ty##3) == (SizeInBits * 3) / 8, #Ty "3 is 3x" #SizeInBits "-bit");
+
+SizeCheck(int16_t, 16);
+SizeCheck(uint16_t, 16);
+SizeCheck(half, 16);
+SizeCheck(float16_t, 16);
+
+SizeCheck(int, 32);
+SizeCheck(uint, 32);
+SizeCheck(int32_t, 32);
+SizeCheck(uint32_t, 32);
+SizeCheck(float, 32);
+SizeCheck(float32_t, 32);
+
+SizeCheck(int64_t, 64);
+SizeCheck(uint64_t, 64);
+SizeCheck(double, 64);
+SizeCheck(float64_t, 64);

From a434cac523da6db542350fd747967520aaae8fbb Mon Sep 17 00:00:00 2001
From: Fred Grim <fgrim@apple.com>
Date: Fri, 16 Aug 2024 08:53:15 -0700
Subject: [PATCH 192/441] [lldb] Add Populate Methods for ELFLinuxPrPsInfo and
 ELFLinuxPrStatus (#104109)

To create elf core files there are multiple notes in the core file that
contain these structs as the note. These populate methods take a Process
and produce fully specified structures that can be used to fill these
note sections. The pr also adds tests to ensure these structs are
correctly populated.
---
 .../Process/elf-core/ThreadElfCore.cpp        | 129 +++++++++++++
 .../Plugins/Process/elf-core/ThreadElfCore.h  |  15 ++
 lldb/unittests/Process/CMakeLists.txt         |   1 +
 .../unittests/Process/elf-core/CMakeLists.txt |  15 ++
 .../Process/elf-core/ThreadElfCoreTest.cpp    | 174 ++++++++++++++++++
 5 files changed, 334 insertions(+)
 create mode 100644 lldb/unittests/Process/elf-core/CMakeLists.txt
 create mode 100644 lldb/unittests/Process/elf-core/ThreadElfCoreTest.cpp

diff --git a/lldb/source/Plugins/Process/elf-core/ThreadElfCore.cpp b/lldb/source/Plugins/Process/elf-core/ThreadElfCore.cpp
index 2a83163884e168b..a3ee9dff572766b 100644
--- a/lldb/source/Plugins/Process/elf-core/ThreadElfCore.cpp
+++ b/lldb/source/Plugins/Process/elf-core/ThreadElfCore.cpp
@@ -13,6 +13,7 @@
 #include "lldb/Utility/DataExtractor.h"
 #include "lldb/Utility/LLDBLog.h"
 #include "lldb/Utility/Log.h"
+#include "lldb/Utility/ProcessInfo.h"
 
 #include "Plugins/Process/Utility/RegisterContextFreeBSD_i386.h"
 #include "Plugins/Process/Utility/RegisterContextFreeBSD_mips64.h"
@@ -318,6 +319,32 @@ Status ELFLinuxPrStatus::Parse(const DataExtractor &data,
   return error;
 }
 
+static struct compat_timeval
+copy_timespecs(const ProcessInstanceInfo::timespec &oth) {
+  using sec_t = decltype(compat_timeval::tv_sec);
+  using usec_t = decltype(compat_timeval::tv_usec);
+  return {static_cast<sec_t>(oth.tv_sec), static_cast<usec_t>(oth.tv_usec)};
+}
+
+std::optional<ELFLinuxPrStatus>
+ELFLinuxPrStatus::Populate(const lldb::ThreadSP &thread_sp) {
+  ELFLinuxPrStatus prstatus{};
+  prstatus.pr_pid = thread_sp->GetID();
+  lldb::ProcessSP process_sp = thread_sp->GetProcess();
+  ProcessInstanceInfo info;
+  if (!process_sp->GetProcessInfo(info))
+    return std::nullopt;
+
+  prstatus.pr_ppid = info.GetParentProcessID();
+  prstatus.pr_pgrp = info.GetProcessGroupID();
+  prstatus.pr_sid = info.GetProcessSessionID();
+  prstatus.pr_utime = copy_timespecs(info.GetUserTime());
+  prstatus.pr_stime = copy_timespecs(info.GetSystemTime());
+  prstatus.pr_cutime = copy_timespecs(info.GetCumulativeUserTime());
+  prstatus.pr_cstime = copy_timespecs(info.GetCumulativeSystemTime());
+  return prstatus;
+}
+
 // Parse PRPSINFO from NOTE entry
 ELFLinuxPrPsInfo::ELFLinuxPrPsInfo() {
   memset(this, 0, sizeof(ELFLinuxPrPsInfo));
@@ -394,6 +421,108 @@ Status ELFLinuxPrPsInfo::Parse(const DataExtractor &data,
   return error;
 }
 
+std::optional<ELFLinuxPrPsInfo>
+ELFLinuxPrPsInfo::Populate(const lldb::ProcessSP &process_sp) {
+  ProcessInstanceInfo info;
+  if (!process_sp->GetProcessInfo(info))
+    return std::nullopt;
+
+  return Populate(info, process_sp->GetState());
+}
+
+std::optional<ELFLinuxPrPsInfo>
+ELFLinuxPrPsInfo::Populate(const lldb_private::ProcessInstanceInfo &info,
+                           lldb::StateType process_state) {
+  ELFLinuxPrPsInfo prpsinfo{};
+  prpsinfo.pr_pid = info.GetProcessID();
+  prpsinfo.pr_nice = info.GetPriorityValue().value_or(0);
+  prpsinfo.pr_zomb = 0;
+  if (auto zombie_opt = info.IsZombie(); zombie_opt.value_or(false)) {
+    prpsinfo.pr_zomb = 1;
+  }
+  /**
+   * In the linux kernel this comes from:
+   * state = READ_ONCE(p->__state);
+   * i = state ? ffz(~state) + 1 : 0;
+   * psinfo->pr_sname = (i > 5) ? '.' : "RSDTZW"[i];
+   *
+   * So we replicate that here. From proc_pid_stats(5)
+   * R = Running
+   * S = Sleeping on uninterrutible wait
+   * D = Waiting on uninterruptable disk sleep
+   * T = Tracing stop
+   * Z = Zombie
+   * W = Paging
+   */
+  switch (process_state) {
+  case lldb::StateType::eStateSuspended:
+    prpsinfo.pr_sname = 'S';
+    prpsinfo.pr_state = 1;
+    break;
+  case lldb::StateType::eStateStopped:
+    [[fallthrough]];
+  case lldb::StateType::eStateStepping:
+    prpsinfo.pr_sname = 'T';
+    prpsinfo.pr_state = 3;
+    break;
+  case lldb::StateType::eStateUnloaded:
+    [[fallthrough]];
+  case lldb::StateType::eStateRunning:
+    prpsinfo.pr_sname = 'R';
+    prpsinfo.pr_state = 0;
+    break;
+  default:
+    break;
+  }
+
+  /**
+   * pr_flags is left as 0. The values (in linux) are specific
+   * to the kernel. We recover them from the proc filesystem
+   * but don't put them in ProcessInfo because it would really
+   * become very linux specific and the utility here seems pretty
+   * dubious
+   */
+
+  if (info.EffectiveUserIDIsValid())
+    prpsinfo.pr_uid = info.GetUserID();
+
+  if (info.EffectiveGroupIDIsValid())
+    prpsinfo.pr_gid = info.GetGroupID();
+
+  if (info.ParentProcessIDIsValid())
+    prpsinfo.pr_ppid = info.GetParentProcessID();
+
+  if (info.ProcessGroupIDIsValid())
+    prpsinfo.pr_pgrp = info.GetProcessGroupID();
+
+  if (info.ProcessSessionIDIsValid())
+    prpsinfo.pr_sid = info.GetProcessSessionID();
+
+  constexpr size_t fname_len = std::extent_v<decltype(prpsinfo.pr_fname)>;
+  static_assert(fname_len > 0, "This should always be non zero");
+  const llvm::StringRef fname = info.GetNameAsStringRef();
+  auto fname_begin = fname.begin();
+  std::copy_n(fname_begin, std::min(fname_len, fname.size()),
+              prpsinfo.pr_fname);
+  prpsinfo.pr_fname[fname_len - 1] = '\0';
+  auto args = info.GetArguments();
+  auto argentry_iterator = std::begin(args);
+  char *psargs = prpsinfo.pr_psargs;
+  char *psargs_end = std::end(prpsinfo.pr_psargs);
+  while (psargs < psargs_end && argentry_iterator != args.end()) {
+    llvm::StringRef argentry = argentry_iterator->ref();
+    size_t len =
+        std::min<size_t>(std::distance(psargs, psargs_end), argentry.size());
+    auto arg_iterator = std::begin(argentry);
+    psargs = std::copy_n(arg_iterator, len, psargs);
+    if (psargs != psargs_end)
+      *(psargs++) = ' ';
+    ++argentry_iterator;
+  }
+  *(psargs - 1) = '\0';
+  return prpsinfo;
+}
+
 // Parse SIGINFO from NOTE entry
 ELFLinuxSigInfo::ELFLinuxSigInfo() { memset(this, 0, sizeof(ELFLinuxSigInfo)); }
 
diff --git a/lldb/source/Plugins/Process/elf-core/ThreadElfCore.h b/lldb/source/Plugins/Process/elf-core/ThreadElfCore.h
index 2f3ed2a017790a0..3fa0b8b0eedb0b7 100644
--- a/lldb/source/Plugins/Process/elf-core/ThreadElfCore.h
+++ b/lldb/source/Plugins/Process/elf-core/ThreadElfCore.h
@@ -13,6 +13,7 @@
 #include "lldb/Target/Thread.h"
 #include "lldb/Utility/DataExtractor.h"
 #include "llvm/ADT/DenseMap.h"
+#include <optional>
 #include <string>
 
 struct compat_timeval {
@@ -20,6 +21,10 @@ struct compat_timeval {
   alignas(8) uint64_t tv_usec;
 };
 
+namespace lldb_private {
+class ProcessInstanceInfo;
+}
+
 // PRSTATUS structure's size differs based on architecture.
 // This is the layout in the x86-64 arch.
 // In the i386 case we parse it manually and fill it again
@@ -56,6 +61,9 @@ struct ELFLinuxPrStatus {
   lldb_private::Status Parse(const lldb_private::DataExtractor &data,
                              const lldb_private::ArchSpec &arch);
 
+  static std::optional<ELFLinuxPrStatus>
+  Populate(const lldb::ThreadSP &thread_sp);
+
   // Return the bytesize of the structure
   // 64 bit - just sizeof
   // 32 bit - hardcoded because we are reusing the struct, but some of the
@@ -112,6 +120,13 @@ struct ELFLinuxPrPsInfo {
   lldb_private::Status Parse(const lldb_private::DataExtractor &data,
                              const lldb_private::ArchSpec &arch);
 
+  static std::optional<ELFLinuxPrPsInfo>
+  Populate(const lldb::ProcessSP &process_sp);
+
+  static std::optional<ELFLinuxPrPsInfo>
+  Populate(const lldb_private::ProcessInstanceInfo &info,
+           lldb::StateType state);
+
   // Return the bytesize of the structure
   // 64 bit - just sizeof
   // 32 bit - hardcoded because we are reusing the struct, but some of the
diff --git a/lldb/unittests/Process/CMakeLists.txt b/lldb/unittests/Process/CMakeLists.txt
index 5fbecfcfa25ebe1..d3b37e006fd899a 100644
--- a/lldb/unittests/Process/CMakeLists.txt
+++ b/lldb/unittests/Process/CMakeLists.txt
@@ -1,5 +1,6 @@
 add_subdirectory(gdb-remote)
 if (CMAKE_SYSTEM_NAME MATCHES "Linux|Android")
+  add_subdirectory(elf-core)
   add_subdirectory(Linux)
   add_subdirectory(POSIX)
 endif()
diff --git a/lldb/unittests/Process/elf-core/CMakeLists.txt b/lldb/unittests/Process/elf-core/CMakeLists.txt
new file mode 100644
index 000000000000000..b852a3ffb863c1d
--- /dev/null
+++ b/lldb/unittests/Process/elf-core/CMakeLists.txt
@@ -0,0 +1,15 @@
+add_lldb_unittest(ProcessElfCoreTests
+  ThreadElfCoreTest.cpp
+
+  LINK_LIBS
+    lldbCore
+    lldbHost
+    lldbUtilityHelpers
+    lldbPluginProcessElfCore
+    lldbPluginPlatformLinux
+
+    LLVMTestingSupport
+
+  LINK_COMPONENTS
+    Support
+  )
diff --git a/lldb/unittests/Process/elf-core/ThreadElfCoreTest.cpp b/lldb/unittests/Process/elf-core/ThreadElfCoreTest.cpp
new file mode 100644
index 000000000000000..ce146f62b0d8264
--- /dev/null
+++ b/lldb/unittests/Process/elf-core/ThreadElfCoreTest.cpp
@@ -0,0 +1,174 @@
+//===-- ThreadElfCoreTest.cpp ------------------------------------*- C++
+//-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#include "Plugins/Process/elf-core/ThreadElfCore.h"
+#include "Plugins/Platform/Linux/PlatformLinux.h"
+#include "TestingSupport/TestUtilities.h"
+#include "lldb/Core/Debugger.h"
+#include "lldb/Host/FileSystem.h"
+#include "lldb/Host/HostInfo.h"
+#include "lldb/Target/Process.h"
+#include "lldb/Target/Target.h"
+#include "lldb/Target/Thread.h"
+#include "lldb/Utility/Listener.h"
+#include "llvm/TargetParser/Triple.h"
+#include "gtest/gtest.h"
+
+#include <memory>
+#include <mutex>
+#include <unistd.h>
+
+using namespace lldb_private;
+
+namespace {
+
+struct ElfCoreTest : public testing::Test {
+  static void SetUpTestCase() {
+    FileSystem::Initialize();
+    HostInfo::Initialize();
+    platform_linux::PlatformLinux::Initialize();
+    std::call_once(TestUtilities::g_debugger_initialize_flag,
+                   []() { Debugger::Initialize(nullptr); });
+  }
+  static void TearDownTestCase() {
+    platform_linux::PlatformLinux::Terminate();
+    HostInfo::Terminate();
+    FileSystem::Terminate();
+  }
+};
+
+struct DummyProcess : public Process {
+  DummyProcess(lldb::TargetSP target_sp, lldb::ListenerSP listener_sp)
+      : Process(target_sp, listener_sp) {
+    SetID(getpid());
+  }
+
+  bool CanDebug(lldb::TargetSP target, bool plugin_specified_by_name) override {
+    return true;
+  }
+  Status DoDestroy() override { return {}; }
+  void RefreshStateAfterStop() override {}
+  size_t DoReadMemory(lldb::addr_t vm_addr, void *buf, size_t size,
+                      Status &error) override {
+    return 0;
+  }
+  bool DoUpdateThreadList(ThreadList &old_thread_list,
+                          ThreadList &new_thread_list) override {
+    return false;
+  }
+  llvm::StringRef GetPluginName() override { return "Dummy"; }
+};
+
+struct DummyThread : public Thread {
+  using Thread::Thread;
+
+  ~DummyThread() override { DestroyThread(); }
+
+  void RefreshStateAfterStop() override {}
+
+  lldb::RegisterContextSP GetRegisterContext() override { return nullptr; }
+
+  lldb::RegisterContextSP
+  CreateRegisterContextForFrame(StackFrame *frame) override {
+    return nullptr;
+  }
+
+  bool CalculateStopInfo() override { return false; }
+};
+
+lldb::TargetSP CreateTarget(lldb::DebuggerSP &debugger_sp, ArchSpec &arch) {
+  lldb::PlatformSP platform_sp;
+  lldb::TargetSP target_sp;
+  debugger_sp->GetTargetList().CreateTarget(
+      *debugger_sp, "", arch, eLoadDependentsNo, platform_sp, target_sp);
+  return target_sp;
+}
+
+lldb::ThreadSP CreateThread(lldb::ProcessSP &process_sp) {
+  lldb::ThreadSP thread_sp =
+      std::make_shared<DummyThread>(*process_sp.get(), gettid());
+  if (thread_sp == nullptr) {
+    return nullptr;
+  }
+  process_sp->GetThreadList().AddThread(thread_sp);
+
+  return thread_sp;
+}
+
+} // namespace
+
+TEST_F(ElfCoreTest, PopulatePrpsInfoTest) {
+  ArchSpec arch{HostInfo::GetTargetTriple()};
+  lldb::DebuggerSP debugger_sp = Debugger::CreateInstance();
+  ASSERT_TRUE(debugger_sp);
+
+  lldb::TargetSP target_sp = CreateTarget(debugger_sp, arch);
+  ASSERT_TRUE(target_sp);
+
+  lldb::ListenerSP listener_sp(Listener::MakeListener("dummy"));
+  lldb::ProcessSP process_sp =
+      std::make_shared<DummyProcess>(target_sp, listener_sp);
+  ASSERT_TRUE(process_sp);
+  auto prpsinfo_opt = ELFLinuxPrPsInfo::Populate(process_sp);
+  ASSERT_TRUE(prpsinfo_opt.has_value());
+  ASSERT_EQ(prpsinfo_opt->pr_pid, getpid());
+  ASSERT_EQ(prpsinfo_opt->pr_state, 0);
+  ASSERT_EQ(prpsinfo_opt->pr_sname, 'R');
+  ASSERT_EQ(prpsinfo_opt->pr_zomb, 0);
+  ASSERT_EQ(prpsinfo_opt->pr_nice, 0);
+  ASSERT_EQ(prpsinfo_opt->pr_flag, 0UL);
+  ASSERT_EQ(prpsinfo_opt->pr_uid, getuid());
+  ASSERT_EQ(prpsinfo_opt->pr_gid, getgid());
+  ASSERT_EQ(prpsinfo_opt->pr_pid, getpid());
+  ASSERT_EQ(prpsinfo_opt->pr_ppid, getppid());
+  ASSERT_EQ(prpsinfo_opt->pr_pgrp, getpgrp());
+  ASSERT_EQ(prpsinfo_opt->pr_sid, getsid(getpid()));
+  ASSERT_EQ(std::string{prpsinfo_opt->pr_fname}, "ProcessElfCoreT");
+  ASSERT_TRUE(std::string{prpsinfo_opt->pr_psargs}.empty());
+  lldb_private::ProcessInstanceInfo info;
+  ASSERT_TRUE(process_sp->GetProcessInfo(info));
+  const char *args[] = {"a.out", "--foo=bar", "--baz=boo", nullptr};
+  info.SetArguments(args, true);
+  prpsinfo_opt =
+      ELFLinuxPrPsInfo::Populate(info, lldb::StateType::eStateStopped);
+  ASSERT_TRUE(prpsinfo_opt.has_value());
+  ASSERT_EQ(prpsinfo_opt->pr_pid, getpid());
+  ASSERT_EQ(prpsinfo_opt->pr_state, 3);
+  ASSERT_EQ(prpsinfo_opt->pr_sname, 'T');
+  ASSERT_EQ(std::string{prpsinfo_opt->pr_fname}, "a.out");
+  ASSERT_FALSE(std::string{prpsinfo_opt->pr_psargs}.empty());
+  ASSERT_EQ(std::string{prpsinfo_opt->pr_psargs}, "a.out --foo=bar --baz=boo");
+}
+
+TEST_F(ElfCoreTest, PopulatePrStatusTest) {
+  ArchSpec arch{HostInfo::GetTargetTriple()};
+  lldb::DebuggerSP debugger_sp = Debugger::CreateInstance();
+  ASSERT_TRUE(debugger_sp);
+
+  lldb::TargetSP target_sp = CreateTarget(debugger_sp, arch);
+  ASSERT_TRUE(target_sp);
+
+  lldb::ListenerSP listener_sp(Listener::MakeListener("dummy"));
+  lldb::ProcessSP process_sp =
+      std::make_shared<DummyProcess>(target_sp, listener_sp);
+  ASSERT_TRUE(process_sp);
+  lldb::ThreadSP thread_sp = CreateThread(process_sp);
+  ASSERT_TRUE(thread_sp);
+  auto prstatus_opt = ELFLinuxPrStatus::Populate(thread_sp);
+  ASSERT_TRUE(prstatus_opt.has_value());
+  ASSERT_EQ(prstatus_opt->si_signo, 0);
+  ASSERT_EQ(prstatus_opt->si_code, 0);
+  ASSERT_EQ(prstatus_opt->si_errno, 0);
+  ASSERT_EQ(prstatus_opt->pr_cursig, 0);
+  ASSERT_EQ(prstatus_opt->pr_sigpend, 0UL);
+  ASSERT_EQ(prstatus_opt->pr_sighold, 0UL);
+  ASSERT_EQ(prstatus_opt->pr_pid, static_cast<uint32_t>(gettid()));
+  ASSERT_EQ(prstatus_opt->pr_ppid, static_cast<uint32_t>(getppid()));
+  ASSERT_EQ(prstatus_opt->pr_pgrp, static_cast<uint32_t>(getpgrp()));
+  ASSERT_EQ(prstatus_opt->pr_sid, static_cast<uint32_t>(getsid(gettid())));
+}

From 42944da5ba7617bbc02f341e9ef401c325310a73 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andrzej=20Warzy=C5=84ski?= <andrzej.warzynski@arm.com>
Date: Fri, 16 Aug 2024 16:53:53 +0100
Subject: [PATCH 193/441] [mlir][vector] Group re-order patterns together
 (#102856)

Group all patterns that re-order vector.transpose and vector.broadcast
Ops (*) under `populateSinkVectorOpsPatterns`. These patterns are
normally used to "sink" redundant Vector Ops, hence grouping together.
Example:

```mlir
%at = vector.transpose %a, [1, 0]: vector<4x2xf32> to vector<2x4xf32>
%bt = vector.transpose %b, [1, 0]: vector<4x2xf32> to vector<2x4xf32>
%r = arith.addf %at, %bt : vector<2x4xf32>
```
would get converted to:
```mlir
%0 = arith.addf %a, %b : vector<4x2xf32>
%r = vector.transpose %0, [1, 0] : vector<2x4xf32>
```

This patch also moves all tests for these patterns so that all of them
are:
  * run under one test-flag: `test-vector-sink-patterns`,
  * located in one file: "vector-sink.mlir".

To facilitate this change:
  * `-test-sink-vector-broadcast` is renamed as
    `test-vector-sink-patterns`,
  * "sink-vector-broadcast.mlir" is renamed as "vector-sink.mlir",
  * tests for `ReorderCastOpsOnBroadcast` and
    `ReorderElementwiseOpsOnTranspose` patterns are moved from
    "vector-reduce-to-contract.mlir" to "vector-sink.mlir",
  * `ReorderElementwiseOpsOnTranspose` patterns are removed from
    `populateVectorReductionToContractPatterns` and added to (newly
    created) `populateSinkVectorOpsPatterns`,
  * `ReorderCastOpsOnBroadcast` patterns are removed from
    `populateVectorReductionToContractPatterns` - these are already
    present in `populateSinkVectorOpsPatterns`.

This should allow us better layering and more straightforward testing.
For the latter, the goal is to be able to easily identify which pattern
a particular test is exercising (especially when it's a specific
pattern).

NOTES FOR DOWNSTREAM USERS

In order to preserve the current functionality, please make sure to add
  * `populateSinkVectorOpsPatterns`,

wherever you are using `populateVectorReductionToContractPatterns`.
Also, rename `populateSinkVectorBroadcastPatterns` as
`populateSinkVectorOpsPatterns`.

(*) I didn't notice any other re-order patterns.
---
 .../Vector/Transforms/VectorRewritePatterns.h |  19 ++-
 .../TransformOps/LinalgTransformOps.cpp       |   2 +-
 .../Vector/Transforms/VectorTransforms.cpp    |  12 +-
 .../Vector/vector-reduce-to-contract.mlir     | 122 ------------------
 ...vector-broadcast.mlir => vector-sink.mlir} | 114 +++++++++++++++-
 .../Dialect/Vector/TestVectorTransforms.cpp   |  18 +--
 6 files changed, 145 insertions(+), 142 deletions(-)
 rename mlir/test/Dialect/Vector/{sink-vector-broadcast.mlir => vector-sink.mlir} (65%)

diff --git a/mlir/include/mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h b/mlir/include/mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h
index 10970fd03e6eb2e..456d021bc793581 100644
--- a/mlir/include/mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h
+++ b/mlir/include/mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h
@@ -144,9 +144,22 @@ void populateVectorTransferFullPartialPatterns(
 void populateVectorTransferCollapseInnerMostContiguousDimsPatterns(
     RewritePatternSet &patterns, PatternBenefit benefit = 1);
 
-/// Patterns that remove redundant vector broadcasts.
-void populateSinkVectorBroadcastPatterns(RewritePatternSet &patterns,
-                                         PatternBenefit benefit = 1);
+/// Patterns that remove redundant Vector Ops by re-ordering them with
+/// e.g. elementwise Ops:
+/// ```
+/// %at = vector.transpose %a, [1, 0]: vector<4x2xf32> to vector<2x4xf32>
+/// %bt = vector.transpose %b, [1, 0]: vector<4x2xf32> to vector<2x4xf32>
+/// %r = arith.addf %at, %bt : vector<2x4xf32>
+/// ```
+/// gets converted to:
+/// ```
+/// %0 = arith.addf %a, %b : vector<4x2xf32>
+/// %r = vector.transpose %0, [1, 0] : vector<2x4xf32>
+/// ```
+/// At the moment, these patterns are limited to vector.broadcast and
+/// vector.transpose.
+void populateSinkVectorOpsPatterns(RewritePatternSet &patterns,
+                                   PatternBenefit benefit = 1);
 
 /// Patterns that fold chained vector reductions. These patterns assume that
 /// elementwise operations (e.g., `arith.addf` with vector operands) are
diff --git a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
index fbf4e29024f7c20..29b5631f61b4885 100644
--- a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
+++ b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
@@ -3452,7 +3452,7 @@ transform::VectorizeChildrenAndApplyPatternsOp::applyToOne(
   if (!getDisableMultiReductionToContractPatterns())
     vector::populateVectorReductionToContractPatterns(patterns);
 
-  vector::populateSinkVectorBroadcastPatterns(patterns);
+  vector::populateSinkVectorOpsPatterns(patterns);
 
   patterns.add<linalg::LinalgCopyVTRForwardingPattern,
                linalg::LinalgCopyVTWForwardingPattern>(ctx,
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp
index ccbaa3e9759975c..ad4e42b31962e17 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp
@@ -2030,8 +2030,7 @@ void mlir::vector::populateVectorContractCanonicalizeMatmulToMMT(
 void mlir::vector::populateVectorReductionToContractPatterns(
     RewritePatternSet &patterns, PatternBenefit benefit) {
   patterns.add<MultiReduceToContract, CombineContractBroadcast,
-               CombineContractABTranspose, CombineContractResultTranspose,
-               ReorderCastOpsOnBroadcast, ReorderElementwiseOpsOnTranspose>(
+               CombineContractABTranspose, CombineContractResultTranspose>(
       patterns.getContext(), benefit);
 }
 
@@ -2043,10 +2042,11 @@ void mlir::vector::
                                                    benefit);
 }
 
-void mlir::vector::populateSinkVectorBroadcastPatterns(
-    RewritePatternSet &patterns, PatternBenefit benefit) {
-  patterns.add<ReorderCastOpsOnBroadcast, ReorderElementwiseOpsOnBroadcast>(
-      patterns.getContext(), benefit);
+void mlir::vector::populateSinkVectorOpsPatterns(RewritePatternSet &patterns,
+                                                 PatternBenefit benefit) {
+  patterns.add<ReorderElementwiseOpsOnTranspose, ReorderCastOpsOnBroadcast,
+               ReorderElementwiseOpsOnBroadcast>(patterns.getContext(),
+                                                 benefit);
 }
 
 void mlir::vector::populateChainedVectorReductionFoldingPatterns(
diff --git a/mlir/test/Dialect/Vector/vector-reduce-to-contract.mlir b/mlir/test/Dialect/Vector/vector-reduce-to-contract.mlir
index c0dbea81df892a8..24070dbf017a58f 100644
--- a/mlir/test/Dialect/Vector/vector-reduce-to-contract.mlir
+++ b/mlir/test/Dialect/Vector/vector-reduce-to-contract.mlir
@@ -245,128 +245,6 @@ func.func @contract_broadcast_would_have_no_reduction_dim_pair(%arg0 : vector<1x
 }
 
 
-//===----------------------------------------------------------------------===//
-// [Pattern: ReorderCastOpsOnBroadcast]
-//
-// Reorder casting ops and vector ops. The casting ops have almost identical
-// pattern, so only arith.extsi op is tested.
-//
-// TODO: Potential duplication with sink-vector-broadcast.mlir
-//===----------------------------------------------------------------------===//
-
-// -----
-
-func.func @broadcast_vector_extsi(%a : vector<4xi8>) -> vector<2x4xi32> {
-  // CHECK: %[[EXT:.+]] = arith.extsi %{{.+}} : vector<4xi8> to vector<4xi32>
-  // CHECK: vector.broadcast %[[EXT:.+]] : vector<4xi32> to vector<2x4xi32>
-  %b = vector.broadcast %a : vector<4xi8> to vector<2x4xi8>
-  %r = arith.extsi %b : vector<2x4xi8> to vector<2x4xi32>
-  return %r : vector<2x4xi32>
-}
-
-// -----
-
-func.func @broadcast_scalar_extsi(%a : i8) -> vector<2x4xi32> {
-  // CHECK: %[[EXT:.+]] = arith.extsi %{{.+}} : i8 to i32
-  // CHECK: vector.broadcast %[[EXT]] : i32 to vector<2x4xi32>
-  %b = vector.broadcast %a : i8 to vector<2x4xi8>
-  %r = arith.extsi %b : vector<2x4xi8> to vector<2x4xi32>
-  return %r : vector<2x4xi32>
-}
-
-// -----
-
-//===----------------------------------------------------------------------===//
-// [Pattern: ReorderElementwiseOpsOnTranspose]
-//
-// TODO: Potential duplication with sink-vector-broadcast.mlir
-//===----------------------------------------------------------------------===//
-func.func @transpose_extsi(%a : vector<4x2xi8>) -> vector<2x4xi32> {
-  // CHECK: %[[EXT:.+]] = arith.extsi %{{.+}} : vector<4x2xi8> to vector<4x2xi32>
-  // CHECK: vector.transpose %[[EXT]], [1, 0] : vector<4x2xi32> to vector<2x4xi32>
-  %b = vector.transpose %a, [1, 0]: vector<4x2xi8> to vector<2x4xi8>
-  %r = arith.extsi %b : vector<2x4xi8> to vector<2x4xi32>
-  return %r : vector<2x4xi32>
-}
-
-//===----------------------------------------------------------------------===//
-// Reorder elementwise ops and vector ops.
-// TODO: Potential duplication with sink-vector-broadcast.mlir
-//===----------------------------------------------------------------------===//
-
-// -----
-
-// CHECK-LABEL: func @transpose_elementwise_same_type
-//  CHECK-SAME: (%[[A:.+]]: vector<4x2xf32>, %[[B:.+]]: vector<4x2xf32>)
-//       CHECK:   %[[ADD:.+]] = arith.addf %[[A]], %[[B]] : vector<4x2xf32>
-//       CHECK:   %[[T:.+]] = vector.transpose %[[ADD]], [1, 0]
-//       CHECK:   return %[[T]]
-
-func.func @transpose_elementwise_same_type(%a : vector<4x2xf32>, %b : vector<4x2xf32>) -> vector<2x4xf32> {
-  %at = vector.transpose %a, [1, 0]: vector<4x2xf32> to vector<2x4xf32>
-  %bt = vector.transpose %b, [1, 0]: vector<4x2xf32> to vector<2x4xf32>
-  %r = arith.addf %at, %bt : vector<2x4xf32>
-  return %r : vector<2x4xf32>
-}
-
-// -----
-
-// CHECK-LABEL: func @transpose_elementwise_diff_operand_types
-//  CHECK-SAME: (%[[COND:.+]]: vector<4x2xi1>, %[[A:.+]]: vector<4x2xf32>, %[[B:.+]]: vector<4x2xf32>)
-//       CHECK:   %[[S:.+]] = arith.select %[[COND]], %[[A]], %[[B]] : vector<4x2xi1>, vector<4x2xf32>
-//       CHECK:   %[[T:.+]] = vector.transpose %[[S]], [1, 0] : vector<4x2xf32> to vector<2x4xf32>
-//       CHECK:   return %[[T]]
-func.func @transpose_elementwise_diff_operand_types(%cond: vector<4x2xi1>, %a : vector<4x2xf32>, %b : vector<4x2xf32>) -> vector<2x4xf32> {
-  %condt = vector.transpose %cond, [1, 0]: vector<4x2xi1> to vector<2x4xi1>
-  %at = vector.transpose %a, [1, 0]: vector<4x2xf32> to vector<2x4xf32>
-  %bt = vector.transpose %b, [1, 0]: vector<4x2xf32> to vector<2x4xf32>
-  %r = arith.select %condt, %at, %bt : vector<2x4xi1>, vector<2x4xf32>
-  return %r : vector<2x4xf32>
-}
-
-// -----
-
-// CHECK-LABEL: func @transpose_elementwise_diff_operand_result_type
-//  CHECK-SAME: (%[[A:.+]]: vector<4x2xf32>, %[[B:.+]]: vector<4x2xf32>)
-//       CHECK:   %[[CMP:.+]] = arith.cmpf olt, %[[A]], %[[B]] : vector<4x2xf32>
-//       CHECK:   %[[T:.+]] = vector.transpose %[[CMP]], [1, 0] : vector<4x2xi1> to vector<2x4xi1>
-//       CHECK:   return %[[T]]
-func.func @transpose_elementwise_diff_operand_result_type(%a : vector<4x2xf32>, %b : vector<4x2xf32>) -> vector<2x4xi1> {
-  %at = vector.transpose %a, [1, 0]: vector<4x2xf32> to vector<2x4xf32>
-  %bt = vector.transpose %b, [1, 0]: vector<4x2xf32> to vector<2x4xf32>
-  %r = arith.cmpf olt, %at, %bt : vector<2x4xf32>
-  return %r : vector<2x4xi1>
-}
-
-// -----
-
-// CHECK-LABEL: func @transpose_elementwise_splat_constant
-//  CHECK-SAME: (%[[A:.+]]: vector<4x6x3x2xf32>)
-//       CHECK:   %[[B:.+]] = arith.constant dense<5.000000e+00> : vector<4x6x3x2xf32>
-//       CHECK:   %[[ADD:.+]] = arith.addf %[[A]], %[[B]] : vector<4x6x3x2xf32>
-//       CHECK:   %[[T:.+]] = vector.transpose %[[ADD]], [1, 0, 3, 2] : vector<4x6x3x2xf32> to vector<6x4x2x3xf32>
-//       CHECK:   return %[[T:.+]] : vector<6x4x2x3xf32>
-
-func.func @transpose_elementwise_splat_constant(%a : vector<4x6x3x2xf32>) -> vector<6x4x2x3xf32> {
-  %b = arith.constant dense<5.0> : vector<6x4x2x3xf32>
-  %at = vector.transpose %a, [1, 0, 3, 2]: vector<4x6x3x2xf32> to vector<6x4x2x3xf32>
-  %r = arith.addf %at, %b : vector<6x4x2x3xf32>
-  return %r : vector<6x4x2x3xf32>
-}
-
-// -----
-
-// CHECK-LABEL: func @transpose_elementwise_diff_map
-//       CHECK:   vector.transpose
-//       CHECK:   vector.transpose
-//       CHECK:   arith.addf
-func.func @transpose_elementwise_diff_map(%a : vector<4x6x3x2xf32>, %b: vector<6x2x4x3xf32>) -> vector<6x4x2x3xf32> {
-  %at = vector.transpose %a, [1, 0, 3, 2]: vector<4x6x3x2xf32> to vector<6x4x2x3xf32>
-  %bt = vector.transpose %b, [0, 2, 1, 3]: vector<6x2x4x3xf32> to vector<6x4x2x3xf32>
-  %r = arith.addf %at, %bt : vector<6x4x2x3xf32>
-  return %r : vector<6x4x2x3xf32>
-}
-
 // -----
 
 // CHECK-DAG: #[[$LHS_MAP:.+]] = affine_map<(d0, d1, d2, d3) -> (d0, d3, d1)>
diff --git a/mlir/test/Dialect/Vector/sink-vector-broadcast.mlir b/mlir/test/Dialect/Vector/vector-sink.mlir
similarity index 65%
rename from mlir/test/Dialect/Vector/sink-vector-broadcast.mlir
rename to mlir/test/Dialect/Vector/vector-sink.mlir
index dd2e98831a708a0..5a3699333265c1c 100644
--- a/mlir/test/Dialect/Vector/sink-vector-broadcast.mlir
+++ b/mlir/test/Dialect/Vector/vector-sink.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -test-sink-vector-broadcast -split-input-file | FileCheck %s
+// RUN: mlir-opt %s -test-vector-sink-patterns -split-input-file | FileCheck %s
 
 //-----------------------------------------------------------------------------
 // [Pattern: ReorderElementwiseOpsOnBroadcast]
@@ -208,3 +208,115 @@ func.func @negative_op_only_supports_vectors(%arg0 : f32) -> vector<1xf32> {
   %1 = vector.fma %0, %0, %0 : vector<1xf32>
   return %1 : vector<1xf32>
 }
+
+//===----------------------------------------------------------------------===//
+// [Pattern: ReorderCastOpsOnBroadcast]
+//
+// Reorder casting ops and vector ops. The casting ops have almost identical
+// pattern, so only arith.extsi op is tested.
+//===----------------------------------------------------------------------===//
+
+// -----
+
+func.func @broadcast_vector_extsi(%a : vector<4xi8>) -> vector<2x4xi32> {
+  // CHECK: %[[EXT:.+]] = arith.extsi %{{.+}} : vector<4xi8> to vector<4xi32>
+  // CHECK: vector.broadcast %[[EXT:.+]] : vector<4xi32> to vector<2x4xi32>
+  %b = vector.broadcast %a : vector<4xi8> to vector<2x4xi8>
+  %r = arith.extsi %b : vector<2x4xi8> to vector<2x4xi32>
+  return %r : vector<2x4xi32>
+}
+
+// -----
+
+func.func @broadcast_scalar_extsi(%a : i8) -> vector<2x4xi32> {
+  // CHECK: %[[EXT:.+]] = arith.extsi %{{.+}} : i8 to i32
+  // CHECK: vector.broadcast %[[EXT]] : i32 to vector<2x4xi32>
+  %b = vector.broadcast %a : i8 to vector<2x4xi8>
+  %r = arith.extsi %b : vector<2x4xi8> to vector<2x4xi32>
+  return %r : vector<2x4xi32>
+}
+
+//===----------------------------------------------------------------------===//
+// [Pattern: ReorderElementwiseOpsOnTranspose]
+//===----------------------------------------------------------------------===//
+
+func.func @transpose_extsi(%a : vector<4x2xi8>) -> vector<2x4xi32> {
+  // CHECK: %[[EXT:.+]] = arith.extsi %{{.+}} : vector<4x2xi8> to vector<4x2xi32>
+  // CHECK: vector.transpose %[[EXT]], [1, 0] : vector<4x2xi32> to vector<2x4xi32>
+  %b = vector.transpose %a, [1, 0]: vector<4x2xi8> to vector<2x4xi8>
+  %r = arith.extsi %b : vector<2x4xi8> to vector<2x4xi32>
+  return %r : vector<2x4xi32>
+}
+
+// -----
+
+// CHECK-LABEL: func @transpose_elementwise_same_type
+//  CHECK-SAME: (%[[A:.+]]: vector<4x2xf32>, %[[B:.+]]: vector<4x2xf32>)
+//       CHECK:   %[[ADD:.+]] = arith.addf %[[A]], %[[B]] : vector<4x2xf32>
+//       CHECK:   %[[T:.+]] = vector.transpose %[[ADD]], [1, 0]
+//       CHECK:   return %[[T]]
+
+func.func @transpose_elementwise_same_type(%a : vector<4x2xf32>, %b : vector<4x2xf32>) -> vector<2x4xf32> {
+  %at = vector.transpose %a, [1, 0]: vector<4x2xf32> to vector<2x4xf32>
+  %bt = vector.transpose %b, [1, 0]: vector<4x2xf32> to vector<2x4xf32>
+  %r = arith.addf %at, %bt : vector<2x4xf32>
+  return %r : vector<2x4xf32>
+}
+
+// -----
+
+// CHECK-LABEL: func @transpose_elementwise_diff_operand_types
+//  CHECK-SAME: (%[[COND:.+]]: vector<4x2xi1>, %[[A:.+]]: vector<4x2xf32>, %[[B:.+]]: vector<4x2xf32>)
+//       CHECK:   %[[S:.+]] = arith.select %[[COND]], %[[A]], %[[B]] : vector<4x2xi1>, vector<4x2xf32>
+//       CHECK:   %[[T:.+]] = vector.transpose %[[S]], [1, 0] : vector<4x2xf32> to vector<2x4xf32>
+//       CHECK:   return %[[T]]
+func.func @transpose_elementwise_diff_operand_types(%cond: vector<4x2xi1>, %a : vector<4x2xf32>, %b : vector<4x2xf32>) -> vector<2x4xf32> {
+  %condt = vector.transpose %cond, [1, 0]: vector<4x2xi1> to vector<2x4xi1>
+  %at = vector.transpose %a, [1, 0]: vector<4x2xf32> to vector<2x4xf32>
+  %bt = vector.transpose %b, [1, 0]: vector<4x2xf32> to vector<2x4xf32>
+  %r = arith.select %condt, %at, %bt : vector<2x4xi1>, vector<2x4xf32>
+  return %r : vector<2x4xf32>
+}
+
+// -----
+
+// CHECK-LABEL: func @transpose_elementwise_diff_operand_result_type
+//  CHECK-SAME: (%[[A:.+]]: vector<4x2xf32>, %[[B:.+]]: vector<4x2xf32>)
+//       CHECK:   %[[CMP:.+]] = arith.cmpf olt, %[[A]], %[[B]] : vector<4x2xf32>
+//       CHECK:   %[[T:.+]] = vector.transpose %[[CMP]], [1, 0] : vector<4x2xi1> to vector<2x4xi1>
+//       CHECK:   return %[[T]]
+func.func @transpose_elementwise_diff_operand_result_type(%a : vector<4x2xf32>, %b : vector<4x2xf32>) -> vector<2x4xi1> {
+  %at = vector.transpose %a, [1, 0]: vector<4x2xf32> to vector<2x4xf32>
+  %bt = vector.transpose %b, [1, 0]: vector<4x2xf32> to vector<2x4xf32>
+  %r = arith.cmpf olt, %at, %bt : vector<2x4xf32>
+  return %r : vector<2x4xi1>
+}
+
+// -----
+
+// CHECK-LABEL: func @transpose_elementwise_splat_constant
+//  CHECK-SAME: (%[[A:.+]]: vector<4x6x3x2xf32>)
+//       CHECK:   %[[B:.+]] = arith.constant dense<5.000000e+00> : vector<4x6x3x2xf32>
+//       CHECK:   %[[ADD:.+]] = arith.addf %[[A]], %[[B]] : vector<4x6x3x2xf32>
+//       CHECK:   %[[T:.+]] = vector.transpose %[[ADD]], [1, 0, 3, 2] : vector<4x6x3x2xf32> to vector<6x4x2x3xf32>
+//       CHECK:   return %[[T:.+]] : vector<6x4x2x3xf32>
+
+func.func @transpose_elementwise_splat_constant(%a : vector<4x6x3x2xf32>) -> vector<6x4x2x3xf32> {
+  %b = arith.constant dense<5.0> : vector<6x4x2x3xf32>
+  %at = vector.transpose %a, [1, 0, 3, 2]: vector<4x6x3x2xf32> to vector<6x4x2x3xf32>
+  %r = arith.addf %at, %b : vector<6x4x2x3xf32>
+  return %r : vector<6x4x2x3xf32>
+}
+
+// -----
+
+// CHECK-LABEL: func @transpose_elementwise_diff_map
+//       CHECK:   vector.transpose
+//       CHECK:   vector.transpose
+//       CHECK:   arith.addf
+func.func @transpose_elementwise_diff_map(%a : vector<4x6x3x2xf32>, %b: vector<6x2x4x3xf32>) -> vector<6x4x2x3xf32> {
+  %at = vector.transpose %a, [1, 0, 3, 2]: vector<4x6x3x2xf32> to vector<6x4x2x3xf32>
+  %bt = vector.transpose %b, [0, 2, 1, 3]: vector<6x2x4x3xf32> to vector<6x4x2x3xf32>
+  %r = arith.addf %at, %bt : vector<6x4x2x3xf32>
+  return %r : vector<6x4x2x3xf32>
+}
diff --git a/mlir/test/lib/Dialect/Vector/TestVectorTransforms.cpp b/mlir/test/lib/Dialect/Vector/TestVectorTransforms.cpp
index 29c763b622e877d..72aaa7dc4f89734 100644
--- a/mlir/test/lib/Dialect/Vector/TestVectorTransforms.cpp
+++ b/mlir/test/lib/Dialect/Vector/TestVectorTransforms.cpp
@@ -374,27 +374,27 @@ struct TestVectorTransferCollapseInnerMostContiguousDims
   }
 };
 
-struct TestSinkVectorBroadcast
-    : public PassWrapper<TestSinkVectorBroadcast, OperationPass<func::FuncOp>> {
-  MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestSinkVectorBroadcast)
+struct TestVectorSinkPatterns
+    : public PassWrapper<TestVectorSinkPatterns, OperationPass<func::FuncOp>> {
+  MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestVectorSinkPatterns)
 
-  TestSinkVectorBroadcast() = default;
-  TestSinkVectorBroadcast(const TestSinkVectorBroadcast &pass) = default;
+  TestVectorSinkPatterns() = default;
+  TestVectorSinkPatterns(const TestVectorSinkPatterns &pass) = default;
 
   void getDependentDialects(DialectRegistry &registry) const override {
     registry.insert<memref::MemRefDialect, affine::AffineDialect>();
   }
 
-  StringRef getArgument() const final { return "test-sink-vector-broadcast"; }
+  StringRef getArgument() const final { return "test-vector-sink-patterns"; }
 
   StringRef getDescription() const final {
     return "Test lowering patterns that eliminate redundant brodacast "
-           "operations.";
+           "and transpose operations.";
   }
 
   void runOnOperation() override {
     RewritePatternSet patterns(&getContext());
-    populateSinkVectorBroadcastPatterns(patterns);
+    populateSinkVectorOpsPatterns(patterns);
     (void)applyPatternsAndFoldGreedily(getOperation(), std::move(patterns));
   }
 };
@@ -919,7 +919,7 @@ void registerTestVectorLowerings() {
 
   PassRegistration<TestVectorTransferCollapseInnerMostContiguousDims>();
 
-  PassRegistration<TestSinkVectorBroadcast>();
+  PassRegistration<TestVectorSinkPatterns>();
 
   PassRegistration<TestVectorReduceToContractPatternsPatterns>();
 

From b221c37082707e35b492baa9ae8045c56b3ced0b Mon Sep 17 00:00:00 2001
From: R <rqou@berkeley.edu>
Date: Fri, 16 Aug 2024 17:14:16 +0100
Subject: [PATCH 194/441] [RISCV] Allow YAML file to control multilib selection
 (#98856)

This changes the bare-metal driver logic such that it _always_ tries
multilib.yaml if it exists, and it falls back to the hardwired/default
RISC-V multilib selection only if a multilib.yaml doesn't exist. In
contrast, the current behavior is that RISC-V can never use
multilib.yaml, but other targets will try it if it exists.

The flags `-march=` and `-mabi=` are exposed for multilib.yaml to match
on. There is no attempt to help YAML file creators to duplicate the
existing hard-wired multilib reuse logic -- they will have to implement
it using `Mappings`.

This should be backwards-compatible with existing sysroots, as
multilib.yaml was previously never used for RISC-V, and the behavior
doesn't change after this PR if the file doesn't exist.
---
 clang/lib/Driver/ToolChain.cpp                | 19 +++++++++++++++++++
 clang/lib/Driver/ToolChains/BareMetal.cpp     | 16 +++++++++-------
 .../test/Driver/print-multi-selection-flags.c | 15 +++++++++++++++
 3 files changed, 43 insertions(+), 7 deletions(-)

diff --git a/clang/lib/Driver/ToolChain.cpp b/clang/lib/Driver/ToolChain.cpp
index 013709999155217..0e8577b1115e38e 100644
--- a/clang/lib/Driver/ToolChain.cpp
+++ b/clang/lib/Driver/ToolChain.cpp
@@ -9,6 +9,7 @@
 #include "clang/Driver/ToolChain.h"
 #include "ToolChains/Arch/AArch64.h"
 #include "ToolChains/Arch/ARM.h"
+#include "ToolChains/Arch/RISCV.h"
 #include "ToolChains/Clang.h"
 #include "ToolChains/CommonArgs.h"
 #include "ToolChains/Flang.h"
@@ -44,6 +45,7 @@
 #include "llvm/Support/VersionTuple.h"
 #include "llvm/Support/VirtualFileSystem.h"
 #include "llvm/TargetParser/AArch64TargetParser.h"
+#include "llvm/TargetParser/RISCVISAInfo.h"
 #include "llvm/TargetParser/TargetParser.h"
 #include "llvm/TargetParser/Triple.h"
 #include <cassert>
@@ -268,6 +270,19 @@ static void getARMMultilibFlags(const Driver &D,
   }
 }
 
+static void getRISCVMultilibFlags(const Driver &D, const llvm::Triple &Triple,
+                                  const llvm::opt::ArgList &Args,
+                                  Multilib::flags_list &Result) {
+  std::string Arch = riscv::getRISCVArch(Args, Triple);
+  // Canonicalize arch for easier matching
+  auto ISAInfo = llvm::RISCVISAInfo::parseArchString(
+      Arch, /*EnableExperimentalExtensions*/ true);
+  if (!llvm::errorToBool(ISAInfo.takeError()))
+    Result.push_back("-march=" + (*ISAInfo)->toString());
+
+  Result.push_back(("-mabi=" + riscv::getRISCVABI(Args, Triple)).str());
+}
+
 Multilib::flags_list
 ToolChain::getMultilibFlags(const llvm::opt::ArgList &Args) const {
   using namespace clang::driver::options;
@@ -288,6 +303,10 @@ ToolChain::getMultilibFlags(const llvm::opt::ArgList &Args) const {
   case llvm::Triple::thumbeb:
     getARMMultilibFlags(D, Triple, Args, Result);
     break;
+  case llvm::Triple::riscv32:
+  case llvm::Triple::riscv64:
+    getRISCVMultilibFlags(D, Triple, Args, Result);
+    break;
   default:
     break;
   }
diff --git a/clang/lib/Driver/ToolChains/BareMetal.cpp b/clang/lib/Driver/ToolChains/BareMetal.cpp
index 102d12700229e68..0f5f32f3f8f495a 100644
--- a/clang/lib/Driver/ToolChains/BareMetal.cpp
+++ b/clang/lib/Driver/ToolChains/BareMetal.cpp
@@ -218,17 +218,19 @@ static std::string computeBaseSysRoot(const Driver &D,
 void BareMetal::findMultilibs(const Driver &D, const llvm::Triple &Triple,
                               const ArgList &Args) {
   DetectedMultilibs Result;
-  if (isRISCVBareMetal(Triple)) {
+  // Look for a multilib.yaml before trying target-specific hardwired logic.
+  // If it exists, always do what it specifies.
+  llvm::SmallString<128> MultilibPath(computeBaseSysRoot(D, Triple));
+  llvm::sys::path::append(MultilibPath, MultilibFilename);
+  if (D.getVFS().exists(MultilibPath)) {
+    findMultilibsFromYAML(*this, D, MultilibPath, Args, Result);
+    SelectedMultilibs = Result.SelectedMultilibs;
+    Multilibs = Result.Multilibs;
+  } else if (isRISCVBareMetal(Triple)) {
     if (findRISCVMultilibs(D, Triple, Args, Result)) {
       SelectedMultilibs = Result.SelectedMultilibs;
       Multilibs = Result.Multilibs;
     }
-  } else {
-    llvm::SmallString<128> MultilibPath(computeBaseSysRoot(D, Triple));
-    llvm::sys::path::append(MultilibPath, MultilibFilename);
-    findMultilibsFromYAML(*this, D, MultilibPath, Args, Result);
-    SelectedMultilibs = Result.SelectedMultilibs;
-    Multilibs = Result.Multilibs;
   }
 }
 
diff --git a/clang/test/Driver/print-multi-selection-flags.c b/clang/test/Driver/print-multi-selection-flags.c
index 9f58d1b557fd74d..2770a3ad5eaa1dd 100644
--- a/clang/test/Driver/print-multi-selection-flags.c
+++ b/clang/test/Driver/print-multi-selection-flags.c
@@ -58,3 +58,18 @@
 // RUN: %clang -print-multi-flags-experimental --target=aarch64-none-elf -march=armv9-a            | FileCheck --check-prefix=CHECK-SVE2 %s
 // CHECK-SVE2: --target=aarch64-unknown-none-elf
 // CHECK-SVE2: -march=armv{{.*}}-a{{.*}}+simd{{.*}}+sve{{.*}}+sve2{{.*}}
+
+// RUN: %clang -print-multi-flags-experimental --target=riscv32-none-elf -march=rv32g | FileCheck --check-prefix=CHECK-RV32 %s
+// CHECK-RV32: --target=riscv32-unknown-none-elf
+// CHECK-RV32: -mabi=ilp32d
+// CHECK-RV32: -march=rv32i{{[0-9]+p[0-9]+}}_m{{[0-9]+p[0-9]+}}_a{{[0-9]+p[0-9]+}}_f{{[0-9]+p[0-9]+}}_d{{[0-9]+p[0-9]+}}_zicsr{{[0-9]+p[0-9]+}}_zifencei{{[0-9]+p[0-9]+}}_zmmul{{[0-9]+p[0-9]+}}
+
+// RUN: %clang -print-multi-flags-experimental --target=riscv64-none-elf -march=rv64g | FileCheck --check-prefix=CHECK-RV64 %s
+// CHECK-RV64: --target=riscv64-unknown-none-elf
+// CHECK-RV64: -mabi=lp64d
+// CHECK-RV64: -march=rv64i{{[0-9]+p[0-9]+}}_m{{[0-9]+p[0-9]+}}_a{{[0-9]+p[0-9]+}}_f{{[0-9]+p[0-9]+}}_d{{[0-9]+p[0-9]+}}_zicsr{{[0-9]+p[0-9]+}}_zifencei{{[0-9]+p[0-9]+}}_zmmul{{[0-9]+p[0-9]+}}
+
+// RUN: %clang -print-multi-flags-experimental --target=riscv32-none-elf -march=rv32e_zicsr_c | FileCheck --check-prefix=CHECK-RV32E-ORDER %s
+// CHECK-RV32E-ORDER: --target=riscv32-unknown-none-elf
+// CHECK-RV32E-ORDER: -mabi=ilp32e
+// CHECK-RV32E-ORDER: -march=rv32e{{[0-9]+p[0-9]+}}_c{{[0-9]+p[0-9]+}}_zicsr{{[0-9]+p[0-9]+}}

From 1164e4aef2844ac3d35153ffe6376db9abda704a Mon Sep 17 00:00:00 2001
From: stefankoncarevic <skoncare@amd.com>
Date: Fri, 16 Aug 2024 18:19:39 +0200
Subject: [PATCH 195/441] [mlir][AMDGPU] Implement AMDGPU DPP operation in
 MLIR. (#89233)

Defined AMDGPU DPP operation in mlir to represent semantics. Introduced
a new enumeration attribute for different permutations and allowed for
different types of arguments. Implemented constant attribute handling
for ROCDL::DPPMovOp operation. The operation now correctly accepts
constant attributes for dppCtrl, rowMask, bankMask, boundCtrl, and
passes them to the corresponding LLVM intrinsic.
---
 llvm/include/llvm/IR/IntrinsicsAMDGPU.td      |   6 +
 mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td |  55 ++++++
 mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td  |  24 +++
 .../AMDGPUToROCDL/AMDGPUToROCDL.cpp           | 156 +++++++++++++++++-
 mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp  |  62 +++++++
 mlir/test/Conversion/AMDGPUToROCDL/dpp.mlir   | 139 ++++++++++++++++
 6 files changed, 439 insertions(+), 3 deletions(-)
 create mode 100644 mlir/test/Conversion/AMDGPUToROCDL/dpp.mlir

diff --git a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
index 8c25467cc5e4b14..7c93c19a410e434 100644
--- a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
+++ b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
@@ -2343,6 +2343,8 @@ def int_amdgcn_buffer_wbinvl1_vol :
 // VI Intrinsics
 //===----------------------------------------------------------------------===//
 
+// The llvm.amdgcn.mov.dpp.i32 intrinsic represents the mov.dpp operation in AMDGPU.
+// This operation is being deprecated and can be replaced with llvm.amdgcn.update.dpp.i32.
 // llvm.amdgcn.mov.dpp.i32 <src> <dpp_ctrl> <row_mask> <bank_mask> <bound_ctrl>
 def int_amdgcn_mov_dpp :
   Intrinsic<[llvm_anyint_ty],
@@ -2352,6 +2354,10 @@ def int_amdgcn_mov_dpp :
              ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>,
              ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>, IntrNoCallback, IntrNoFree]>;
 
+// The llvm.amdgcn.update.dpp.i32 intrinsic represents the update.dpp operation in AMDGPU.
+// It takes an old value, a source operand, a DPP control operand, a row mask, a bank mask, and a bound control.
+// This operation is equivalent to a sequence of v_mov_b32 operations.
+// It is preferred over llvm.amdgcn.mov.dpp.i32 for future use.
 // llvm.amdgcn.update.dpp.i32 <old> <src> <dpp_ctrl> <row_mask> <bank_mask> <bound_ctrl>
 // Should be equivalent to:
 // v_mov_b32 <dest> <old>
diff --git a/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td b/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td
index dddbd837c1cbc70..97e0580c8980800 100644
--- a/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td
+++ b/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td
@@ -402,6 +402,61 @@ def AMDGPU_RawBufferAtomicUminOp :
   let hasVerifier = 1;
 }
 
+def AMDGPU_DPPPerm : I32EnumAttr<"DPPPerm",
+    "The possible permutations for a DPP operation",
+    [  
+      I32EnumAttrCase<"quad_perm",  0>,
+      I32EnumAttrCase<"row_shl",    1>,
+      I32EnumAttrCase<"row_shr",    2>,
+      I32EnumAttrCase<"row_ror",    3>,
+      I32EnumAttrCase<"wave_shl",   4>,
+      I32EnumAttrCase<"wave_shr",   5>,
+      I32EnumAttrCase<"wave_ror",   6>,
+      I32EnumAttrCase<"wave_rol",   7>,
+      I32EnumAttrCase<"row_mirror", 8>,
+      I32EnumAttrCase<"row_half_mirror", 9>,
+      I32EnumAttrCase<"row_bcast_15", 10>,
+      I32EnumAttrCase<"row_bcast_31", 11>
+    ]> {
+  let genSpecializedAttr = 0;  
+  let cppNamespace = "::mlir::amdgpu";
+}
+
+def AMDGPU_DPPPermAttr : EnumAttr<AMDGPU_Dialect, AMDGPU_DPPPerm,
+  "dpp_perm">;
+
+def AMDGPU_DPPOp : AMDGPU_Op<"dpp", [SameTypeOperands, AllTypesMatch<["result", "old", "src"]>]>,
+  Arguments<(ins AnyType:$old,
+                  AnyType:$src,
+                  AMDGPU_DPPPermAttr:$kind,
+                  OptionalAttr<AnyAttrOf<[I32Attr, ArrayAttr, UnitAttr]>>:$permArgument,
+                  DefaultValuedAttr<I32Attr, "0xf">:$row_mask,
+                  DefaultValuedAttr<I32Attr, "0xf">:$bank_mask,
+                  DefaultValuedAttr<BoolAttr, "false">:$bound_ctrl)> {
+  let summary = "AMDGPU DPP operation";
+  let description = [{
+    This operation represents DPP functionality in a GPU program.
+     DPP provides the following operations:
+    - Full crossbar in a group of four (`quad_perm`)
+    - Wavefront shift left by one lane (`wave_shl`)
+    - Wavefront shift right by one lane (`wave_shr`)
+    - Wavefront rotate right by one lane (`wave_ror`)
+    - Wavefront rotate left by one lane (`wave_rol`)
+    - Row shift left by 1–15 lanes (`row_shl`)
+    - Row shift right by 1–15 lanes (`row_shr`)
+    - Row rotate right by 1–15 lanes (`row_ror`)
+    - Reverse within a row (`row_mirror`)
+    - Reverse within a half-row (`row_half_mirror`)
+    - Broadcast the 15th lane of each row to the next row (`row_bcast`)
+    - Broadcast lane 31 to rows 2 and 3 (`row_bcast`)
+  }];
+  let results = (outs AnyType:$result);
+  let assemblyFormat = [{
+    $old $src $kind (`(` $permArgument^ `)`)? attr-dict `:` type($result)
+  }];
+  let hasVerifier = 1;
+}
+
 def AMDGPU_LDSBarrierOp : AMDGPU_Op<"lds_barrier"> {
   let summary = "Barrier that includes a wait for LDS memory operations.";
   let description = [{
diff --git a/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td b/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td
index b81b7e577b213a0..a1e6fc3e299009c 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td
@@ -555,6 +555,30 @@ def ROCDL_RawBufferAtomicUMinOp :
   let hasCustomAssemblyFormat = 1;
 }
 
+// DPP Update intrinsic
+def ROCDL_DPPUpdateOp : ROCDL_IntrOp<"update.dpp", [], [0],
+    [AllTypesMatch<["res", "src", "old"]>], 1>,
+  Arguments<(ins LLVM_Type:$old, LLVM_Type:$src, I32Attr:$dppCtrl, I32Attr:$rowMask,
+      I32Attr:$bankMask, I1Attr:$boundCtrl)> {
+  let results = (outs LLVM_Type:$res);
+  let assemblyFormat = [{
+    attr-dict $old `,` $src `with` $dppCtrl `,` $rowMask `,` $bankMask `,` $boundCtrl `:` type($src)
+  }];
+  string llvmBuilder = [{
+      auto vdataType = moduleTranslation.convertType(op.getSrc().getType());
+      llvm::Value *args[] = {
+        moduleTranslation.lookupValue(op.getOld()),
+        moduleTranslation.lookupValue(op.getSrc()),
+          builder.getInt32(op.getDppCtrl()),
+          builder.getInt32(op.getRowMask()),
+          builder.getInt32(op.getBankMask()),
+          builder.getInt1(op.getBoundCtrl())
+      };
+      $res = createIntrinsicCall(builder,
+        llvm::Intrinsic::amdgcn_update_dpp, args, {vdataType});
+  }];
+}
+
 //===---------------------------------------------------------------------===//
 // 8-bit float intrinsics
 //===---------------------------------------------------------------------===//
diff --git a/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp b/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
index b808738804030f8..809e9448e80abfd 100644
--- a/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
+++ b/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
@@ -844,6 +844,155 @@ LogicalResult PackedStochRoundFp8OpLowering::matchAndRewrite(
   return success();
 }
 
+// Implement the AMDGPU_DPPLowering class that will convert the amdgpu.dpp
+// operation into the corresponding ROCDL instructions.
+struct AMDGPUDPPLowering : public ConvertOpToLLVMPattern<DPPOp> {
+  AMDGPUDPPLowering(LLVMTypeConverter &converter, Chipset chipset)
+      : ConvertOpToLLVMPattern<DPPOp>(converter), chipset(chipset) {}
+  Chipset chipset;
+
+  LogicalResult
+  matchAndRewrite(DPPOp DppOp, DPPOp::Adaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override {
+
+    // Convert the source operand to the corresponding LLVM type
+    Location loc = DppOp.getLoc();
+    Value src = adaptor.getSrc();
+    Value old = adaptor.getOld();
+    Type srcType = src.getType();
+    Type oldType = old.getType();
+    Type llvmType = nullptr;
+    if (srcType.getIntOrFloatBitWidth() < 32) {
+      llvmType = rewriter.getI32Type();
+    } else if (isa<FloatType>(srcType)) {
+      llvmType = (srcType.getIntOrFloatBitWidth() == 32)
+                     ? rewriter.getF32Type()
+                     : rewriter.getF64Type();
+    } else if (isa<IntegerType>(srcType)) {
+      llvmType = (srcType.getIntOrFloatBitWidth() == 32)
+                     ? rewriter.getI32Type()
+                     : rewriter.getI64Type();
+    }
+    auto llvmSrcIntType = typeConverter->convertType(
+        rewriter.getIntegerType(srcType.getIntOrFloatBitWidth()));
+
+    // If the source type is less of 32, use bitcast to convert it to i32.
+    auto convertOperand = [&](Value operand, Type operandType) {
+      if (operandType.getIntOrFloatBitWidth() <= 16) {
+        if (llvm::isa<FloatType>(operandType)) {
+          operand =
+              rewriter.create<LLVM::BitcastOp>(loc, llvmSrcIntType, operand);
+        }
+        auto llvmVecType = typeConverter->convertType(mlir::VectorType::get(
+            32 / operandType.getIntOrFloatBitWidth(), llvmSrcIntType));
+        Value undefVec = rewriter.create<LLVM::UndefOp>(loc, llvmVecType);
+        operand = rewriter.create<LLVM::InsertElementOp>(
+            loc, undefVec, operand, createI32Constant(rewriter, loc, 0));
+        operand = rewriter.create<LLVM::BitcastOp>(loc, llvmType, operand);
+      }
+      return operand;
+    };
+
+    src = convertOperand(src, srcType);
+    old = convertOperand(old, oldType);
+
+    // This is taken from the following file llvm/lib/Target/AMDGPU/SIDefines.h
+    enum DppCtrl : unsigned {
+      ROW_SHL0 = 0x100,
+      ROW_SHR0 = 0x110,
+      ROW_ROR0 = 0x120,
+      WAVE_SHL1 = 0x130,
+      WAVE_ROL1 = 0x134,
+      WAVE_SHR1 = 0x138,
+      WAVE_ROR1 = 0x13C,
+      ROW_MIRROR = 0x140,
+      ROW_HALF_MIRROR = 0x141,
+      BCAST15 = 0x142,
+      BCAST31 = 0x143,
+    };
+
+    auto kind = DppOp.getKind();
+    auto permArgument = DppOp.getPermArgument();
+    uint32_t DppCtrl = 0;
+
+    switch (kind) {
+
+    case DPPPerm::quad_perm:
+      if (auto quadPermAttr = cast<ArrayAttr>(*permArgument)) {
+        int32_t i = 0;
+        for (auto elem : quadPermAttr.getAsRange<IntegerAttr>()) {
+          uint32_t num = elem.getInt();
+          DppCtrl |= num << (i * 2);
+          i++;
+        }
+      }
+      break;
+    case DPPPerm::row_shl:
+      if (auto intAttr = cast<IntegerAttr>(*permArgument)) {
+        DppCtrl = intAttr.getInt() + DppCtrl::ROW_SHL0;
+      }
+      break;
+    case DPPPerm::row_shr:
+      if (auto intAttr = cast<IntegerAttr>(*permArgument)) {
+        DppCtrl = intAttr.getInt() + DppCtrl::ROW_SHR0;
+      }
+      break;
+    case DPPPerm::row_ror:
+      if (auto intAttr = cast<IntegerAttr>(*permArgument)) {
+        DppCtrl = intAttr.getInt() + DppCtrl::ROW_ROR0;
+      }
+      break;
+    case DPPPerm::wave_shl:
+      DppCtrl = DppCtrl::WAVE_SHL1;
+      break;
+    case DPPPerm::wave_shr:
+      DppCtrl = DppCtrl::WAVE_SHR1;
+      break;
+    case DPPPerm::wave_rol:
+      DppCtrl = DppCtrl::WAVE_ROL1;
+      break;
+    case DPPPerm::wave_ror:
+      DppCtrl = DppCtrl::WAVE_ROR1;
+      break;
+    case DPPPerm::row_mirror:
+      DppCtrl = DppCtrl::ROW_MIRROR;
+      break;
+    case DPPPerm::row_half_mirror:
+      DppCtrl = DppCtrl::ROW_HALF_MIRROR;
+      break;
+    case DPPPerm::row_bcast_15:
+      DppCtrl = DppCtrl::BCAST15;
+      break;
+    case DPPPerm::row_bcast_31:
+      DppCtrl = DppCtrl::BCAST31;
+      break;
+    }
+
+    // Check for row_mask, bank_mask, bound_ctrl if they exist and create
+    // constants
+    auto rowMask = DppOp->getAttrOfType<IntegerAttr>("row_mask").getInt();
+    auto bankMask = DppOp->getAttrOfType<IntegerAttr>("bank_mask").getInt();
+    bool boundCtrl = DppOp->getAttrOfType<BoolAttr>("bound_ctrl").getValue();
+
+    // create a ROCDL_DPPMovOp instruction with the appropriate attributes
+    auto dppMovOp = rewriter.create<ROCDL::DPPUpdateOp>(
+        loc, llvmType, old, src, DppCtrl, rowMask, bankMask, boundCtrl);
+
+    Value result = dppMovOp.getRes();
+    if (srcType.getIntOrFloatBitWidth() < 32) {
+      result = rewriter.create<LLVM::TruncOp>(loc, llvmSrcIntType, result);
+      if (!llvm::isa<IntegerType>(srcType)) {
+        result = rewriter.create<LLVM::BitcastOp>(loc, srcType, result);
+      }
+    }
+
+    // We are replacing the AMDGPU_DPPOp instruction with the new
+    // ROCDL_DPPMovOp instruction
+    rewriter.replaceOp(DppOp, ValueRange(result));
+    return success();
+  }
+};
+
 struct ConvertAMDGPUToROCDLPass
     : public impl::ConvertAMDGPUToROCDLBase<ConvertAMDGPUToROCDLPass> {
   ConvertAMDGPUToROCDLPass() = default;
@@ -895,9 +1044,10 @@ void mlir::populateAMDGPUToROCDLConversionPatterns(LLVMTypeConverter &converter,
                                ROCDL::RawPtrBufferAtomicUminOp>,
            RawBufferOpLowering<RawBufferAtomicCmpswapOp,
                                ROCDL::RawPtrBufferAtomicCmpSwap>,
-           LDSBarrierOpLowering, SchedBarrierOpLowering, MFMAOpLowering,
-           WMMAOpLowering, ExtPackedFp8OpLowering, PackedTrunc2xFp8OpLowering,
-           PackedStochRoundFp8OpLowering>(converter, chipset);
+           AMDGPUDPPLowering, LDSBarrierOpLowering, SchedBarrierOpLowering,
+           MFMAOpLowering, WMMAOpLowering, ExtPackedFp8OpLowering,
+           PackedTrunc2xFp8OpLowering, PackedStochRoundFp8OpLowering>(converter,
+                                                                      chipset);
 }
 
 std::unique_ptr<Pass> mlir::createConvertAMDGPUToROCDLPass() {
diff --git a/mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp b/mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp
index e3beceaa3bbb5bf..c1a785fb25478da 100644
--- a/mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp
+++ b/mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp
@@ -326,6 +326,68 @@ LogicalResult MFMAOp::verify() {
   return success();
 }
 
+//===----------------------------------------------------------------------===//
+// DPPOp
+//===----------------------------------------------------------------------===//
+LogicalResult DPPOp::verify() {
+  Type srcType = getSrc().getType();
+  if (srcType.getIntOrFloatBitWidth() > 64) {
+    return emitOpError("integer and floating point types larger than 64 bits "
+                       "are not supported");
+  }
+
+  DPPPerm kind = getKind();
+  Attribute permArgument = getPermArgument().value_or(Attribute{});
+
+  switch (kind) {
+
+  case DPPPerm::quad_perm: {
+    auto quadPermAttr = dyn_cast_or_null<ArrayAttr>(permArgument);
+    if (!quadPermAttr || quadPermAttr.size() != 4) {
+      return emitOpError("quad_perm attribute must have exactly 4 elements");
+    }
+    for (auto elem : quadPermAttr.getAsRange<IntegerAttr>()) {
+      uint32_t num = elem.getInt();
+      if (num < 0 || num > 3) {
+        return emitOpError(
+            "Each element of quad_perm must be in the range [0, 3]");
+      }
+    }
+  } break;
+
+  case DPPPerm::row_shl:
+  case DPPPerm::row_shr:
+  case DPPPerm::row_ror: {
+    if (!permArgument) {
+      return emitOpError("Attribute '" + Twine(stringifyDPPPerm(kind)) +
+                         "' value not specified");
+    }
+    if (auto intAttr = dyn_cast<IntegerAttr>(permArgument)) {
+      uint32_t attrValue = intAttr.getInt();
+      if (attrValue < 1 || attrValue > 15) {
+        return emitOpError("Attribute value must be between 1 and 15");
+      }
+    }
+  } break;
+
+  case DPPPerm::wave_shl:
+  case DPPPerm::wave_shr:
+  case DPPPerm::wave_rol:
+  case DPPPerm::wave_ror:
+  case DPPPerm::row_mirror:
+  case DPPPerm::row_half_mirror:
+  case DPPPerm::row_bcast_15:
+  case DPPPerm::row_bcast_31: {
+    if (permArgument && !isa<UnitAttr>(permArgument)) {
+      return emitOpError("Expected unit attribute for permArgument, but found "
+                         "non-trivial argument");
+    }
+    break;
+  }
+  }
+  return success();
+}
+
 #include "mlir/Dialect/AMDGPU/IR/AMDGPUEnums.cpp.inc"
 
 #define GET_ATTRDEF_CLASSES
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/dpp.mlir b/mlir/test/Conversion/AMDGPUToROCDL/dpp.mlir
new file mode 100644
index 000000000000000..14691e73e62d78c
--- /dev/null
+++ b/mlir/test/Conversion/AMDGPUToROCDL/dpp.mlir
@@ -0,0 +1,139 @@
+// RUN: mlir-opt -convert-amdgpu-to-rocdl=chipset=gfx908 %s | FileCheck %s
+// RUN: mlir-opt -convert-amdgpu-to-rocdl=chipset=gfx90a %s | FileCheck %s
+// RUN: mlir-opt -convert-amdgpu-to-rocdl=chipset=gfx942 %s | FileCheck %s
+
+func.func @test_dpp(%arg0: i32, %arg1: i32) -> i32 {
+  // CHECK-LABEL: func @test_dpp
+  // CHECK: rocdl.update.dpp %arg0, %arg1 with 257, 10, 15, false : i32
+  // CHECK: return %0 : i32
+  %0 = amdgpu.dpp %arg0 %arg1 row_shl ( 0x1 : i32 ) { row_mask = 0xa : i32, bound_ctrl = false } : i32
+    return %0 : i32
+}
+
+func.func @quad_dpp(%arg0: i32, %arg1: i32) -> i32 {
+  // CHECK-LABEL: func @quad_dpp
+  // CHECK: rocdl.update.dpp %arg0, %arg1 with 145, 1, 1, true : i32
+  // CHECK: return %0 : i32
+  %0 = amdgpu.dpp %arg0 %arg1 quad_perm ( [1,0,1,2] ) { row_mask = 0x1 : i32, bank_mask = 0x1 : i32, bound_ctrl = true } : i32
+    return %0 : i32
+}
+
+func.func @wave_shr_dpp(%arg0: i32, %arg1: i32) -> i32 {
+  // CHECK-LABEL: func @wave_shr_dpp
+  // CHECK: rocdl.update.dpp %arg0, %arg1 with 312, 10, 1, true : i32
+  // CHECK: return %0 : i32
+  %0 = amdgpu.dpp %arg0 %arg1 wave_shr { row_mask = 0xa : i32, bank_mask = 0x1 : i32, bound_ctrl = true } : i32
+    return %0 : i32
+}
+
+func.func @row_half_mirror_update_dpp(%arg0: i32, %arg1: i32) -> i32 {
+  // CHECK-LABEL: func @row_half_mirror_update_dpp
+  // CHECK: rocdl.update.dpp %arg0, %arg1 with 321, 15, 1, false : i32
+  // CHECK: return %0 : i32
+%0 = amdgpu.dpp %arg0 %arg1 row_half_mirror { bank_mask = 0x1 : i32 } : i32
+    return %0 : i32
+}
+
+func.func @wave_rol_update_dpp(%arg0: i32, %arg1: i32) -> i32 {
+  // CHECK-LABEL: func @wave_rol_update_dpp
+  // CHECK: rocdl.update.dpp %arg0, %arg1 with 308, 10, 1, false : i32
+  // CHECK: return %0 : i32
+  %0 = amdgpu.dpp %arg0 %arg1 wave_rol { row_mask = 0xa : i32, bank_mask = 0x1 : i32 } : i32
+    return %0 : i32
+}
+
+func.func @row_bcast_dpp_f32(%arg0: f32, %arg1: f32) -> f32 {
+  // CHECK-LABEL: func @row_bcast_dpp_f32
+  // CHECK: rocdl.update.dpp %arg0, %arg1 with 322, 15, 15, true : f32
+  // CHECK: return %0 : f32
+  %0 = amdgpu.dpp %arg0 %arg1 row_bcast_15 { bound_ctrl = true } : f32
+    return %0 : f32
+}
+
+func.func @test_dpp_f32(%arg0: f32, %arg1: f32) -> f32 {
+  // CHECK-LABEL: func @test_dpp_f32
+  // CHECK: rocdl.update.dpp %arg0, %arg1 with 320, 1, 4, true : f32
+  // CHECK: return %0 : f32
+  %0 = amdgpu.dpp %arg0 %arg1 row_mirror { row_mask = 0x1 : i32, bank_mask = 0x4 : i32, bound_ctrl = true } : f32
+    return %0 : f32
+}
+
+func.func @quad_perm_update_dpp_f32(%arg0: f32, %arg1: f32) -> f32 {
+  // CHECK-LABEL: func @quad_perm_update_dpp_f32
+  // CHECK: rocdl.update.dpp %arg0, %arg1 with  88, 15, 1, false : f32
+  // CHECK: return %0 : f32
+  %0 = amdgpu.dpp %arg0 %arg1 quad_perm ( [0,2,1,1] ) { bank_mask = 0x1 : i32 } : f32
+    return %0 : f32
+}
+
+func.func @quad_perm_dpp(%arg0: i64, %arg1: i64) -> i64 {
+  // CHECK-LABEL: func @quad_perm_dpp
+  // CHECK: rocdl.update.dpp %arg0, %arg1 with 88, 15, 15, false : i64
+  // CHECK: return %0 : i64
+  %0 = amdgpu.dpp %arg0 %arg1 quad_perm ( [0,2,1,1] ) : i64
+    return %0 : i64
+}
+
+func.func @row_bcast_dpp(%arg0: f64, %arg1: f64) -> f64 {
+  // CHECK-LABEL: func @row_bcast_dpp
+  // CHECK: rocdl.update.dpp %arg0, %arg1 with 323, 4, 1, false : f64
+  // CHECK: return %0 : f64
+  %0 = amdgpu.dpp %arg0 %arg1 row_bcast_31 { row_mask = 0x4 : i32, bank_mask = 0x1 : i32} : f64
+    return %0 : f64
+}
+
+func.func @test_dpp_f16(%arg0: f16, %arg1: f16) -> f16 {
+  // CHECK-LABEL:  func @test_dpp_f16
+  // CHECK: llvm.bitcast %arg1 : f16 to i16
+  // CHECK: llvm.mlir.undef : vector<2xi16>
+  // CHECK: llvm.mlir.constant(0 : i32) : i32
+  // CHECK: llvm.insertelement %0, %1[%2 : i32] : vector<2xi16>
+  // CHECK: llvm.bitcast %3 : vector<2xi16> to i32
+  // CHECK: llvm.bitcast %arg0 : f16 to i16
+  // CHECK: llvm.mlir.undef : vector<2xi16>
+  // CHECK: llvm.mlir.constant(0 : i32) : i32
+  // CHECK: llvm.insertelement %5, %6[%7 : i32] : vector<2xi16>
+  // CHECK: llvm.bitcast %8 : vector<2xi16> to i32
+  // CHECK: rocdl.update.dpp %9, %4 with 273, 15, 3, false : i32
+  // CHECK: llvm.trunc %10 : i32 to i16
+  // CHECK: llvm.bitcast %11 : i16 to f16
+  // CHECK: return %12 : f16
+  %0 = amdgpu.dpp %arg0 %arg1 row_shr ( 0x1 : i32 ){ bank_mask = 0x3 : i32 } : f16
+    return %0 : f16
+}
+
+func.func @row_shl_dpp_i16(%arg0: i16, %arg1: i16) -> i16 {
+  // CHECK-LABEL: func @row_shl_dpp_i16
+  // CHECK: llvm.mlir.undef : vector<2xi16>
+  // CHECK: llvm.mlir.constant(0 : i32) : i32
+  // CHECK: llvm.insertelement %arg1, %0[%1 : i32] : vector<2xi16>
+  // CHECK: llvm.bitcast %2 : vector<2xi16> to i32
+  // CHECK: llvm.mlir.undef : vector<2xi16>
+  // CHECK: llvm.mlir.constant(0 : i32) : i32
+  // CHECK: llvm.insertelement %arg0, %4[%5 : i32] : vector<2xi16>
+  // CHECK: llvm.bitcast %6 : vector<2xi16> to i32
+  // CHECK: rocdl.update.dpp %7, %3 with 298, 10, 1, false : i32
+  // CHECK: llvm.trunc %8 : i32 to i16
+  // CHECK: return %9 : i16
+  %0 = amdgpu.dpp %arg0 %arg1 row_ror ( 0xa : i32 ) { row_mask = 0xa : i32, bank_mask = 0x1 : i32 } : i16
+    return %0 : i16
+}
+
+func.func @row_bcast_update_dpp_f16(%arg0: f16, %arg1: f16) -> f16 {
+  // CHECK-LABEL: func @row_bcast_update_dpp_f16
+  // CHECK: llvm.bitcast %arg1 : f16 to i16
+  // CHECK: llvm.mlir.undef : vector<2xi16>
+  // CHECK: llvm.mlir.constant(0 : i32) : i32
+  // CHECK: llvm.insertelement %0, %1[%2 : i32] : vector<2xi16>
+  // CHECK: llvm.bitcast %arg0 : f16 to i16
+  // CHECK: llvm.mlir.undef : vector<2xi16>
+  // CHECK: llvm.mlir.constant(0 : i32) : i32
+  // CHECK  llvm.insertelement %5, %6[%7 : i32] : vector<2xi16>
+  // CHECK: llvm.bitcast %8 : vector<2xi16> to i32
+  // CHECK: rocdl.update.dpp %9, %4 with 322, 15, 15, true : i32
+  // CHECK: llvm.trunc %10 : i32 to i16
+  // CHECK: llvm.bitcast %11 : i16 to f16
+  // CHECK: return %12 : f16
+  %0 = amdgpu.dpp %arg0 %arg1 row_bcast_15 { bound_ctrl = true } : f16
+    return %0 : f16
+}

From 7afb51e035709e7f2532452054a39fe968444504 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Fri, 16 Aug 2024 09:21:11 -0700
Subject: [PATCH 196/441] [SelectionDAG][X86] Add
 SelectionDAG::getSignedConstant and use it in a few places. (#104555)

PR #80309 proposes to have users of APInt's uint64_t
constructor opt-in to implicit truncation. Currently, that patch
requires SelectionDAG::getConstant to opt-in.

This patch adds getSignedConstant so we can start fixing some of the
cases that require implicit truncation.
---
 llvm/include/llvm/CodeGen/SelectionDAG.h      |  3 +++
 llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp |  2 +-
 .../lib/CodeGen/SelectionDAG/SelectionDAG.cpp |  9 +++++++++
 .../SelectionDAG/SelectionDAGBuilder.cpp      |  2 +-
 .../CodeGen/SelectionDAG/TargetLowering.cpp   |  4 ++--
 llvm/lib/Target/X86/X86ISelDAGToDAG.cpp       | 10 ++++++----
 llvm/lib/Target/X86/X86ISelLowering.cpp       | 19 ++++++++++---------
 7 files changed, 32 insertions(+), 17 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h
index c762f09d55f1617..ade282bd7e2e299 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAG.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAG.h
@@ -672,6 +672,9 @@ class SelectionDAG {
   SDValue getConstant(const APInt &Val, const SDLoc &DL, EVT VT,
                       bool isTarget = false, bool isOpaque = false);
 
+  SDValue getSignedConstant(int64_t Val, const SDLoc &DL, EVT VT,
+                            bool isTarget = false, bool isOpaque = false);
+
   SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget = false,
                              bool IsOpaque = false);
 
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index e7f765382b0e469..debecc40ebb823c 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -2601,7 +2601,7 @@ SDValue SelectionDAGLegalize::expandFrexp(SDNode *Node) const {
   SDValue IsDenormal =
       DAG.getSetCC(dl, SetCCVT, Abs, SmallestNormalizedInt, ISD::SETULT);
 
-  SDValue MinExp = DAG.getConstant(MinExpVal, dl, ExpVT);
+  SDValue MinExp = DAG.getSignedConstant(MinExpVal, dl, ExpVT);
   SDValue Zero = DAG.getConstant(0, dl, ExpVT);
 
   SDValue ScaledAsInt = DAG.getNode(ISD::BITCAST, dl, AsIntVT, ScaleUp);
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 7bf90ceb93cb4e3..2401bb5a347e2d0 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -1747,6 +1747,15 @@ SDValue SelectionDAG::getConstant(const ConstantInt &Val, const SDLoc &DL,
   return Result;
 }
 
+SDValue SelectionDAG::getSignedConstant(int64_t Val, const SDLoc &DL, EVT VT,
+                                        bool isT, bool isO) {
+  unsigned Size = VT.getScalarSizeInBits();
+  assert(
+      isIntN(Size, Val) &&
+      "getSignedConstant with a int64_t value that doesn't fit in the type!");
+  return getConstant(APInt(Size, Val, true), DL, VT, isT, isO);
+}
+
 SDValue SelectionDAG::getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget,
                                          bool IsOpaque) {
   return getConstant(APInt::getAllOnes(VT.getScalarSizeInBits()), DL, VT,
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 7cdd3d47b641d7b..8e24f5bd75ec86e 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -4474,7 +4474,7 @@ void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
 
   // Mask out the low bits for alignment purposes.
   AllocSize = DAG.getNode(ISD::AND, dl, AllocSize.getValueType(), AllocSize,
-                          DAG.getConstant(~StackAlignMask, dl, IntPtr));
+                          DAG.getSignedConstant(~StackAlignMask, dl, IntPtr));
 
   SDValue Ops[] = {
       getRoot(), AllocSize,
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index c1a87de93e73594..3c499ca60078082 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -6349,9 +6349,9 @@ SDValue TargetLowering::BuildSDIV(SDNode *N, SelectionDAG &DAG,
     }
 
     MagicFactors.push_back(DAG.getConstant(magics.Magic, dl, SVT));
-    Factors.push_back(DAG.getConstant(NumeratorFactor, dl, SVT));
+    Factors.push_back(DAG.getSignedConstant(NumeratorFactor, dl, SVT));
     Shifts.push_back(DAG.getConstant(magics.ShiftAmount, dl, ShSVT));
-    ShiftMasks.push_back(DAG.getConstant(ShiftMask, dl, SVT));
+    ShiftMasks.push_back(DAG.getSignedConstant(ShiftMask, dl, SVT));
     return true;
   };
 
diff --git a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
index 74804e5c9783dec..d0a54ab8993c267 100644
--- a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
@@ -314,7 +314,8 @@ namespace {
         Disp = CurDAG->getTargetBlockAddress(AM.BlockAddr, MVT::i32, AM.Disp,
                                              AM.SymbolFlags);
       else
-        Disp = CurDAG->getTargetConstant(AM.Disp, DL, MVT::i32);
+        Disp =
+            CurDAG->getSignedConstant(AM.Disp, DL, MVT::i32, /*isTarget=*/true);
 
       if (AM.Segment.getNode())
         Segment = AM.Segment;
@@ -2130,7 +2131,7 @@ static bool foldMaskedShiftToScaledMask(SelectionDAG &DAG, SDValue N,
     X = NewX;
   }
 
-  SDValue NewMask = DAG.getConstant(Mask >> ShiftAmt, DL, VT);
+  SDValue NewMask = DAG.getSignedConstant(Mask >> ShiftAmt, DL, VT);
   SDValue NewAnd = DAG.getNode(ISD::AND, DL, VT, X, NewMask);
   SDValue NewShift = DAG.getNode(ISD::SHL, DL, VT, NewAnd, Shift.getOperand(1));
 
@@ -3733,7 +3734,8 @@ bool X86DAGToDAGISel::foldLoadStoreIntoMemOperand(SDNode *Node) {
       }
 
       if (MemVT != MVT::i64 || isInt<32>(OperandV)) {
-        Operand = CurDAG->getTargetConstant(OperandV, SDLoc(Node), MemVT);
+        Operand = CurDAG->getSignedConstant(OperandV, SDLoc(Node), MemVT,
+                                            /*isTarget=*/true);
         NewOpc = SelectImmOpcode(Opc);
       }
     }
@@ -4507,7 +4509,7 @@ bool X86DAGToDAGISel::tryShrinkShlLogicImm(SDNode *N) {
     X = NewX;
   }
 
-  SDValue NewCst = CurDAG->getConstant(ShiftedVal, dl, NVT);
+  SDValue NewCst = CurDAG->getSignedConstant(ShiftedVal, dl, NVT);
   insertDAGNode(*CurDAG, SDValue(N, 0), NewCst);
   SDValue NewBinOp = CurDAG->getNode(Opcode, dl, NVT, X, NewCst);
   insertDAGNode(*CurDAG, SDValue(N, 0), NewBinOp);
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 6385dab9c55e656..df6295a88e13498 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -10763,9 +10763,9 @@ static SDValue lowerShuffleAsBlend(const SDLoc &DL, MVT VT, SDValue V1,
     for (int i = 0, Size = Mask.size(); i < Size; ++i)
       for (int j = 0; j < Scale; ++j)
         VSELECTMask.push_back(
-            Mask[i] < 0 ? DAG.getUNDEF(MVT::i8)
-                        : DAG.getConstant(Mask[i] < Size ? -1 : 0, DL,
-                                          MVT::i8));
+            Mask[i] < 0
+                ? DAG.getUNDEF(MVT::i8)
+                : DAG.getSignedConstant(Mask[i] < Size ? -1 : 0, DL, MVT::i8));
 
     V1 = DAG.getBitcast(BlendVT, V1);
     V2 = DAG.getBitcast(BlendVT, V2);
@@ -18654,7 +18654,7 @@ SDValue X86TargetLowering::LowerGlobalOrExternal(SDValue Op, SelectionDAG &DAG,
   // addition for it.
   if (Offset != 0)
     Result = DAG.getNode(ISD::ADD, dl, PtrVT, Result,
-                         DAG.getConstant(Offset, dl, PtrVT));
+                         DAG.getSignedConstant(Offset, dl, PtrVT));
 
   return Result;
 }
@@ -24975,9 +24975,9 @@ X86TargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
       Result = DAG.getNode(ISD::SUB, dl, VT, SP, Size); // Value
     }
     if (Alignment && *Alignment > StackAlign)
-      Result =
-          DAG.getNode(ISD::AND, dl, VT, Result,
-                      DAG.getConstant(~(Alignment->value() - 1ULL), dl, VT));
+      Result = DAG.getNode(
+          ISD::AND, dl, VT, Result,
+          DAG.getSignedConstant(~(Alignment->value() - 1ULL), dl, VT));
     Chain = DAG.getCopyToReg(Chain, dl, SPReg, Result); // Output chain
   } else if (SplitStack) {
     MachineRegisterInfo &MRI = MF.getRegInfo();
@@ -25009,8 +25009,9 @@ X86TargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
     Chain = SP.getValue(1);
 
     if (Alignment) {
-      SP = DAG.getNode(ISD::AND, dl, VT, SP.getValue(0),
-                       DAG.getConstant(~(Alignment->value() - 1ULL), dl, VT));
+      SP = DAG.getNode(
+          ISD::AND, dl, VT, SP.getValue(0),
+          DAG.getSignedConstant(~(Alignment->value() - 1ULL), dl, VT));
       Chain = DAG.getCopyToReg(Chain, dl, SPReg, SP);
     }
 

From 97f0ab71c002e8a14142be3a117b3091ab75c552 Mon Sep 17 00:00:00 2001
From: Andrey Timonin <timonina1909@gmail.com>
Date: Fri, 16 Aug 2024 19:25:06 +0300
Subject: [PATCH 197/441] [mlir][emitc] Add 'emitc.switch' op to the dialect
 (#102331)

This PR is continuation of the [previous
one](https://github.com/llvm/llvm-project/pull/101478). As a result, the
`emitc::SwitchOp` op was developed inspired by `scf::IndexSwitchOp`.

Main points of PR:

- Added the `emitc::SwitchOp` op  to the EmitC dialect + CppEmitter
- Corresponding tests were added
- Conversion from the SCF dialect to the EmitC dialect for the op
---
 mlir/include/mlir/Dialect/EmitC/IR/EmitC.td   |  84 +-
 mlir/lib/Conversion/SCFToEmitC/SCFToEmitC.cpp |  70 +-
 mlir/lib/Dialect/EmitC/IR/EmitC.cpp           | 132 +++
 mlir/lib/Target/Cpp/TranslateToCpp.cpp        |  43 +-
 mlir/test/Conversion/SCFToEmitC/switch.mlir   | 117 +++
 mlir/test/Dialect/EmitC/invalid_ops.mlir      |  88 +-
 mlir/test/Dialect/EmitC/ops.mlir              |  20 +
 mlir/test/Target/Cpp/invalid.mlir             |   1 +
 mlir/test/Target/Cpp/switch.mlir              | 856 ++++++++++++++++++
 9 files changed, 1391 insertions(+), 20 deletions(-)
 create mode 100644 mlir/test/Conversion/SCFToEmitC/switch.mlir
 create mode 100644 mlir/test/Target/Cpp/switch.mlir

diff --git a/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td b/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td
index 16fa58ce47013d5..40903b4e288ca62 100644
--- a/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td
+++ b/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td
@@ -1188,7 +1188,7 @@ def EmitC_AssignOp : EmitC_Op<"assign", []> {
 }
 
 def EmitC_YieldOp : EmitC_Op<"yield",
-      [Pure, Terminator, ParentOneOf<["ExpressionOp", "IfOp", "ForOp"]>]> {
+      [Pure, Terminator, ParentOneOf<["ExpressionOp", "IfOp", "ForOp", "SwitchOp"]>]> {
   let summary = "Block termination operation";
   let description = [{
     The `emitc.yield` terminates its parent EmitC op's region, optionally yielding
@@ -1302,5 +1302,87 @@ def EmitC_SubscriptOp : EmitC_Op<"subscript", []> {
   let assemblyFormat = "$value `[` $indices `]` attr-dict `:` functional-type(operands, results)";
 }
 
+def EmitC_SwitchOp : EmitC_Op<"switch", [RecursiveMemoryEffects,
+    SingleBlockImplicitTerminator<"emitc::YieldOp">,
+    DeclareOpInterfaceMethods<RegionBranchOpInterface,
+                              ["getRegionInvocationBounds",
+                               "getEntrySuccessorRegions"]>]> {
+  let summary = "Switch operation";
+  let description = [{
+    The `emitc.switch` is a control-flow operation that branches to one of
+    the given regions based on the values of the argument and the cases.
+    The operand to a switch operation is a opaque, integral or pointer
+    wide types.
+
+    The operation always has a "default" region and any number of case regions
+    denoted by integer constants. Control-flow transfers to the case region
+    whose constant value equals the value of the argument. If the argument does
+    not equal any of the case values, control-flow transfer to the "default"
+    region.
+
+    The operation does not return any value. Moreover, case regions must be
+    explicitly terminated using the `emitc.yield` operation. Default region is
+    yielded implicitly.
+
+    Example:
+
+    ```mlir
+    // Example:
+    emitc.switch %0 : i32
+    case 2 {
+      %1 = emitc.call_opaque "func_b" () : () -> i32
+      emitc.yield
+    }
+    case 5 {
+      %2 = emitc.call_opaque "func_a" () : () -> i32
+      emitc.yield
+    }
+    default {
+      %3 = "emitc.variable"(){value = 42.0 : f32} : () -> f32
+      emitc.call_opaque "func2" (%3) : (f32) -> ()
+    }
+    ```
+    ```c++
+    // Code emitted for the operations above.
+    switch (v1) {
+    case 2: {
+      int32_t v2 = func_b();
+      break;
+    }
+    case 5: {
+      int32_t v3 = func_a();
+      break;
+    }
+    default: {
+      float v4 = 4.200000000e+01f;
+      func2(v4);
+      break;
+    }
+    ```
+  }];
+
+  let arguments = (ins IntegerIndexOrOpaqueType:$arg, DenseI64ArrayAttr:$cases);
+  let results = (outs);
+  let regions = (region SizedRegion<1>:$defaultRegion,
+                        VariadicRegion<SizedRegion<1>>:$caseRegions);
+
+  let assemblyFormat = [{
+    $arg `:` type($arg) attr-dict custom<SwitchCases>($cases, $caseRegions) `\n`
+    `` `default` $defaultRegion
+  }];
+
+  let extraClassDeclaration = [{
+    /// Get the number of cases.
+    unsigned getNumCases();
+
+    /// Get the default region body.
+    Block &getDefaultBlock();
+
+    /// Get the body of a case region.
+    Block &getCaseBlock(unsigned idx);
+  }];
+
+  let hasVerifier = 1;
+}
 
 #endif // MLIR_DIALECT_EMITC_IR_EMITC
diff --git a/mlir/lib/Conversion/SCFToEmitC/SCFToEmitC.cpp b/mlir/lib/Conversion/SCFToEmitC/SCFToEmitC.cpp
index 0a8924222525546..ede811c6e1bb1e7 100644
--- a/mlir/lib/Conversion/SCFToEmitC/SCFToEmitC.cpp
+++ b/mlir/lib/Conversion/SCFToEmitC/SCFToEmitC.cpp
@@ -94,6 +94,19 @@ static void lowerYield(SmallVector<Value> &resultVariables,
   rewriter.eraseOp(yield);
 }
 
+// Lower the contents of an scf::if/scf::index_switch regions to an
+// emitc::if/emitc::switch region. The contents of the lowering region is
+// moved into the respective lowered region, but the scf::yield is replaced not
+// only with an emitc::yield, but also with a sequence of emitc::assign ops that
+// set the yielded values into the result variables.
+static void lowerRegion(SmallVector<Value> &resultVariables,
+                        PatternRewriter &rewriter, Region &region,
+                        Region &loweredRegion) {
+  rewriter.inlineRegionBefore(region, loweredRegion, loweredRegion.end());
+  Operation *terminator = loweredRegion.back().getTerminator();
+  lowerYield(resultVariables, rewriter, cast<scf::YieldOp>(terminator));
+}
+
 LogicalResult ForLowering::matchAndRewrite(ForOp forOp,
                                            PatternRewriter &rewriter) const {
   Location loc = forOp.getLoc();
@@ -145,18 +158,6 @@ LogicalResult IfLowering::matchAndRewrite(IfOp ifOp,
   SmallVector<Value> resultVariables =
       createVariablesForResults(ifOp, rewriter);
 
-  // Utility function to lower the contents of an scf::if region to an emitc::if
-  // region. The contents of the scf::if regions is moved into the respective
-  // emitc::if regions, but the scf::yield is replaced not only with an
-  // emitc::yield, but also with a sequence of emitc::assign ops that set the
-  // yielded values into the result variables.
-  auto lowerRegion = [&resultVariables, &rewriter](Region &region,
-                                                   Region &loweredRegion) {
-    rewriter.inlineRegionBefore(region, loweredRegion, loweredRegion.end());
-    Operation *terminator = loweredRegion.back().getTerminator();
-    lowerYield(resultVariables, rewriter, cast<scf::YieldOp>(terminator));
-  };
-
   Region &thenRegion = ifOp.getThenRegion();
   Region &elseRegion = ifOp.getElseRegion();
 
@@ -166,20 +167,59 @@ LogicalResult IfLowering::matchAndRewrite(IfOp ifOp,
       rewriter.create<emitc::IfOp>(loc, ifOp.getCondition(), false, false);
 
   Region &loweredThenRegion = loweredIf.getThenRegion();
-  lowerRegion(thenRegion, loweredThenRegion);
+  lowerRegion(resultVariables, rewriter, thenRegion, loweredThenRegion);
 
   if (hasElseBlock) {
     Region &loweredElseRegion = loweredIf.getElseRegion();
-    lowerRegion(elseRegion, loweredElseRegion);
+    lowerRegion(resultVariables, rewriter, elseRegion, loweredElseRegion);
   }
 
   rewriter.replaceOp(ifOp, resultVariables);
   return success();
 }
 
+// Lower scf::index_switch to emitc::switch, implementing result values as
+// emitc::variable's updated within the case and default regions.
+struct IndexSwitchOpLowering : public OpRewritePattern<IndexSwitchOp> {
+  using OpRewritePattern<IndexSwitchOp>::OpRewritePattern;
+
+  LogicalResult matchAndRewrite(IndexSwitchOp indexSwitchOp,
+                                PatternRewriter &rewriter) const override;
+};
+
+LogicalResult
+IndexSwitchOpLowering::matchAndRewrite(IndexSwitchOp indexSwitchOp,
+                                       PatternRewriter &rewriter) const {
+  Location loc = indexSwitchOp.getLoc();
+
+  // Create an emitc::variable op for each result. These variables will be
+  // assigned to by emitc::assign ops within the case and default regions.
+  SmallVector<Value> resultVariables =
+      createVariablesForResults(indexSwitchOp, rewriter);
+
+  auto loweredSwitch = rewriter.create<emitc::SwitchOp>(
+      loc, indexSwitchOp.getArg(), indexSwitchOp.getCases(),
+      indexSwitchOp.getNumCases());
+
+  // Lowering all case regions.
+  for (auto pair : llvm::zip(indexSwitchOp.getCaseRegions(),
+                             loweredSwitch.getCaseRegions())) {
+    lowerRegion(resultVariables, rewriter, std::get<0>(pair),
+                std::get<1>(pair));
+  }
+
+  // Lowering default region.
+  lowerRegion(resultVariables, rewriter, indexSwitchOp.getDefaultRegion(),
+              loweredSwitch.getDefaultRegion());
+
+  rewriter.replaceOp(indexSwitchOp, resultVariables);
+  return success();
+}
+
 void mlir::populateSCFToEmitCConversionPatterns(RewritePatternSet &patterns) {
   patterns.add<ForLowering>(patterns.getContext());
   patterns.add<IfLowering>(patterns.getContext());
+  patterns.add<IndexSwitchOpLowering>(patterns.getContext());
 }
 
 void SCFToEmitCPass::runOnOperation() {
@@ -188,7 +228,7 @@ void SCFToEmitCPass::runOnOperation() {
 
   // Configure conversion to lower out SCF operations.
   ConversionTarget target(getContext());
-  target.addIllegalOp<scf::ForOp, scf::IfOp>();
+  target.addIllegalOp<scf::ForOp, scf::IfOp, scf::IndexSwitchOp>();
   target.markUnknownOpDynamicallyLegal([](Operation *) { return true; });
   if (failed(
           applyPartialConversion(getOperation(), target, std::move(patterns))))
diff --git a/mlir/lib/Dialect/EmitC/IR/EmitC.cpp b/mlir/lib/Dialect/EmitC/IR/EmitC.cpp
index 70e3e728e019506..d731a6756ff6308 100644
--- a/mlir/lib/Dialect/EmitC/IR/EmitC.cpp
+++ b/mlir/lib/Dialect/EmitC/IR/EmitC.cpp
@@ -1096,6 +1096,138 @@ GetGlobalOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
   return success();
 }
 
+//===----------------------------------------------------------------------===//
+// SwitchOp
+//===----------------------------------------------------------------------===//
+
+/// Parse the case regions and values.
+static ParseResult
+parseSwitchCases(OpAsmParser &parser, DenseI64ArrayAttr &cases,
+                 SmallVectorImpl<std::unique_ptr<Region>> &caseRegions) {
+  SmallVector<int64_t> caseValues;
+  while (succeeded(parser.parseOptionalKeyword("case"))) {
+    int64_t value;
+    Region &region = *caseRegions.emplace_back(std::make_unique<Region>());
+    if (parser.parseInteger(value) ||
+        parser.parseRegion(region, /*arguments=*/{}))
+      return failure();
+    caseValues.push_back(value);
+  }
+  cases = parser.getBuilder().getDenseI64ArrayAttr(caseValues);
+  return success();
+}
+
+/// Print the case regions and values.
+static void printSwitchCases(OpAsmPrinter &p, Operation *op,
+                             DenseI64ArrayAttr cases, RegionRange caseRegions) {
+  for (auto [value, region] : llvm::zip(cases.asArrayRef(), caseRegions)) {
+    p.printNewline();
+    p << "case " << value << ' ';
+    p.printRegion(*region, /*printEntryBlockArgs=*/false);
+  }
+}
+
+static LogicalResult verifyRegion(emitc::SwitchOp op, Region &region,
+                                  const Twine &name) {
+  auto yield = dyn_cast<emitc::YieldOp>(region.front().back());
+  if (!yield)
+    return op.emitOpError("expected region to end with emitc.yield, but got ")
+           << region.front().back().getName();
+
+  if (yield.getNumOperands() != 0) {
+    return (op.emitOpError("expected each region to return ")
+            << "0 values, but " << name << " returns "
+            << yield.getNumOperands())
+               .attachNote(yield.getLoc())
+           << "see yield operation here";
+  }
+
+  return success();
+}
+
+LogicalResult emitc::SwitchOp::verify() {
+  if (!isIntegerIndexOrOpaqueType(getArg().getType()))
+    return emitOpError("unsupported type ") << getArg().getType();
+
+  if (getCases().size() != getCaseRegions().size()) {
+    return emitOpError("has ")
+           << getCaseRegions().size() << " case regions but "
+           << getCases().size() << " case values";
+  }
+
+  DenseSet<int64_t> valueSet;
+  for (int64_t value : getCases())
+    if (!valueSet.insert(value).second)
+      return emitOpError("has duplicate case value: ") << value;
+
+  if (failed(verifyRegion(*this, getDefaultRegion(), "default region")))
+    return failure();
+
+  for (auto [idx, caseRegion] : llvm::enumerate(getCaseRegions()))
+    if (failed(verifyRegion(*this, caseRegion, "case region #" + Twine(idx))))
+      return failure();
+
+  return success();
+}
+
+unsigned emitc::SwitchOp::getNumCases() { return getCases().size(); }
+
+Block &emitc::SwitchOp::getDefaultBlock() { return getDefaultRegion().front(); }
+
+Block &emitc::SwitchOp::getCaseBlock(unsigned idx) {
+  assert(idx < getNumCases() && "case index out-of-bounds");
+  return getCaseRegions()[idx].front();
+}
+
+void SwitchOp::getSuccessorRegions(
+    RegionBranchPoint point, SmallVectorImpl<RegionSuccessor> &successors) {
+  llvm::copy(getRegions(), std::back_inserter(successors));
+}
+
+void SwitchOp::getEntrySuccessorRegions(
+    ArrayRef<Attribute> operands,
+    SmallVectorImpl<RegionSuccessor> &successors) {
+  FoldAdaptor adaptor(operands, *this);
+
+  // If a constant was not provided, all regions are possible successors.
+  auto arg = dyn_cast_or_null<IntegerAttr>(adaptor.getArg());
+  if (!arg) {
+    llvm::copy(getRegions(), std::back_inserter(successors));
+    return;
+  }
+
+  // Otherwise, try to find a case with a matching value. If not, the
+  // default region is the only successor.
+  for (auto [caseValue, caseRegion] : llvm::zip(getCases(), getCaseRegions())) {
+    if (caseValue == arg.getInt()) {
+      successors.emplace_back(&caseRegion);
+      return;
+    }
+  }
+  successors.emplace_back(&getDefaultRegion());
+}
+
+void SwitchOp::getRegionInvocationBounds(
+    ArrayRef<Attribute> operands, SmallVectorImpl<InvocationBounds> &bounds) {
+  auto operandValue = llvm::dyn_cast_or_null<IntegerAttr>(operands.front());
+  if (!operandValue) {
+    // All regions are invoked at most once.
+    bounds.append(getNumRegions(), InvocationBounds(/*lb=*/0, /*ub=*/1));
+    return;
+  }
+
+  unsigned liveIndex = getNumRegions() - 1;
+  const auto *iteratorToInt = llvm::find(getCases(), operandValue.getInt());
+
+  liveIndex = iteratorToInt != getCases().end()
+                  ? std::distance(getCases().begin(), iteratorToInt)
+                  : liveIndex;
+
+  for (unsigned regIndex = 0, regNum = getNumRegions(); regIndex < regNum;
+       ++regIndex)
+    bounds.emplace_back(/*lb=*/0, /*ub=*/regIndex == liveIndex);
+}
+
 //===----------------------------------------------------------------------===//
 // TableGen'd op method definitions
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Target/Cpp/TranslateToCpp.cpp b/mlir/lib/Target/Cpp/TranslateToCpp.cpp
index 1dadb9dd691e70a..8e112e6f7dda652 100644
--- a/mlir/lib/Target/Cpp/TranslateToCpp.cpp
+++ b/mlir/lib/Target/Cpp/TranslateToCpp.cpp
@@ -449,6 +449,43 @@ static LogicalResult printOperation(CppEmitter &emitter, emitc::SubOp subOp) {
   return printBinaryOperation(emitter, operation, "-");
 }
 
+static LogicalResult emitSwitchCase(CppEmitter &emitter,
+                                    raw_indented_ostream &os, Region &region) {
+  for (Region::OpIterator iteratorOp = region.op_begin(), end = region.op_end();
+       std::next(iteratorOp) != end; ++iteratorOp) {
+    if (failed(emitter.emitOperation(*iteratorOp, /*trailingSemicolon=*/true)))
+      return failure();
+  }
+  os << "break;\n";
+  return success();
+}
+
+static LogicalResult printOperation(CppEmitter &emitter,
+                                    emitc::SwitchOp switchOp) {
+  raw_indented_ostream &os = emitter.ostream();
+
+  os << "\nswitch (" << emitter.getOrCreateName(switchOp.getArg()) << ") {";
+
+  for (auto pair : llvm::zip(switchOp.getCases(), switchOp.getCaseRegions())) {
+    os << "\ncase " << std::get<0>(pair) << ": {\n";
+    os.indent();
+
+    if (failed(emitSwitchCase(emitter, os, std::get<1>(pair))))
+      return failure();
+
+    os.unindent() << "}";
+  }
+
+  os << "\ndefault: {\n";
+  os.indent();
+
+  if (failed(emitSwitchCase(emitter, os, switchOp.getDefaultRegion())))
+    return failure();
+
+  os.unindent() << "}";
+  return success();
+}
+
 static LogicalResult printOperation(CppEmitter &emitter, emitc::CmpOp cmpOp) {
   Operation *operation = cmpOp.getOperation();
 
@@ -998,7 +1035,7 @@ static LogicalResult printFunctionBody(CppEmitter &emitter,
       // trailing semicolon is handled within the printOperation function.
       bool trailingSemicolon =
           !isa<cf::CondBranchOp, emitc::DeclareFuncOp, emitc::ForOp,
-               emitc::IfOp, emitc::VerbatimOp>(op);
+               emitc::IfOp, emitc::SwitchOp, emitc::VerbatimOp>(op);
 
       if (failed(emitter.emitOperation(
               op, /*trailingSemicolon=*/trailingSemicolon)))
@@ -1509,8 +1546,8 @@ LogicalResult CppEmitter::emitOperation(Operation &op, bool trailingSemicolon) {
                 emitc::GlobalOp, emitc::IfOp, emitc::IncludeOp,
                 emitc::LogicalAndOp, emitc::LogicalNotOp, emitc::LogicalOrOp,
                 emitc::MulOp, emitc::RemOp, emitc::ReturnOp, emitc::SubOp,
-                emitc::UnaryMinusOp, emitc::UnaryPlusOp, emitc::VariableOp,
-                emitc::VerbatimOp>(
+                emitc::SwitchOp, emitc::UnaryMinusOp, emitc::UnaryPlusOp,
+                emitc::VariableOp, emitc::VerbatimOp>(
               [&](auto op) { return printOperation(*this, op); })
           // Func ops.
           .Case<func::CallOp, func::FuncOp, func::ReturnOp>(
diff --git a/mlir/test/Conversion/SCFToEmitC/switch.mlir b/mlir/test/Conversion/SCFToEmitC/switch.mlir
new file mode 100644
index 000000000000000..659d9f43963efa5
--- /dev/null
+++ b/mlir/test/Conversion/SCFToEmitC/switch.mlir
@@ -0,0 +1,117 @@
+// RUN: mlir-opt -allow-unregistered-dialect -convert-scf-to-emitc %s | FileCheck %s
+
+// CHECK-LABEL:   func.func @switch_no_result(
+// CHECK-SAME:                                %[[VAL_0:.*]]: index) {
+// CHECK:           emitc.switch %[[VAL_0]]
+// CHECK:           case 2 {
+// CHECK:             %[[VAL_1:.*]] = arith.constant 10 : i32
+// CHECK:             emitc.yield
+// CHECK:           }
+// CHECK:           case 5 {
+// CHECK:             %[[VAL_2:.*]] = arith.constant 20 : i32
+// CHECK:             emitc.yield
+// CHECK:           }
+// CHECK:           default {
+// CHECK:             %[[VAL_3:.*]] = arith.constant 30 : i32
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
+func.func @switch_no_result(%arg0 : index) {
+    scf.index_switch %arg0
+    case 2 {
+      %1 = arith.constant 10 : i32
+      scf.yield
+    }
+    case 5 {
+      %2 = arith.constant 20 : i32
+      scf.yield
+    }
+    default {
+      %3 = arith.constant 30 : i32
+    }
+  return
+}
+
+// CHECK-LABEL:   func.func @switch_one_result(
+// CHECK-SAME:                                 %[[VAL_0:.*]]: index) {
+// CHECK:           %[[VAL_1:.*]] = "emitc.variable"() <{value = #emitc.opaque<"">}> : () -> i32
+// CHECK:           emitc.switch %[[VAL_0]]
+// CHECK:           case 2 {
+// CHECK:             %[[VAL_2:.*]] = arith.constant 10 : i32
+// CHECK:             emitc.assign %[[VAL_2]] : i32 to %[[VAL_1]] : i32
+// CHECK:             emitc.yield
+// CHECK:           }
+// CHECK:           case 5 {
+// CHECK:             %[[VAL_3:.*]] = arith.constant 20 : i32
+// CHECK:             emitc.assign %[[VAL_3]] : i32 to %[[VAL_1]] : i32
+// CHECK:             emitc.yield
+// CHECK:           }
+// CHECK:           default {
+// CHECK:             %[[VAL_4:.*]] = arith.constant 30 : i32
+// CHECK:             emitc.assign %[[VAL_4]] : i32 to %[[VAL_1]] : i32
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
+func.func @switch_one_result(%arg0 : index) {
+    %0 = scf.index_switch %arg0 -> i32
+    case 2 {
+      %1 = arith.constant 10 : i32
+      scf.yield %1 : i32
+    }
+    case 5 {
+      %2 = arith.constant 20 : i32
+      scf.yield %2 : i32
+    }
+    default {
+      %3 = arith.constant 30 : i32
+      scf.yield %3 : i32
+    }
+  return
+}
+
+// CHECK-LABEL:   func.func @switch_two_results(
+// CHECK-SAME:                                  %[[VAL_0:.*]]: index) {
+// CHECK:           %[[VAL_1:.*]] = "emitc.variable"() <{value = #emitc.opaque<"">}> : () -> i32
+// CHECK:           %[[VAL_2:.*]] = "emitc.variable"() <{value = #emitc.opaque<"">}> : () -> f32
+// CHECK:           emitc.switch %[[VAL_0]]
+// CHECK:           case 2 {
+// CHECK:             %[[VAL_3:.*]] = arith.constant 10 : i32
+// CHECK:             %[[VAL_4:.*]] = arith.constant 1.200000e+00 : f32
+// CHECK:             emitc.assign %[[VAL_3]] : i32 to %[[VAL_1]] : i32
+// CHECK:             emitc.assign %[[VAL_4]] : f32 to %[[VAL_2]] : f32
+// CHECK:             emitc.yield
+// CHECK:           }
+// CHECK:           case 5 {
+// CHECK:             %[[VAL_5:.*]] = arith.constant 20 : i32
+// CHECK:             %[[VAL_6:.*]] = arith.constant 2.400000e+00 : f32
+// CHECK:             emitc.assign %[[VAL_5]] : i32 to %[[VAL_1]] : i32
+// CHECK:             emitc.assign %[[VAL_6]] : f32 to %[[VAL_2]] : f32
+// CHECK:             emitc.yield
+// CHECK:           }
+// CHECK:           default {
+// CHECK:             %[[VAL_7:.*]] = arith.constant 30 : i32
+// CHECK:             %[[VAL_8:.*]] = arith.constant 3.600000e+00 : f32
+// CHECK:             emitc.assign %[[VAL_7]] : i32 to %[[VAL_1]] : i32
+// CHECK:             emitc.assign %[[VAL_8]] : f32 to %[[VAL_2]] : f32
+// CHECK:           }
+// CHECK:           return
+// CHECK:         }
+func.func @switch_two_results(%arg0 : index) {
+    %0, %1 = scf.index_switch %arg0 -> i32, f32
+    case 2 {
+      %2 = arith.constant 10 : i32
+      %3 = arith.constant 1.2 : f32
+      scf.yield %2, %3 : i32, f32
+    }
+    case 5 {
+      %4 = arith.constant 20 : i32
+      %5 = arith.constant 2.4 : f32
+      scf.yield %4, %5 : i32, f32
+    }
+    default {
+      %6 = arith.constant 30 : i32
+      %7 = arith.constant 3.6 : f32
+      scf.yield %6, %7 : i32, f32
+    }
+  return
+}
diff --git a/mlir/test/Dialect/EmitC/invalid_ops.mlir b/mlir/test/Dialect/EmitC/invalid_ops.mlir
index 4181b726593e4a5..4b5bcf46c1aab97 100644
--- a/mlir/test/Dialect/EmitC/invalid_ops.mlir
+++ b/mlir/test/Dialect/EmitC/invalid_ops.mlir
@@ -227,7 +227,7 @@ func.func @sub_pointer_pointer(%arg0: !emitc.ptr<f32>, %arg1: !emitc.ptr<f32>) {
 // -----
 
 func.func @test_misplaced_yield() {
-  // expected-error @+1 {{'emitc.yield' op expects parent op to be one of 'emitc.expression, emitc.if, emitc.for'}}
+  // expected-error @+1 {{'emitc.yield' op expects parent op to be one of 'emitc.expression, emitc.if, emitc.for, emitc.switch'}}
   emitc.yield
   return
 }
@@ -466,3 +466,89 @@ func.func @member_of_ptr(%arg0: i32) {
   %0 = "emitc.member_of_ptr" (%arg0) {member = "a"} : (i32) -> i32
   return
 }
+
+// -----
+
+func.func @emitc_switch() {
+  %0 = "emitc.variable"(){value = 1 : i16} : () -> i16
+
+  // expected-error@+1 {{'emitc.switch' op expected region to end with emitc.yield, but got emitc.call_opaque}}
+  emitc.switch %0 : i16
+  case 2 {
+    %1 = emitc.call_opaque "func_b" () : () -> i32
+  }
+  case 5 {
+    %2 = emitc.call_opaque "func_a" () : () -> i32
+    emitc.yield
+  }
+  default {
+    %3 = "emitc.variable"(){value = 42.0 : f32} : () -> f32
+    emitc.call_opaque "func2" (%3) : (f32) -> ()
+    emitc.yield
+  }
+  return
+}
+
+// -----
+
+func.func @emitc_switch() {
+  %0 = "emitc.variable"(){value = 1 : i32} : () -> i32
+
+  emitc.switch %0 : i32
+  case 2 {
+    %1 = emitc.call_opaque "func_b" () : () -> i32
+    emitc.yield
+  }
+  // expected-error@+1 {{custom op 'emitc.switch' expected integer value}}
+  case {
+    %2 = emitc.call_opaque "func_a" () : () -> i32
+    emitc.yield
+  }
+  default {
+    %3 = "emitc.variable"(){value = 42.0 : f32} : () -> f32
+    emitc.call_opaque "func2" (%3) : (f32) -> ()
+    emitc.yield
+  }
+  return
+}
+
+// -----
+
+func.func @emitc_switch() {
+  %0 = "emitc.variable"(){value = 1 : i8} : () -> i8
+
+  emitc.switch %0 : i8
+  case 2 {
+    %1 = emitc.call_opaque "func_b" () : () -> i32
+    emitc.yield
+  }
+  case 3 {
+    %2 = emitc.call_opaque "func_a" () : () -> i32
+    emitc.yield
+  }
+  // expected-error@+1 {{custom op 'emitc.switch' expected 'default'}}
+  return
+}
+
+// -----
+
+func.func @emitc_switch() {
+  %0 = "emitc.variable"(){value = 1 : i64} : () -> i64
+
+  // expected-error@+1 {{'emitc.switch' op has duplicate case value: 2}}
+  emitc.switch %0 : i64
+  case 2 {
+    %1 = emitc.call_opaque "func_b" () : () -> i32
+    emitc.yield
+  }
+  case 2 {
+    %2 = emitc.call_opaque "func_a" () : () -> i32
+    emitc.yield
+  }
+  default {
+    %3 = "emitc.variable"(){value = 42.0 : f32} : () -> f32
+    emitc.call_opaque "func2" (%3) : (f32) -> ()
+    emitc.yield
+  }
+  return
+}
diff --git a/mlir/test/Dialect/EmitC/ops.mlir b/mlir/test/Dialect/EmitC/ops.mlir
index 20ac077e4402b4e..64f22e8ad6b983c 100644
--- a/mlir/test/Dialect/EmitC/ops.mlir
+++ b/mlir/test/Dialect/EmitC/ops.mlir
@@ -261,3 +261,23 @@ func.func @member_access(%arg0: !emitc.opaque<"mystruct">, %arg1: !emitc.opaque<
   %2 = "emitc.member_of_ptr" (%arg2) {member = "a"} : (!emitc.ptr<!emitc.opaque<"mystruct">>) -> i32
   return
 }
+
+func.func @switch() {
+  %0 = "emitc.variable"(){value = 1 : index} : () -> !emitc.ptrdiff_t
+
+  emitc.switch %0 : !emitc.ptrdiff_t
+  case 1 {
+    %1 = emitc.call_opaque "func_b" () : () -> i32
+    emitc.yield
+  }
+  case 2 {
+    %2 = emitc.call_opaque "func_a" () : () -> i32
+    emitc.yield
+  }
+  default {
+    %3 = "emitc.variable"(){value = 42.0 : f32} : () -> f32
+    emitc.call_opaque "func2" (%3) : (f32) -> ()
+  }
+
+  return 
+}
diff --git a/mlir/test/Target/Cpp/invalid.mlir b/mlir/test/Target/Cpp/invalid.mlir
index 513371a09cde1d2..4e777239fba6629 100644
--- a/mlir/test/Target/Cpp/invalid.mlir
+++ b/mlir/test/Target/Cpp/invalid.mlir
@@ -80,6 +80,7 @@ func.func @array_as_result(%arg: !emitc.array<4xi8>) -> (!emitc.array<4xi8>) {
 }
 
 // -----
+
 func.func @ptr_to_array() {
   // expected-error@+1 {{cannot emit pointer to array type '!emitc.ptr<!emitc.array<9xi16>>'}}
   %v = "emitc.variable"(){value = #emitc.opaque<"NULL">} : () -> !emitc.ptr<!emitc.array<9xi16>>
diff --git a/mlir/test/Target/Cpp/switch.mlir b/mlir/test/Target/Cpp/switch.mlir
new file mode 100644
index 000000000000000..0f2e716a98f16bd
--- /dev/null
+++ b/mlir/test/Target/Cpp/switch.mlir
@@ -0,0 +1,856 @@
+// RUN: mlir-translate -mlir-to-cpp %s | FileCheck %s -check-prefix=CPP-DEFAULT
+// RUN: mlir-translate -mlir-to-cpp -declare-variables-at-top %s | FileCheck %s -check-prefix=CPP-DECLTOP
+
+// CPP-DEFAULT-LABEL: void emitc_switch_ptrdiff_t() {
+// CPP-DEFAULT:         ptrdiff_t v1 = 1;
+// CPP-DEFAULT:         switch (v1) {
+// CPP-DEFAULT:         case 2: {
+// CPP-DEFAULT:           int32_t v2 = func_b();
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         case 5: {
+// CPP-DEFAULT:           int32_t v3 = func_a();
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         default: {
+// CPP-DEFAULT:           float v4 = 4.200000000e+01f;
+// CPP-DEFAULT:           func2(v4);
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         return;
+// CPP-DEFAULT:       }
+
+// CPP-DECLTOP-LABEL: void emitc_switch_ptrdiff_t() {
+// CPP-DECLTOP:         ptrdiff_t v1;
+// CPP-DECLTOP:         float v2;
+// CPP-DECLTOP:         int32_t v3;
+// CPP-DECLTOP:         int32_t v4;
+// CPP-DECLTOP:         v1 = 1;
+// CPP-DECLTOP:         switch (v1) {
+// CPP-DECLTOP:         case 2: {
+// CPP-DECLTOP:           v3 = func_b();
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         case 5: {
+// CPP-DECLTOP:           v4 = func_a();
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         default: {
+// CPP-DECLTOP:           v2 = 4.200000000e+01f;
+// CPP-DECLTOP:           func2(v2);
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         return;
+// CPP-DECLTOP:       }
+func.func @emitc_switch_ptrdiff_t() {
+  %0 = "emitc.variable"(){value = 1 : index} : () -> !emitc.ptrdiff_t
+
+  emitc.switch %0 : !emitc.ptrdiff_t
+  case 2 {
+    %1 = emitc.call_opaque "func_b" () : () -> i32
+    emitc.yield
+  }
+  case 5 {
+    %2 = emitc.call_opaque "func_a" () : () -> i32
+    emitc.yield
+  }
+  default {
+    %3 = "emitc.variable"(){value = 42.0 : f32} : () -> f32
+    emitc.call_opaque "func2" (%3) : (f32) -> ()
+  }
+  return
+}
+
+// CPP-DEFAULT-LABEL: void emitc_switch_ssize_t() {
+// CPP-DEFAULT:         ssize_t v1 = 1;
+// CPP-DEFAULT:         switch (v1) {
+// CPP-DEFAULT:         case 2: {
+// CPP-DEFAULT:           int32_t v2 = func_b();
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         case 5: {
+// CPP-DEFAULT:           int32_t v3 = func_a();
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         default: {
+// CPP-DEFAULT:           float v4 = 4.200000000e+01f;
+// CPP-DEFAULT:           func2(v4);
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         return;
+// CPP-DEFAULT:       }
+
+// CPP-DECLTOP-LABEL: void emitc_switch_ssize_t() {
+// CPP-DECLTOP:         ssize_t v1;
+// CPP-DECLTOP:         float v2;
+// CPP-DECLTOP:         int32_t v3;
+// CPP-DECLTOP:         int32_t v4;
+// CPP-DECLTOP:         v1 = 1;
+// CPP-DECLTOP:         switch (v1) {
+// CPP-DECLTOP:         case 2: {
+// CPP-DECLTOP:           v3 = func_b();
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         case 5: {
+// CPP-DECLTOP:           v4 = func_a();
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         default: {
+// CPP-DECLTOP:           v2 = 4.200000000e+01f;
+// CPP-DECLTOP:           func2(v2);
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         return;
+// CPP-DECLTOP:       }
+func.func @emitc_switch_ssize_t() {
+  %0 = "emitc.variable"(){value = 1 : index} : () -> !emitc.ssize_t
+
+  emitc.switch %0 : !emitc.ssize_t
+  case 2 {
+    %1 = emitc.call_opaque "func_b" () : () -> i32
+    emitc.yield
+  }
+  case 5 {
+    %2 = emitc.call_opaque "func_a" () : () -> i32
+    emitc.yield
+  }
+  default {
+    %3 = "emitc.variable"(){value = 42.0 : f32} : () -> f32
+    emitc.call_opaque "func2" (%3) : (f32) -> ()
+    emitc.yield
+  }
+  return
+}
+
+// CPP-DEFAULT-LABEL: void emitc_switch_size_t() {
+// CPP-DEFAULT:         size_t v1 = 1;
+// CPP-DEFAULT:         switch (v1) {
+// CPP-DEFAULT:         case 2: {
+// CPP-DEFAULT:           int32_t v2 = func_b();
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         case 5: {
+// CPP-DEFAULT:           int32_t v3 = func_a();
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         default: {
+// CPP-DEFAULT:           float v4 = 4.200000000e+01f;
+// CPP-DEFAULT:           func2(v4);
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         return;
+// CPP-DEFAULT:       }
+
+// CPP-DECLTOP-LABEL: void emitc_switch_size_t() {
+// CPP-DECLTOP:         size_t v1;
+// CPP-DECLTOP:         float v2;
+// CPP-DECLTOP:         int32_t v3;
+// CPP-DECLTOP:         int32_t v4;
+// CPP-DECLTOP:         v1 = 1;
+// CPP-DECLTOP:         switch (v1) {
+// CPP-DECLTOP:         case 2: {
+// CPP-DECLTOP:           v3 = func_b();
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         case 5: {
+// CPP-DECLTOP:           v4 = func_a();
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         default: {
+// CPP-DECLTOP:           v2 = 4.200000000e+01f;
+// CPP-DECLTOP:           func2(v2);
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         return;
+// CPP-DECLTOP:       }
+func.func @emitc_switch_size_t() {
+  %0 = "emitc.variable"(){value = 1 : index} : () -> !emitc.size_t
+
+  emitc.switch %0 : !emitc.size_t
+  case 2 {
+    %1 = emitc.call_opaque "func_b" () : () -> i32
+    emitc.yield
+  }
+  case 5 {
+    %2 = emitc.call_opaque "func_a" () : () -> i32
+    emitc.yield
+  }
+  default {
+    %3 = "emitc.variable"(){value = 42.0 : f32} : () -> f32
+    emitc.call_opaque "func2" (%3) : (f32) -> ()
+    emitc.yield
+  }
+  return
+}
+
+// CPP-DEFAULT-LABEL: void emitc_switch_index() {
+// CPP-DEFAULT:         size_t v1 = 1;
+// CPP-DEFAULT:         switch (v1) {
+// CPP-DEFAULT:         case 2: {
+// CPP-DEFAULT:           int32_t v2 = func_b();
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         case 5: {
+// CPP-DEFAULT:           int32_t v3 = func_a();
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         default: {
+// CPP-DEFAULT:           float v4 = 4.200000000e+01f;
+// CPP-DEFAULT:           func2(v4);
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         return;
+// CPP-DEFAULT:       }
+
+// CPP-DECLTOP-LABEL: void emitc_switch_index() {
+// CPP-DECLTOP:         size_t v1;
+// CPP-DECLTOP:         float v2;
+// CPP-DECLTOP:         int32_t v3;
+// CPP-DECLTOP:         int32_t v4;
+// CPP-DECLTOP:         v1 = 1;
+// CPP-DECLTOP:         switch (v1) {
+// CPP-DECLTOP:         case 2: {
+// CPP-DECLTOP:           v3 = func_b();
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         case 5: {
+// CPP-DECLTOP:           v4 = func_a();
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         default: {
+// CPP-DECLTOP:           v2 = 4.200000000e+01f;
+// CPP-DECLTOP:           func2(v2);
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         return;
+// CPP-DECLTOP:       }
+func.func @emitc_switch_index() {
+  %0 = "emitc.variable"(){value = 1 : index} : () -> index
+
+  emitc.switch %0 : index
+  case 2 {
+    %1 = emitc.call_opaque "func_b" () : () -> i32
+    emitc.yield
+  }
+  case 5 {
+    %2 = emitc.call_opaque "func_a" () : () -> i32
+    emitc.yield
+  }
+  default {
+    %3 = "emitc.variable"(){value = 42.0 : f32} : () -> f32
+    emitc.call_opaque "func2" (%3) : (f32) -> ()
+    emitc.yield
+  }
+  return
+}
+
+// CPP-DEFAULT-LABEL: void emitc_switch_opaque() {
+// CPP-DEFAULT:         size_t v1 = 1;
+// CPP-DEFAULT:         switch (v1) {
+// CPP-DEFAULT:         case 2: {
+// CPP-DEFAULT:           int32_t v2 = func_b();
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         case 5: {
+// CPP-DEFAULT:           int32_t v3 = func_a();
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         default: {
+// CPP-DEFAULT:           float v4 = 4.200000000e+01f;
+// CPP-DEFAULT:           func2(v4);
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         return;
+// CPP-DEFAULT:       }
+
+// CPP-DECLTOP-LABEL: void emitc_switch_opaque() {
+// CPP-DECLTOP:         size_t v1;
+// CPP-DECLTOP:         float v2;
+// CPP-DECLTOP:         int32_t v3;
+// CPP-DECLTOP:         int32_t v4;
+// CPP-DECLTOP:         v1 = 1;
+// CPP-DECLTOP:         switch (v1) {
+// CPP-DECLTOP:         case 2: {
+// CPP-DECLTOP:           v3 = func_b();
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         case 5: {
+// CPP-DECLTOP:           v4 = func_a();
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         default: {
+// CPP-DECLTOP:           v2 = 4.200000000e+01f;
+// CPP-DECLTOP:           func2(v2);
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         return;
+// CPP-DECLTOP:       }
+func.func @emitc_switch_opaque() {
+  %0 = "emitc.variable"() {value = #emitc.opaque<"1">} 
+  : () -> !emitc.opaque<"size_t">
+
+  emitc.switch %0 : !emitc.opaque<"size_t">
+  case 2 {
+    %1 = emitc.call_opaque "func_b" () : () -> i32
+    emitc.yield
+  }
+  case 5 {
+    %2 = emitc.call_opaque "func_a" () : () -> i32
+    emitc.yield
+  }
+  default {
+    %3 = "emitc.variable"(){value = 42.0 : f32} : () -> f32
+    emitc.call_opaque "func2" (%3) : (f32) -> ()
+    emitc.yield
+  }
+  return
+}
+
+// CPP-DEFAULT-LABEL: void emitc_switch_i1() {
+// CPP-DEFAULT:         bool v1 = true;
+// CPP-DEFAULT:         switch (v1) {
+// CPP-DEFAULT:         case 2: {
+// CPP-DEFAULT:           int32_t v2 = func_b();
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         case 5: {
+// CPP-DEFAULT:           int32_t v3 = func_a();
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         default: {
+// CPP-DEFAULT:           float v4 = 4.200000000e+01f;
+// CPP-DEFAULT:           func2(v4);
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         return;
+// CPP-DEFAULT:       }
+
+// CPP-DECLTOP-LABEL: void emitc_switch_i1() {
+// CPP-DECLTOP:         bool v1;
+// CPP-DECLTOP:         float v2;
+// CPP-DECLTOP:         int32_t v3;
+// CPP-DECLTOP:         int32_t v4;
+// CPP-DECLTOP:         v1 = true;
+// CPP-DECLTOP:         switch (v1) {
+// CPP-DECLTOP:         case 2: {
+// CPP-DECLTOP:           v3 = func_b();
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         case 5: {
+// CPP-DECLTOP:           v4 = func_a();
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         default: {
+// CPP-DECLTOP:           v2 = 4.200000000e+01f;
+// CPP-DECLTOP:           func2(v2);
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         return;
+// CPP-DECLTOP:       }
+func.func @emitc_switch_i1() {
+  %0 = "emitc.variable"(){value = 1 : i1} : () -> i1
+
+  emitc.switch %0 : i1
+  case 2 {
+    %1 = emitc.call_opaque "func_b" () : () -> i32
+    emitc.yield
+  }
+  case 5 {
+    %2 = emitc.call_opaque "func_a" () : () -> i32
+    emitc.yield
+  }
+  default {
+    %3 = "emitc.variable"(){value = 42.0 : f32} : () -> f32
+    emitc.call_opaque "func2" (%3) : (f32) -> ()
+    emitc.yield
+  }
+  return
+}
+
+// CPP-DEFAULT-LABEL: void emitc_switch_i8() {
+// CPP-DEFAULT:         int8_t v1 = 1;
+// CPP-DEFAULT:         switch (v1) {
+// CPP-DEFAULT:         case 2: {
+// CPP-DEFAULT:           int32_t v2 = func_b();
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         case 5: {
+// CPP-DEFAULT:           int32_t v3 = func_a();
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         default: {
+// CPP-DEFAULT:           float v4 = 4.200000000e+01f;
+// CPP-DEFAULT:           func2(v4);
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         return;
+// CPP-DEFAULT:       }
+
+// CPP-DECLTOP-LABEL: void emitc_switch_i8() {
+// CPP-DECLTOP:         int8_t v1;
+// CPP-DECLTOP:         float v2;
+// CPP-DECLTOP:         int32_t v3;
+// CPP-DECLTOP:         int32_t v4;
+// CPP-DECLTOP:         v1 = 1;
+// CPP-DECLTOP:         switch (v1) {
+// CPP-DECLTOP:         case 2: {
+// CPP-DECLTOP:           v3 = func_b();
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         case 5: {
+// CPP-DECLTOP:           v4 = func_a();
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         default: {
+// CPP-DECLTOP:           v2 = 4.200000000e+01f;
+// CPP-DECLTOP:           func2(v2);
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         return;
+// CPP-DECLTOP:       }
+func.func @emitc_switch_i8() {
+  %0 = "emitc.variable"(){value = 1 : i8} : () -> i8
+
+  emitc.switch %0 : i8
+  case 2 {
+    %1 = emitc.call_opaque "func_b" () : () -> i32
+    emitc.yield
+  }
+  case 5 {
+    %2 = emitc.call_opaque "func_a" () : () -> i32
+    emitc.yield
+  }
+  default {
+    %3 = "emitc.variable"(){value = 42.0 : f32} : () -> f32
+    emitc.call_opaque "func2" (%3) : (f32) -> ()
+    emitc.yield
+  }
+  return
+}
+
+// CPP-DEFAULT-LABEL: void emitc_switch_ui8() {
+// CPP-DEFAULT:         uint8_t v1 = 1;
+// CPP-DEFAULT:         switch (v1) {
+// CPP-DEFAULT:         case 2: {
+// CPP-DEFAULT:           int32_t v2 = func_b();
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         case 5: {
+// CPP-DEFAULT:           int32_t v3 = func_a();
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         default: {
+// CPP-DEFAULT:           float v4 = 4.200000000e+01f;
+// CPP-DEFAULT:           func2(v4);
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         return;
+// CPP-DEFAULT:       }
+
+// CPP-DECLTOP-LABEL: void emitc_switch_ui8() {
+// CPP-DECLTOP:         uint8_t v1;
+// CPP-DECLTOP:         float v2;
+// CPP-DECLTOP:         int32_t v3;
+// CPP-DECLTOP:         int32_t v4;
+// CPP-DECLTOP:         v1 = 1;
+// CPP-DECLTOP:         switch (v1) {
+// CPP-DECLTOP:         case 2: {
+// CPP-DECLTOP:           v3 = func_b();
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         case 5: {
+// CPP-DECLTOP:           v4 = func_a();
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         default: {
+// CPP-DECLTOP:           v2 = 4.200000000e+01f;
+// CPP-DECLTOP:           func2(v2);
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         return;
+// CPP-DECLTOP:       }
+func.func @emitc_switch_ui8() {
+  %0 = "emitc.variable"(){value = 1 : ui8} : () -> ui8
+
+  emitc.switch %0 : ui8
+  case 2 {
+    %1 = emitc.call_opaque "func_b" () : () -> i32
+    emitc.yield
+  }
+  case 5 {
+    %2 = emitc.call_opaque "func_a" () : () -> i32
+    emitc.yield
+  }
+  default {
+    %3 = "emitc.variable"(){value = 42.0 : f32} : () -> f32
+    emitc.call_opaque "func2" (%3) : (f32) -> ()
+    emitc.yield
+  }
+  return
+}
+
+// CPP-DEFAULT-LABEL: void emitc_switch_i16() {
+// CPP-DEFAULT:         int16_t v1 = 1;
+// CPP-DEFAULT:         switch (v1) {
+// CPP-DEFAULT:         case 2: {
+// CPP-DEFAULT:           int32_t v2 = func_b();
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         case 5: {
+// CPP-DEFAULT:           int32_t v3 = func_a();
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         default: {
+// CPP-DEFAULT:           float v4 = 4.200000000e+01f;
+// CPP-DEFAULT:           func2(v4);
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         return;
+// CPP-DEFAULT:       }
+
+// CPP-DECLTOP-LABEL: void emitc_switch_i16() {
+// CPP-DECLTOP:         int16_t v1;
+// CPP-DECLTOP:         float v2;
+// CPP-DECLTOP:         int32_t v3;
+// CPP-DECLTOP:         int32_t v4;
+// CPP-DECLTOP:         v1 = 1;
+// CPP-DECLTOP:         switch (v1) {
+// CPP-DECLTOP:         case 2: {
+// CPP-DECLTOP:           v3 = func_b();
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         case 5: {
+// CPP-DECLTOP:           v4 = func_a();
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         default: {
+// CPP-DECLTOP:           v2 = 4.200000000e+01f;
+// CPP-DECLTOP:           func2(v2);
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         return;
+// CPP-DECLTOP:       }
+func.func @emitc_switch_i16() {
+  %0 = "emitc.variable"(){value = 1 : i16} : () -> i16
+
+  emitc.switch %0 : i16
+  case 2 {
+    %1 = emitc.call_opaque "func_b" () : () -> i32
+    emitc.yield
+  }
+  case 5 {
+    %2 = emitc.call_opaque "func_a" () : () -> i32
+    emitc.yield
+  }
+  default {
+    %3 = "emitc.variable"(){value = 42.0 : f32} : () -> f32
+    emitc.call_opaque "func2" (%3) : (f32) -> ()
+    emitc.yield
+  }
+  return
+}
+
+// CPP-DEFAULT-LABEL: void emitc_switch_ui16() {
+// CPP-DEFAULT:         uint16_t v1 = 1;
+// CPP-DEFAULT:         switch (v1) {
+// CPP-DEFAULT:         case 2: {
+// CPP-DEFAULT:           int32_t v2 = func_b();
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         case 5: {
+// CPP-DEFAULT:           int32_t v3 = func_a();
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         default: {
+// CPP-DEFAULT:           float v4 = 4.200000000e+01f;
+// CPP-DEFAULT:           func2(v4);
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         return;
+// CPP-DEFAULT:       }
+
+// CPP-DECLTOP-LABEL: void emitc_switch_ui16() {
+// CPP-DECLTOP:         uint16_t v1;
+// CPP-DECLTOP:         float v2;
+// CPP-DECLTOP:         int32_t v3;
+// CPP-DECLTOP:         int32_t v4;
+// CPP-DECLTOP:         v1 = 1;
+// CPP-DECLTOP:         switch (v1) {
+// CPP-DECLTOP:         case 2: {
+// CPP-DECLTOP:           v3 = func_b();
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         case 5: {
+// CPP-DECLTOP:           v4 = func_a();
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         default: {
+// CPP-DECLTOP:           v2 = 4.200000000e+01f;
+// CPP-DECLTOP:           func2(v2);
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         return;
+// CPP-DECLTOP:       }
+func.func @emitc_switch_ui16() {
+  %0 = "emitc.variable"(){value = 1 : ui16} : () -> ui16
+
+  emitc.switch %0 : ui16
+  case 2 {
+    %1 = emitc.call_opaque "func_b" () : () -> i32
+    emitc.yield
+  }
+  case 5 {
+    %2 = emitc.call_opaque "func_a" () : () -> i32
+    emitc.yield
+  }
+  default {
+    %3 = "emitc.variable"(){value = 42.0 : f32} : () -> f32
+    emitc.call_opaque "func2" (%3) : (f32) -> ()
+    emitc.yield
+  }
+  return
+}
+
+// CPP-DEFAULT-LABEL: void emitc_switch_i32() {
+// CPP-DEFAULT:         int32_t v1 = 1;
+// CPP-DEFAULT:         switch (v1) {
+// CPP-DEFAULT:         case 2: {
+// CPP-DEFAULT:           int32_t v2 = func_b();
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         case 5: {
+// CPP-DEFAULT:           int32_t v3 = func_a();
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         default: {
+// CPP-DEFAULT:           float v4 = 4.200000000e+01f;
+// CPP-DEFAULT:           func2(v4);
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         return;
+// CPP-DEFAULT:       }
+
+// CPP-DECLTOP-LABEL: void emitc_switch_i32() {
+// CPP-DECLTOP:         int32_t v1;
+// CPP-DECLTOP:         float v2;
+// CPP-DECLTOP:         int32_t v3;
+// CPP-DECLTOP:         int32_t v4;
+// CPP-DECLTOP:         v1 = 1;
+// CPP-DECLTOP:         switch (v1) {
+// CPP-DECLTOP:         case 2: {
+// CPP-DECLTOP:           v3 = func_b();
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         case 5: {
+// CPP-DECLTOP:           v4 = func_a();
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         default: {
+// CPP-DECLTOP:           v2 = 4.200000000e+01f;
+// CPP-DECLTOP:           func2(v2);
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         return;
+// CPP-DECLTOP:       }
+func.func @emitc_switch_i32() {
+  %0 = "emitc.variable"(){value = 1 : i32} : () -> i32
+
+  emitc.switch %0 : i32
+  case 2 {
+    %1 = emitc.call_opaque "func_b" () : () -> i32
+    emitc.yield
+  }
+  case 5 {
+    %2 = emitc.call_opaque "func_a" () : () -> i32
+    emitc.yield
+  }
+  default {
+    %3 = "emitc.variable"(){value = 42.0 : f32} : () -> f32
+    emitc.call_opaque "func2" (%3) : (f32) -> ()
+    emitc.yield
+  }
+  return
+}
+
+// CPP-DEFAULT-LABEL: void emitc_switch_ui32() {
+// CPP-DEFAULT:         uint32_t v1 = 1;
+// CPP-DEFAULT:         switch (v1) {
+// CPP-DEFAULT:         case 2: {
+// CPP-DEFAULT:           int32_t v2 = func_b();
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         case 5: {
+// CPP-DEFAULT:           int32_t v3 = func_a();
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         default: {
+// CPP-DEFAULT:           float v4 = 4.200000000e+01f;
+// CPP-DEFAULT:           func2(v4);
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         return;
+// CPP-DEFAULT:       }
+
+// CPP-DECLTOP-LABEL: void emitc_switch_ui32() {
+// CPP-DECLTOP:         uint32_t v1;
+// CPP-DECLTOP:         float v2;
+// CPP-DECLTOP:         int32_t v3;
+// CPP-DECLTOP:         int32_t v4;
+// CPP-DECLTOP:         v1 = 1;
+// CPP-DECLTOP:         switch (v1) {
+// CPP-DECLTOP:         case 2: {
+// CPP-DECLTOP:           v3 = func_b();
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         case 5: {
+// CPP-DECLTOP:           v4 = func_a();
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         default: {
+// CPP-DECLTOP:           v2 = 4.200000000e+01f;
+// CPP-DECLTOP:           func2(v2);
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         return;
+// CPP-DECLTOP:       }
+func.func @emitc_switch_ui32() {
+  %0 = "emitc.variable"(){value = 1 : ui32} : () -> ui32
+
+  emitc.switch %0 : ui32
+  case 2 {
+    %1 = emitc.call_opaque "func_b" () : () -> i32
+    emitc.yield
+  }
+  case 5 {
+    %2 = emitc.call_opaque "func_a" () : () -> i32
+    emitc.yield
+  }
+  default {
+    %3 = "emitc.variable"(){value = 42.0 : f32} : () -> f32
+    emitc.call_opaque "func2" (%3) : (f32) -> ()
+    emitc.yield
+  }
+  return
+}
+
+// CPP-DEFAULT-LABEL: void emitc_switch_i64() {
+// CPP-DEFAULT:         int64_t v1 = 1;
+// CPP-DEFAULT:         switch (v1) {
+// CPP-DEFAULT:         case 2: {
+// CPP-DEFAULT:           int32_t v2 = func_b();
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         case 5: {
+// CPP-DEFAULT:           int32_t v3 = func_a();
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         default: {
+// CPP-DEFAULT:           float v4 = 4.200000000e+01f;
+// CPP-DEFAULT:           func2(v4);
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         return;
+// CPP-DEFAULT:       }
+
+// CPP-DECLTOP-LABEL: void emitc_switch_i64() {
+// CPP-DECLTOP:         int64_t v1;
+// CPP-DECLTOP:         float v2;
+// CPP-DECLTOP:         int32_t v3;
+// CPP-DECLTOP:         int32_t v4;
+// CPP-DECLTOP:         v1 = 1;
+// CPP-DECLTOP:         switch (v1) {
+// CPP-DECLTOP:         case 2: {
+// CPP-DECLTOP:           v3 = func_b();
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         case 5: {
+// CPP-DECLTOP:           v4 = func_a();
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         default: {
+// CPP-DECLTOP:           v2 = 4.200000000e+01f;
+// CPP-DECLTOP:           func2(v2);
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         return;
+// CPP-DECLTOP:       }
+func.func @emitc_switch_i64() {
+  %0 = "emitc.variable"(){value = 1 : i64} : () -> i64
+
+  emitc.switch %0 : i64
+  case 2 {
+    %1 = emitc.call_opaque "func_b" () : () -> i32
+    emitc.yield
+  }
+  case 5 {
+    %2 = emitc.call_opaque "func_a" () : () -> i32
+    emitc.yield
+  }
+  default {
+    %3 = "emitc.variable"(){value = 42.0 : f32} : () -> f32
+    emitc.call_opaque "func2" (%3) : (f32) -> ()
+    emitc.yield
+  }
+  return
+}
+
+// CPP-DEFAULT-LABEL: void emitc_switch_ui64() {
+// CPP-DEFAULT:         uint64_t v1 = 1;
+// CPP-DEFAULT:         switch (v1) {
+// CPP-DEFAULT:         case 2: {
+// CPP-DEFAULT:           int32_t v2 = func_b();
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         case 5: {
+// CPP-DEFAULT:           int32_t v3 = func_a();
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         default: {
+// CPP-DEFAULT:           float v4 = 4.200000000e+01f;
+// CPP-DEFAULT:           func2(v4);
+// CPP-DEFAULT:           break;
+// CPP-DEFAULT:         }
+// CPP-DEFAULT:         return;
+// CPP-DEFAULT:       }
+
+// CPP-DECLTOP-LABEL: void emitc_switch_ui64() {
+// CPP-DECLTOP:         uint64_t v1;
+// CPP-DECLTOP:         float v2;
+// CPP-DECLTOP:         int32_t v3;
+// CPP-DECLTOP:         int32_t v4;
+// CPP-DECLTOP:         v1 = 1;
+// CPP-DECLTOP:         switch (v1) {
+// CPP-DECLTOP:         case 2: {
+// CPP-DECLTOP:           v3 = func_b();
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         case 5: {
+// CPP-DECLTOP:           v4 = func_a();
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         default: {
+// CPP-DECLTOP:           v2 = 4.200000000e+01f;
+// CPP-DECLTOP:           func2(v2);
+// CPP-DECLTOP:           break;
+// CPP-DECLTOP:         }
+// CPP-DECLTOP:         return;
+// CPP-DECLTOP:       }
+func.func @emitc_switch_ui64() {
+  %0 = "emitc.variable"(){value = 1 : ui64} : () -> ui64
+
+  emitc.switch %0 : ui64
+  case 2 {
+    %1 = emitc.call_opaque "func_b" () : () -> i32
+    emitc.yield
+  }
+  case 5 {
+    %2 = emitc.call_opaque "func_a" () : () -> i32
+    emitc.yield
+  }
+  default {
+    %3 = "emitc.variable"(){value = 42.0 : f32} : () -> f32
+    emitc.call_opaque "func2" (%3) : (f32) -> ()
+    emitc.yield
+  }
+  return
+}

From ddda37a6c72f0d2fb315d53043ed5e7a39eddfe0 Mon Sep 17 00:00:00 2001
From: Rahul Joshi <rjoshi@nvidia.com>
Date: Fri, 16 Aug 2024 09:26:57 -0700
Subject: [PATCH 198/441] [TableGen] Refactor Intrinsic handling in TableGen
 (#103980)

CodeGenIntrinsic changes:
  - Use `const` Record pointers, and `StringRef` when possible.
  - Default initialize several fields with their definition instead of in
 the constructor.
- Simplify various string checks in the constructor using StringRef
starts_with()/ends_with() functions.
- Eliminate first argument to `setDefaultProperties` and use `TheDef`
class member instead.

IntrinsicEmitter changes:
  - Emit `namespace llvm::Intrinsic` instead of nested namespaces.
  - End generated comments with a .
  - Use range based for loops, and early continue within loops.
  - Emit `static constexpr` instead of `static const` for arrays.
- Change `compareFnAttributes` to use std::tie() to compare intrinsic
attributes and return a default value when all attributes are equal.

STLExtras:
  - Add std::replace wrapper which takes a range.
---
 llvm/include/llvm/ADT/STLExtras.h             |   7 +
 llvm/lib/Object/COFFImportFile.cpp            |   2 +-
 .../TableGen/Basic/CodeGenIntrinsics.cpp      |  72 +++-----
 llvm/utils/TableGen/Basic/CodeGenIntrinsics.h |  61 +++----
 .../utils/TableGen/Basic/SDNodeProperties.cpp |   4 +-
 llvm/utils/TableGen/Basic/SDNodeProperties.h  |   2 +-
 .../GlobalISel/GlobalISelMatchTable.cpp       |   4 +-
 .../Common/GlobalISel/PatternParser.cpp       |   2 +-
 llvm/utils/TableGen/IntrinsicEmitter.cpp      | 154 +++++++-----------
 .../utils/TableGen/SearchableTableEmitter.cpp |   5 +-
 polly/lib/Support/GICHelper.cpp               |  10 +-
 11 files changed, 142 insertions(+), 181 deletions(-)

diff --git a/llvm/include/llvm/ADT/STLExtras.h b/llvm/include/llvm/ADT/STLExtras.h
index 3f5ea3a5275f35b..a810a1c318c42cb 100644
--- a/llvm/include/llvm/ADT/STLExtras.h
+++ b/llvm/include/llvm/ADT/STLExtras.h
@@ -1843,6 +1843,13 @@ OutputIt replace_copy(R &&Range, OutputIt Out, const T &OldValue,
                            NewValue);
 }
 
+/// Provide wrappers to std::replace which take ranges instead of having to pass
+/// begin/end explicitly.
+template <typename R, typename T>
+void replace(R &&Range, const T &OldValue, const T &NewValue) {
+  return std::replace(adl_begin(Range), adl_end(Range), OldValue, NewValue);
+}
+
 /// Provide wrappers to std::move which take ranges instead of having to
 /// pass begin/end explicitly.
 template <typename R, typename OutputIt>
diff --git a/llvm/lib/Object/COFFImportFile.cpp b/llvm/lib/Object/COFFImportFile.cpp
index e67b02405a3a177..cc0a5da7e0d16a8 100644
--- a/llvm/lib/Object/COFFImportFile.cpp
+++ b/llvm/lib/Object/COFFImportFile.cpp
@@ -705,7 +705,7 @@ Error writeImportLibrary(StringRef ImportName, StringRef Path,
         Name = std::string(SymbolName);
       } else {
         Expected<std::string> ReplacedName =
-            replace(SymbolName, E.Name, E.ExtName);
+            object::replace(SymbolName, E.Name, E.ExtName);
         if (!ReplacedName)
           return ReplacedName.takeError();
         Name.swap(*ReplacedName);
diff --git a/llvm/utils/TableGen/Basic/CodeGenIntrinsics.cpp b/llvm/utils/TableGen/Basic/CodeGenIntrinsics.cpp
index 9153645e3d9a117..4bca904e9f38bdf 100644
--- a/llvm/utils/TableGen/Basic/CodeGenIntrinsics.cpp
+++ b/llvm/utils/TableGen/Basic/CodeGenIntrinsics.cpp
@@ -29,16 +29,16 @@ CodeGenIntrinsicTable::CodeGenIntrinsicTable(const RecordKeeper &RC) {
   std::vector<Record *> IntrProperties =
       RC.getAllDerivedDefinitions("IntrinsicProperty");
 
-  std::vector<Record *> DefaultProperties;
-  for (Record *Rec : IntrProperties)
+  std::vector<const Record *> DefaultProperties;
+  for (const Record *Rec : IntrProperties)
     if (Rec->getValueAsBit("IsDefault"))
       DefaultProperties.push_back(Rec);
 
   std::vector<Record *> Defs = RC.getAllDerivedDefinitions("Intrinsic");
   Intrinsics.reserve(Defs.size());
 
-  for (unsigned I = 0, e = Defs.size(); I != e; ++I)
-    Intrinsics.push_back(CodeGenIntrinsic(Defs[I], DefaultProperties));
+  for (const Record *Def : Defs)
+    Intrinsics.push_back(CodeGenIntrinsic(Def, DefaultProperties));
 
   llvm::sort(Intrinsics,
              [](const CodeGenIntrinsic &LHS, const CodeGenIntrinsic &RHS) {
@@ -54,52 +54,34 @@ CodeGenIntrinsicTable::CodeGenIntrinsicTable(const RecordKeeper &RC) {
   Targets.back().Count = Intrinsics.size() - Targets.back().Offset;
 }
 
-CodeGenIntrinsic::CodeGenIntrinsic(Record *R,
-                                   ArrayRef<Record *> DefaultProperties) {
-  TheDef = R;
-  std::string DefName = std::string(R->getName());
+CodeGenIntrinsic::CodeGenIntrinsic(const Record *R,
+                                   ArrayRef<const Record *> DefaultProperties)
+    : TheDef(R) {
+  StringRef DefName = TheDef->getName();
   ArrayRef<SMLoc> DefLoc = R->getLoc();
-  Properties = 0;
-  isOverloaded = false;
-  isCommutative = false;
-  canThrow = false;
-  isNoReturn = false;
-  isNoCallback = false;
-  isNoSync = false;
-  isNoFree = false;
-  isWillReturn = false;
-  isCold = false;
-  isNoDuplicate = false;
-  isNoMerge = false;
-  isConvergent = false;
-  isSpeculatable = false;
-  hasSideEffects = false;
-  isStrictFP = false;
 
-  if (DefName.size() <= 4 || DefName.substr(0, 4) != "int_")
+  if (!DefName.starts_with("int_"))
     PrintFatalError(DefLoc,
                     "Intrinsic '" + DefName + "' does not start with 'int_'!");
 
   EnumName = DefName.substr(4);
 
-  if (R->getValue(
-          "ClangBuiltinName")) // Ignore a missing ClangBuiltinName field.
-    ClangBuiltinName = std::string(R->getValueAsString("ClangBuiltinName"));
-  if (R->getValue("MSBuiltinName")) // Ignore a missing MSBuiltinName field.
-    MSBuiltinName = std::string(R->getValueAsString("MSBuiltinName"));
+  // Ignore a missing ClangBuiltinName field.
+  ClangBuiltinName =
+      R->getValueAsOptionalString("ClangBuiltinName").value_or("");
+  // Ignore a missing MSBuiltinName field.
+  MSBuiltinName = R->getValueAsOptionalString("MSBuiltinName").value_or("");
 
-  TargetPrefix = std::string(R->getValueAsString("TargetPrefix"));
-  Name = std::string(R->getValueAsString("LLVMName"));
+  TargetPrefix = R->getValueAsString("TargetPrefix");
+  Name = R->getValueAsString("LLVMName").str();
 
   if (Name == "") {
     // If an explicit name isn't specified, derive one from the DefName.
-    Name = "llvm.";
-
-    for (unsigned i = 0, e = EnumName.size(); i != e; ++i)
-      Name += (EnumName[i] == '_') ? '.' : EnumName[i];
+    Name = "llvm." + EnumName.str();
+    llvm::replace(Name, '_', '.');
   } else {
     // Verify it starts with "llvm.".
-    if (Name.size() <= 5 || Name.substr(0, 5) != "llvm.")
+    if (!StringRef(Name).starts_with("llvm."))
       PrintFatalError(DefLoc, "Intrinsic '" + DefName +
                                   "'s name does not start with 'llvm.'!");
   }
@@ -107,8 +89,8 @@ CodeGenIntrinsic::CodeGenIntrinsic(Record *R,
   // If TargetPrefix is specified, make sure that Name starts with
   // "llvm.<targetprefix>.".
   if (!TargetPrefix.empty()) {
-    if (Name.size() < 6 + TargetPrefix.size() ||
-        Name.substr(5, 1 + TargetPrefix.size()) != (TargetPrefix + "."))
+    StringRef Prefix = StringRef(Name).drop_front(5); // Drop llvm.
+    if (!Prefix.consume_front(TargetPrefix) || !Prefix.starts_with('.'))
       PrintFatalError(DefLoc, "Intrinsic '" + DefName +
                                   "' does not start with 'llvm." +
                                   TargetPrefix + ".'!");
@@ -129,7 +111,7 @@ CodeGenIntrinsic::CodeGenIntrinsic(Record *R,
   // Parse the intrinsic properties.
   ListInit *PropList = R->getValueAsListInit("IntrProperties");
   for (unsigned i = 0, e = PropList->size(); i != e; ++i) {
-    Record *Property = PropList->getElementAsRecord(i);
+    const Record *Property = PropList->getElementAsRecord(i);
     assert(Property->isSubClassOf("IntrinsicProperty") &&
            "Expected a property!");
 
@@ -137,7 +119,7 @@ CodeGenIntrinsic::CodeGenIntrinsic(Record *R,
   }
 
   // Set default properties to true.
-  setDefaultProperties(R, DefaultProperties);
+  setDefaultProperties(DefaultProperties);
 
   // Also record the SDPatternOperator Properties.
   Properties = parseSDPatternOperatorProperties(R);
@@ -148,16 +130,16 @@ CodeGenIntrinsic::CodeGenIntrinsic(Record *R,
 }
 
 void CodeGenIntrinsic::setDefaultProperties(
-    Record *R, ArrayRef<Record *> DefaultProperties) {
+    ArrayRef<const Record *> DefaultProperties) {
   // opt-out of using default attributes.
-  if (R->getValueAsBit("DisableDefaultAttributes"))
+  if (TheDef->getValueAsBit("DisableDefaultAttributes"))
     return;
 
-  for (Record *Rec : DefaultProperties)
+  for (const Record *Rec : DefaultProperties)
     setProperty(Rec);
 }
 
-void CodeGenIntrinsic::setProperty(Record *R) {
+void CodeGenIntrinsic::setProperty(const Record *R) {
   if (R->getName() == "IntrNoMem")
     ME = MemoryEffects::none();
   else if (R->getName() == "IntrReadMem") {
diff --git a/llvm/utils/TableGen/Basic/CodeGenIntrinsics.h b/llvm/utils/TableGen/Basic/CodeGenIntrinsics.h
index 417063a9490ef19..19e29af2fa85f36 100644
--- a/llvm/utils/TableGen/Basic/CodeGenIntrinsics.h
+++ b/llvm/utils/TableGen/Basic/CodeGenIntrinsics.h
@@ -26,12 +26,12 @@ class Record;
 class RecordKeeper;
 
 struct CodeGenIntrinsic {
-  Record *TheDef;       // The actual record defining this intrinsic.
+  const Record *TheDef; // The actual record defining this intrinsic.
   std::string Name;     // The name of the LLVM function "llvm.bswap.i32"
-  std::string EnumName; // The name of the enum "bswap_i32"
-  std::string ClangBuiltinName; // Name of the corresponding GCC builtin, or "".
-  std::string MSBuiltinName;    // Name of the corresponding MS builtin, or "".
-  std::string TargetPrefix;     // Target prefix, e.g. "ppc" for t-s intrinsics.
+  StringRef EnumName;   // The name of the enum "bswap_i32"
+  StringRef ClangBuiltinName; // Name of the corresponding GCC builtin, or "".
+  StringRef MSBuiltinName;    // Name of the corresponding MS builtin, or "".
+  StringRef TargetPrefix;     // Target prefix, e.g. "ppc" for t-s intrinsics.
 
   /// This structure holds the return values and parameter values of an
   /// intrinsic. If the number of return values is > 1, then the intrinsic
@@ -43,13 +43,13 @@ struct CodeGenIntrinsic {
     /// only populated when in the context of a target .td file. When building
     /// Intrinsics.td, this isn't available, because we don't know the target
     /// pointer size.
-    std::vector<Record *> RetTys;
+    std::vector<const Record *> RetTys;
 
     /// The MVT::SimpleValueType for each parameter type. Note that this list is
     /// only populated when in the context of a target .td file.  When building
     /// Intrinsics.td, this isn't available, because we don't know the target
     /// pointer size.
-    std::vector<Record *> ParamTys;
+    std::vector<const Record *> ParamTys;
   };
 
   IntrinsicSignature IS;
@@ -58,54 +58,54 @@ struct CodeGenIntrinsic {
   MemoryEffects ME = MemoryEffects::unknown();
 
   /// SDPatternOperator Properties applied to the intrinsic.
-  unsigned Properties;
+  unsigned Properties = 0;
 
   /// This is set to true if the intrinsic is overloaded by its argument
   /// types.
-  bool isOverloaded;
+  bool isOverloaded = false;
 
   /// True if the intrinsic is commutative.
-  bool isCommutative;
+  bool isCommutative = false;
 
   /// True if the intrinsic can throw.
-  bool canThrow;
+  bool canThrow = false;
 
   /// True if the intrinsic is marked as noduplicate.
-  bool isNoDuplicate;
+  bool isNoDuplicate = false;
 
   /// True if the intrinsic is marked as nomerge.
-  bool isNoMerge;
+  bool isNoMerge = false;
 
   /// True if the intrinsic is no-return.
-  bool isNoReturn;
+  bool isNoReturn = false;
 
   /// True if the intrinsic is no-callback.
-  bool isNoCallback;
+  bool isNoCallback = false;
 
   /// True if the intrinsic is no-sync.
-  bool isNoSync;
+  bool isNoSync = false;
 
   /// True if the intrinsic is no-free.
-  bool isNoFree;
+  bool isNoFree = false;
 
   /// True if the intrinsic is will-return.
-  bool isWillReturn;
+  bool isWillReturn = false;
 
   /// True if the intrinsic is cold.
-  bool isCold;
+  bool isCold = false;
 
   /// True if the intrinsic is marked as convergent.
-  bool isConvergent;
+  bool isConvergent = false;
 
   /// True if the intrinsic has side effects that aren't captured by any
   /// of the other flags.
-  bool hasSideEffects;
+  bool hasSideEffects = false;
 
   // True if the intrinsic is marked as speculatable.
-  bool isSpeculatable;
+  bool isSpeculatable = false;
 
   // True if the intrinsic is marked as strictfp.
-  bool isStrictFP;
+  bool isStrictFP = false;
 
   enum ArgAttrKind {
     NoCapture,
@@ -139,12 +139,12 @@ struct CodeGenIntrinsic {
 
   bool hasProperty(enum SDNP Prop) const { return Properties & (1 << Prop); }
 
-  /// Goes through all IntrProperties that have IsDefault
-  /// value set and sets the property.
-  void setDefaultProperties(Record *R, ArrayRef<Record *> DefaultProperties);
+  /// Goes through all IntrProperties that have IsDefault value set and sets
+  /// the property.
+  void setDefaultProperties(ArrayRef<const Record *> DefaultProperties);
 
-  /// Helper function to set property \p Name to true;
-  void setProperty(Record *R);
+  /// Helper function to set property \p Name to true.
+  void setProperty(const Record *R);
 
   /// Returns true if the parameter at \p ParamIdx is a pointer type. Returns
   /// false if the parameter is not a pointer, or \p ParamIdx is greater than
@@ -155,7 +155,8 @@ struct CodeGenIntrinsic {
 
   bool isParamImmArg(unsigned ParamIdx) const;
 
-  CodeGenIntrinsic(Record *R, ArrayRef<Record *> DefaultProperties);
+  CodeGenIntrinsic(const Record *R,
+                   ArrayRef<const Record *> DefaultProperties = {});
 };
 
 class CodeGenIntrinsicTable {
@@ -163,7 +164,7 @@ class CodeGenIntrinsicTable {
 
 public:
   struct TargetSet {
-    std::string Name;
+    StringRef Name;
     size_t Offset;
     size_t Count;
   };
diff --git a/llvm/utils/TableGen/Basic/SDNodeProperties.cpp b/llvm/utils/TableGen/Basic/SDNodeProperties.cpp
index 2aec41aac6257fb..46babbbc4194463 100644
--- a/llvm/utils/TableGen/Basic/SDNodeProperties.cpp
+++ b/llvm/utils/TableGen/Basic/SDNodeProperties.cpp
@@ -13,9 +13,9 @@
 
 using namespace llvm;
 
-unsigned llvm::parseSDPatternOperatorProperties(Record *R) {
+unsigned llvm::parseSDPatternOperatorProperties(const Record *R) {
   unsigned Properties = 0;
-  for (Record *Property : R->getValueAsListOfDefs("Properties")) {
+  for (const Record *Property : R->getValueAsListOfDefs("Properties")) {
     auto Offset = StringSwitch<unsigned>(Property->getName())
                       .Case("SDNPCommutative", SDNPCommutative)
                       .Case("SDNPAssociative", SDNPAssociative)
diff --git a/llvm/utils/TableGen/Basic/SDNodeProperties.h b/llvm/utils/TableGen/Basic/SDNodeProperties.h
index 10d0bb965fbb6c0..1fe4044edcea236 100644
--- a/llvm/utils/TableGen/Basic/SDNodeProperties.h
+++ b/llvm/utils/TableGen/Basic/SDNodeProperties.h
@@ -32,7 +32,7 @@ enum SDNP {
   SDNPWantParent
 };
 
-unsigned parseSDPatternOperatorProperties(Record *R);
+unsigned parseSDPatternOperatorProperties(const Record *R);
 
 } // namespace llvm
 
diff --git a/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp
index 9b9686c627b7522..139bf2d82180323 100644
--- a/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp
+++ b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp
@@ -1314,7 +1314,7 @@ void IntrinsicIDOperandMatcher::emitPredicateOpcodes(MatchTable &Table,
   Table << MatchTable::Opcode("GIM_CheckIntrinsicID")
         << MatchTable::Comment("MI") << MatchTable::ULEB128Value(InsnVarID)
         << MatchTable::Comment("Op") << MatchTable::ULEB128Value(OpIdx)
-        << MatchTable::NamedValue(2, "Intrinsic::" + II->EnumName)
+        << MatchTable::NamedValue(2, "Intrinsic::" + II->EnumName.str())
         << MatchTable::LineBreak;
 }
 
@@ -2103,7 +2103,7 @@ void IntrinsicIDRenderer::emitRenderOpcodes(MatchTable &Table,
                                             RuleMatcher &Rule) const {
   Table << MatchTable::Opcode("GIR_AddIntrinsicID") << MatchTable::Comment("MI")
         << MatchTable::ULEB128Value(InsnID)
-        << MatchTable::NamedValue(2, "Intrinsic::" + II->EnumName)
+        << MatchTable::NamedValue(2, "Intrinsic::" + II->EnumName.str())
         << MatchTable::LineBreak;
 }
 
diff --git a/llvm/utils/TableGen/Common/GlobalISel/PatternParser.cpp b/llvm/utils/TableGen/Common/GlobalISel/PatternParser.cpp
index 1d6c4c73a26405d..82fa3c8f3121f05 100644
--- a/llvm/utils/TableGen/Common/GlobalISel/PatternParser.cpp
+++ b/llvm/utils/TableGen/Common/GlobalISel/PatternParser.cpp
@@ -112,7 +112,7 @@ static const CodeGenIntrinsic *getCodeGenIntrinsic(Record *R) {
 
   auto &Ptr = AllIntrinsics[R];
   if (!Ptr)
-    Ptr = std::make_unique<CodeGenIntrinsic>(R, std::vector<Record *>());
+    Ptr = std::make_unique<CodeGenIntrinsic>(R);
   return Ptr.get();
 }
 
diff --git a/llvm/utils/TableGen/IntrinsicEmitter.cpp b/llvm/utils/TableGen/IntrinsicEmitter.cpp
index a7e99fa4c05068c..b1307153e9109bc 100644
--- a/llvm/utils/TableGen/IntrinsicEmitter.cpp
+++ b/llvm/utils/TableGen/IntrinsicEmitter.cpp
@@ -35,18 +35,18 @@
 #include <vector>
 using namespace llvm;
 
-cl::OptionCategory GenIntrinsicCat("Options for -gen-intrinsic-enums");
-cl::opt<std::string>
+static cl::OptionCategory GenIntrinsicCat("Options for -gen-intrinsic-enums");
+static cl::opt<std::string>
     IntrinsicPrefix("intrinsic-prefix",
                     cl::desc("Generate intrinsics with this target prefix"),
                     cl::value_desc("target prefix"), cl::cat(GenIntrinsicCat));
 
 namespace {
 class IntrinsicEmitter {
-  RecordKeeper &Records;
+  const RecordKeeper &Records;
 
 public:
-  IntrinsicEmitter(RecordKeeper &R) : Records(R) {}
+  IntrinsicEmitter(const RecordKeeper &R) : Records(R) {}
 
   void run(raw_ostream &OS, bool Enums);
 
@@ -123,7 +123,7 @@ void IntrinsicEmitter::EmitEnumInfo(const CodeGenIntrinsicTable &Ints,
     std::vector<std::string> KnownTargets;
     for (const auto &Target : Ints.Targets)
       if (!Target.Name.empty())
-        KnownTargets.push_back(Target.Name);
+        KnownTargets.push_back(Target.Name.str());
     PrintFatalError("tried to generate intrinsics for unknown target " +
                     IntrinsicPrefix +
                     "\nKnown targets are: " + join(KnownTargets, ", ") + "\n");
@@ -136,12 +136,11 @@ void IntrinsicEmitter::EmitEnumInfo(const CodeGenIntrinsicTable &Ints,
     std::string UpperPrefix = StringRef(IntrinsicPrefix).upper();
     OS << "#ifndef LLVM_IR_INTRINSIC_" << UpperPrefix << "_ENUMS_H\n";
     OS << "#define LLVM_IR_INTRINSIC_" << UpperPrefix << "_ENUMS_H\n\n";
-    OS << "namespace llvm {\n";
-    OS << "namespace Intrinsic {\n";
+    OS << "namespace llvm::Intrinsic {\n";
     OS << "enum " << UpperPrefix << "Intrinsics : unsigned {\n";
   }
 
-  OS << "// Enum values for intrinsics\n";
+  OS << "// Enum values for intrinsics.\n";
   for (unsigned i = Set->Offset, e = Set->Offset + Set->Count; i != e; ++i) {
     OS << "    " << Ints[i].EnumName;
 
@@ -162,8 +161,7 @@ void IntrinsicEmitter::EmitEnumInfo(const CodeGenIntrinsicTable &Ints,
     OS << "#endif\n\n";
   } else {
     OS << "}; // enum\n";
-    OS << "} // namespace Intrinsic\n";
-    OS << "} // namespace llvm\n\n";
+    OS << "} // namespace llvm::Intrinsic\n\n";
     OS << "#endif\n";
   }
 }
@@ -171,7 +169,7 @@ void IntrinsicEmitter::EmitEnumInfo(const CodeGenIntrinsicTable &Ints,
 void IntrinsicEmitter::EmitArgKind(raw_ostream &OS) {
   if (!IntrinsicPrefix.empty())
     return;
-  OS << "// llvm::Intrinsic::IITDescriptor::ArgKind\n";
+  OS << "// llvm::Intrinsic::IITDescriptor::ArgKind.\n";
   OS << "#ifdef GET_INTRINSIC_ARGKIND\n";
   if (auto RecArgKind = Records.getDef("ArgKind")) {
     for (auto &RV : RecArgKind->getValues())
@@ -205,7 +203,7 @@ void IntrinsicEmitter::EmitIITInfo(raw_ostream &OS) {
 
 void IntrinsicEmitter::EmitTargetInfo(const CodeGenIntrinsicTable &Ints,
                                       raw_ostream &OS) {
-  OS << "// Target mapping\n";
+  OS << "// Target mapping.\n";
   OS << "#ifdef GET_INTRINSIC_TARGET_DATA\n";
   OS << "struct IntrinsicTargetInfo {\n"
      << "  llvm::StringLiteral Name;\n"
@@ -222,19 +220,19 @@ void IntrinsicEmitter::EmitTargetInfo(const CodeGenIntrinsicTable &Ints,
 
 void IntrinsicEmitter::EmitIntrinsicToNameTable(
     const CodeGenIntrinsicTable &Ints, raw_ostream &OS) {
-  OS << "// Intrinsic ID to name table\n";
+  OS << "// Intrinsic ID to name table.\n";
   OS << "#ifdef GET_INTRINSIC_NAME_TABLE\n";
   OS << "  // Note that entry #0 is the invalid intrinsic!\n";
-  for (unsigned i = 0, e = Ints.size(); i != e; ++i)
-    OS << "  \"" << Ints[i].Name << "\",\n";
+  for (const auto &Int : Ints)
+    OS << "  \"" << Int.Name << "\",\n";
   OS << "#endif\n\n";
 }
 
 void IntrinsicEmitter::EmitIntrinsicToOverloadTable(
     const CodeGenIntrinsicTable &Ints, raw_ostream &OS) {
-  OS << "// Intrinsic ID to overload bitset\n";
+  OS << "// Intrinsic ID to overload bitset.\n";
   OS << "#ifdef GET_INTRINSIC_OVERLOAD_TABLE\n";
-  OS << "static const uint8_t OTable[] = {\n";
+  OS << "static constexpr uint8_t OTable[] = {\n";
   OS << "  0";
   for (unsigned i = 0, e = Ints.size(); i != e; ++i) {
     // Add one to the index so we emit a null bit for the invalid #0 intrinsic.
@@ -315,7 +313,7 @@ void IntrinsicEmitter::EmitGenerator(const CodeGenIntrinsicTable &Ints,
   OS << "// Global intrinsic function declaration type table.\n";
   OS << "#ifdef GET_INTRINSIC_GENERATOR_GLOBAL\n";
 
-  OS << "static const unsigned IIT_Table[] = {\n  ";
+  OS << "static constexpr unsigned IIT_Table[] = {\n  ";
 
   for (unsigned i = 0, e = FixedEncodings.size(); i != e; ++i) {
     if ((i & 7) == 7)
@@ -338,7 +336,7 @@ void IntrinsicEmitter::EmitGenerator(const CodeGenIntrinsicTable &Ints,
   OS << "0\n};\n\n";
 
   // Emit the shared table of register lists.
-  OS << "static const unsigned char IIT_LongEncodingTable[] = {\n";
+  OS << "static constexpr unsigned char IIT_LongEncodingTable[] = {\n";
   if (!LongEncodingTable.empty())
     LongEncodingTable.emit(OS, printIITEntry);
   OS << "  255\n};\n\n";
@@ -346,72 +344,44 @@ void IntrinsicEmitter::EmitGenerator(const CodeGenIntrinsicTable &Ints,
   OS << "#endif\n\n"; // End of GET_INTRINSIC_GENERATOR_GLOBAL
 }
 
-namespace {
-std::optional<bool> compareFnAttributes(const CodeGenIntrinsic *L,
-                                        const CodeGenIntrinsic *R) {
-  // Sort throwing intrinsics after non-throwing intrinsics.
-  if (L->canThrow != R->canThrow)
-    return R->canThrow;
-
-  if (L->isNoDuplicate != R->isNoDuplicate)
-    return R->isNoDuplicate;
-
-  if (L->isNoMerge != R->isNoMerge)
-    return R->isNoMerge;
-
-  if (L->isNoReturn != R->isNoReturn)
-    return R->isNoReturn;
-
-  if (L->isNoCallback != R->isNoCallback)
-    return R->isNoCallback;
-
-  if (L->isNoSync != R->isNoSync)
-    return R->isNoSync;
-
-  if (L->isNoFree != R->isNoFree)
-    return R->isNoFree;
-
-  if (L->isWillReturn != R->isWillReturn)
-    return R->isWillReturn;
-
-  if (L->isCold != R->isCold)
-    return R->isCold;
+static bool compareFnAttributes(const CodeGenIntrinsic *L,
+                                const CodeGenIntrinsic *R, bool Default) {
+  auto TieBoolAttributes = [](const CodeGenIntrinsic *I) -> auto {
+    // Sort throwing intrinsics after non-throwing intrinsics.
+    return std::tie(I->canThrow, I->isNoDuplicate, I->isNoMerge, I->isNoReturn,
+                    I->isNoCallback, I->isNoSync, I->isNoFree, I->isWillReturn,
+                    I->isCold, I->isConvergent, I->isSpeculatable,
+                    I->hasSideEffects, I->isStrictFP);
+  };
 
-  if (L->isConvergent != R->isConvergent)
-    return R->isConvergent;
+  auto TieL = TieBoolAttributes(L);
+  auto TieR = TieBoolAttributes(R);
 
-  if (L->isSpeculatable != R->isSpeculatable)
-    return R->isSpeculatable;
-
-  if (L->hasSideEffects != R->hasSideEffects)
-    return R->hasSideEffects;
-
-  if (L->isStrictFP != R->isStrictFP)
-    return R->isStrictFP;
+  if (TieL != TieR)
+    return TieL < TieR;
 
   // Try to order by readonly/readnone attribute.
   uint32_t LK = L->ME.toIntValue();
   uint32_t RK = R->ME.toIntValue();
   if (LK != RK)
-    return (LK > RK);
+    return LK > RK;
 
-  return std::nullopt;
+  return Default;
 }
 
+namespace {
 struct FnAttributeComparator {
   bool operator()(const CodeGenIntrinsic *L, const CodeGenIntrinsic *R) const {
-    return compareFnAttributes(L, R).value_or(false);
+    return compareFnAttributes(L, R, false);
   }
 };
 
 struct AttributeComparator {
   bool operator()(const CodeGenIntrinsic *L, const CodeGenIntrinsic *R) const {
-    if (std::optional<bool> Res = compareFnAttributes(L, R))
-      return *Res;
-
-    // Order by argument attributes.
+    // Order by argument attributes if function attributes are equal.
     // This is reliable because each side is already sorted internally.
-    return (L->ArgumentAttributes < R->ArgumentAttributes);
+    return compareFnAttributes(L, R,
+                               L->ArgumentAttributes < R->ArgumentAttributes);
   }
 };
 } // End anonymous namespace
@@ -559,7 +529,7 @@ void IntrinsicEmitter::EmitAttributes(const CodeGenIntrinsicTable &Ints,
 
   // Emit an array of AttributeList.  Most intrinsics will have at least one
   // entry, for the function itself (index ~1), which is usually nounwind.
-  OS << "  static const uint16_t IntrinsicsToAttributesMap[] = {\n";
+  OS << "  static constexpr uint16_t IntrinsicsToAttributesMap[] = {\n";
 
   for (unsigned i = 0, e = Ints.size(); i != e; ++i) {
     const CodeGenIntrinsic &intrinsic = Ints[i];
@@ -624,31 +594,34 @@ void IntrinsicEmitter::EmitAttributes(const CodeGenIntrinsicTable &Ints,
 
 void IntrinsicEmitter::EmitIntrinsicToBuiltinMap(
     const CodeGenIntrinsicTable &Ints, bool IsClang, raw_ostream &OS) {
-  StringRef CompilerName = (IsClang ? "Clang" : "MS");
-  StringRef UpperCompilerName = (IsClang ? "CLANG" : "MS");
-  typedef std::map<std::string, std::map<std::string, std::string>> BIMTy;
+  StringRef CompilerName = IsClang ? "Clang" : "MS";
+  StringRef UpperCompilerName = IsClang ? "CLANG" : "MS";
+  // map<TargetPrefix, map<BuiltinName, EnumName>>. Note that we iterate over
+  // both maps in the code below. For the inner map, entries need to be emitted
+  // in the sorted order of `BuiltinName` because we use std::lower_bound to
+  // search these entries. For the outer map, it doesn't need be be sorted, but
+  // we use a map to eliminate non-determinism in the emitted code.
+  typedef std::map<StringRef, std::map<StringRef, StringRef>> BIMTy;
   BIMTy BuiltinMap;
   StringToOffsetTable Table;
-  for (unsigned i = 0, e = Ints.size(); i != e; ++i) {
-    const std::string &BuiltinName =
-        IsClang ? Ints[i].ClangBuiltinName : Ints[i].MSBuiltinName;
-    if (!BuiltinName.empty()) {
-      // Get the map for this target prefix.
-      std::map<std::string, std::string> &BIM =
-          BuiltinMap[Ints[i].TargetPrefix];
-
-      if (!BIM.insert(std::pair(BuiltinName, Ints[i].EnumName)).second)
-        PrintFatalError(Ints[i].TheDef->getLoc(),
-                        "Intrinsic '" + Ints[i].TheDef->getName() +
-                            "': duplicate " + CompilerName + " builtin name!");
-      Table.GetOrAddStringOffset(BuiltinName);
-    }
+  for (const CodeGenIntrinsic &Int : Ints) {
+    StringRef BuiltinName = IsClang ? Int.ClangBuiltinName : Int.MSBuiltinName;
+    if (BuiltinName.empty())
+      continue;
+    // Get the map for this target prefix.
+    std::map<StringRef, StringRef> &BIM = BuiltinMap[Int.TargetPrefix];
+
+    if (!BIM.insert(std::pair(BuiltinName, Int.EnumName)).second)
+      PrintFatalError(Int.TheDef->getLoc(),
+                      "Intrinsic '" + Int.TheDef->getName() + "': duplicate " +
+                          CompilerName + " builtin name!");
+    Table.GetOrAddStringOffset(BuiltinName);
   }
 
   OS << "// Get the LLVM intrinsic that corresponds to a builtin.\n";
   OS << "// This is used by the C front-end.  The builtin name is passed\n";
   OS << "// in as BuiltinName, and a target prefix (e.g. 'ppc') is passed\n";
-  OS << "// in as TargetPrefix.  The result is assigned to 'IntrinsicID'.\n";
+  OS << "// in as TargetPrefix. The result is assigned to 'IntrinsicID'.\n";
   OS << "#ifdef GET_LLVM_INTRINSIC_FOR_" << UpperCompilerName << "_BUILTIN\n";
 
   OS << "Intrinsic::ID Intrinsic::getIntrinsicFor" << CompilerName
@@ -662,7 +635,7 @@ void IntrinsicEmitter::EmitIntrinsicToBuiltinMap(
     return;
   }
 
-  OS << "  static const char BuiltinNames[] = {\n";
+  OS << "  static constexpr char BuiltinNames[] = {\n";
   Table.EmitCharArray(OS);
   OS << "  };\n\n";
 
@@ -689,7 +662,7 @@ void IntrinsicEmitter::EmitIntrinsicToBuiltinMap(
     OS << "{\n";
 
     // Emit the comparisons for this target prefix.
-    OS << "    static const BuiltinEntry " << I.first << "Names[] = {\n";
+    OS << "    static constexpr BuiltinEntry " << I.first << "Names[] = {\n";
     for (const auto &P : I.second) {
       OS << "      {Intrinsic::" << P.second << ", "
          << Table.GetOrAddStringOffset(P.first) << "}, // " << P.first << "\n";
@@ -703,8 +676,7 @@ void IntrinsicEmitter::EmitIntrinsicToBuiltinMap(
     OS << "      return I->IntrinID;\n";
     OS << "  }\n";
   }
-  OS << "  return ";
-  OS << "Intrinsic::not_intrinsic;\n";
+  OS << "  return Intrinsic::not_intrinsic;\n";
   OS << "}\n";
   OS << "#endif\n\n";
 }
@@ -721,4 +693,4 @@ static void EmitIntrinsicImpl(RecordKeeper &RK, raw_ostream &OS) {
 }
 
 static TableGen::Emitter::Opt Y("gen-intrinsic-impl", EmitIntrinsicImpl,
-                                "Generate intrinsic information");
+                                "Generate intrinsic implementation code");
diff --git a/llvm/utils/TableGen/SearchableTableEmitter.cpp b/llvm/utils/TableGen/SearchableTableEmitter.cpp
index ac50c9ed2aeba96..59ae3bf3daedcb8 100644
--- a/llvm/utils/TableGen/SearchableTableEmitter.cpp
+++ b/llvm/utils/TableGen/SearchableTableEmitter.cpp
@@ -125,7 +125,7 @@ class SearchableTableEmitter {
     else if (BitInit *BI = dyn_cast<BitInit>(I))
       return BI->getValue() ? "true" : "false";
     else if (Field.IsIntrinsic)
-      return "Intrinsic::" + getIntrinsic(I).EnumName;
+      return "Intrinsic::" + getIntrinsic(I).EnumName.str();
     else if (Field.IsInstruction)
       return I->getAsString();
     else if (Field.Enum) {
@@ -148,8 +148,7 @@ class SearchableTableEmitter {
   CodeGenIntrinsic &getIntrinsic(Init *I) {
     std::unique_ptr<CodeGenIntrinsic> &Intr = Intrinsics[I];
     if (!Intr)
-      Intr = std::make_unique<CodeGenIntrinsic>(cast<DefInit>(I)->getDef(),
-                                                std::vector<Record *>());
+      Intr = std::make_unique<CodeGenIntrinsic>(cast<DefInit>(I)->getDef());
     return *Intr;
   }
 
diff --git a/polly/lib/Support/GICHelper.cpp b/polly/lib/Support/GICHelper.cpp
index 04d422cf9946231..027e0194732f4e6 100644
--- a/polly/lib/Support/GICHelper.cpp
+++ b/polly/lib/Support/GICHelper.cpp
@@ -143,11 +143,11 @@ static void replace(std::string &str, StringRef find, StringRef replace) {
 }
 
 static void makeIslCompatible(std::string &str) {
-  replace(str, ".", "_");
-  replace(str, "\"", "_");
-  replace(str, " ", "__");
-  replace(str, "=>", "TO");
-  replace(str, "+", "_");
+  llvm::replace(str, '.', '_');
+  llvm::replace(str, '\"', '_');
+  replace(str, StringRef(" "), StringRef("__"));
+  replace(str, StringRef("=>"), StringRef("TO"));
+  llvm::replace(str, '+', '_');
 }
 
 std::string polly::getIslCompatibleName(const std::string &Prefix,

From d8c2874172ebe70d34c08dea5fcf885283c93562 Mon Sep 17 00:00:00 2001
From: earnol <earnol@users.noreply.github.com>
Date: Fri, 16 Aug 2024 12:34:22 -0400
Subject: [PATCH 199/441] [test] Prevent generation of the bigendian code
 inside clang test CodeGen/bit-int-ubsan.c (#104607)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Add missing -triple x86_64-pc-linux-gnu line into RUN line, which should be here.

---------

Co-authored-by: Eänolituri Lómitaurë <vladislav.aranov@ericsson.com>
Co-authored-by: Aaron Ballman <aaron@aaronballman.com>
Co-authored-by: Paul Kirth <paulkirth@google.com>
Co-authored-by: Vitaly Buka <vitalybuka@gmail.com>
---
 clang/test/CodeGen/bit-int-ubsan.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/test/CodeGen/bit-int-ubsan.c b/clang/test/CodeGen/bit-int-ubsan.c
index 10c0d8a23f8f971..40e4fde1d1f417f 100644
--- a/clang/test/CodeGen/bit-int-ubsan.c
+++ b/clang/test/CodeGen/bit-int-ubsan.c
@@ -1,5 +1,5 @@
 // REQUIRES: x86-registered-target
-// RUN: %clang_cc1 -Wno-constant-conversion -Wno-array-bounds -Wno-division-by-zero -Wno-shift-negative-value -Wno-shift-count-negative -Wno-int-to-pointer-cast -fsanitize=array-bounds,enum,float-cast-overflow,integer-divide-by-zero,implicit-unsigned-integer-truncation,implicit-signed-integer-truncation,implicit-integer-sign-change,unsigned-integer-overflow,signed-integer-overflow,shift-base,shift-exponent -O0 -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -Wno-constant-conversion -Wno-array-bounds -Wno-division-by-zero -Wno-shift-negative-value -Wno-shift-count-negative -Wno-int-to-pointer-cast -fsanitize=array-bounds,enum,float-cast-overflow,integer-divide-by-zero,implicit-unsigned-integer-truncation,implicit-signed-integer-truncation,implicit-integer-sign-change,unsigned-integer-overflow,signed-integer-overflow,shift-base,shift-exponent -O0 -emit-llvm -o - %s | FileCheck %s
 
 // The runtime test checking the _BitInt ubsan feature is located in compiler-rt/test/ubsan/TestCases/Integer/bit-int.c
 

From 3f18a0a71cc29c502591a3d29a1845a011415f2a Mon Sep 17 00:00:00 2001
From: Mircea Trofin <mtrofin@google.com>
Date: Fri, 16 Aug 2024 09:45:29 -0700
Subject: [PATCH 200/441] [nfc] Improve testability of PGOInstrumentationGen
 (#104490)

Passing to the `PGOInstrumentationGen` pass whether it needs to produce contextual profiling instrumentation as a flag, in the process restructuring a bit the places that need to be aware of that (some were unnecessarily in `PGOInstrumentationUse`)
---
 .../Instrumentation/PGOInstrumentation.h      |   7 +-
 llvm/lib/Passes/PassBuilderPipelines.cpp      |   8 +-
 llvm/lib/Passes/PassRegistry.def              |   2 +
 .../Instrumentation/PGOInstrumentation.cpp    | 112 +++++++++++-------
 .../ctx-instrumentation-invalid-roots.ll      |   2 +-
 .../PGOProfile/ctx-instrumentation.ll         |   4 +-
 6 files changed, 83 insertions(+), 52 deletions(-)

diff --git a/llvm/include/llvm/Transforms/Instrumentation/PGOInstrumentation.h b/llvm/include/llvm/Transforms/Instrumentation/PGOInstrumentation.h
index 7199f27dbc991a8..1d0af87e965af5f 100644
--- a/llvm/include/llvm/Transforms/Instrumentation/PGOInstrumentation.h
+++ b/llvm/include/llvm/Transforms/Instrumentation/PGOInstrumentation.h
@@ -53,15 +53,18 @@ class PGOInstrumentationGenCreateVar
   bool ProfileSampling;
 };
 
+enum class PGOInstrumentationType { Invalid = 0, FDO, CSFDO, CTXPROF };
 /// The instrumentation (profile-instr-gen) pass for IR based PGO.
 class PGOInstrumentationGen : public PassInfoMixin<PGOInstrumentationGen> {
 public:
-  PGOInstrumentationGen(bool IsCS = false) : IsCS(IsCS) {}
+  PGOInstrumentationGen(
+      PGOInstrumentationType InstrumentationType = PGOInstrumentationType ::FDO)
+      : InstrumentationType(InstrumentationType) {}
   PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM);
 
 private:
   // If this is a context sensitive instrumentation.
-  bool IsCS;
+  const PGOInstrumentationType InstrumentationType;
 };
 
 /// The profile annotation (profile-instr-use) pass for IR based PGO.
diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp
index 0201e69f3e216a4..1184123c7710f05 100644
--- a/llvm/lib/Passes/PassBuilderPipelines.cpp
+++ b/llvm/lib/Passes/PassBuilderPipelines.cpp
@@ -844,7 +844,8 @@ void PassBuilder::addPGOInstrPasses(ModulePassManager &MPM,
   }
 
   // Perform PGO instrumentation.
-  MPM.addPass(PGOInstrumentationGen(IsCS));
+  MPM.addPass(PGOInstrumentationGen(IsCS ? PGOInstrumentationType::CSFDO
+                                         : PGOInstrumentationType::FDO));
 
   addPostPGOLoopRotation(MPM, Level);
   // Add the profile lowering pass.
@@ -879,7 +880,8 @@ void PassBuilder::addPGOInstrPassesForO0(
   }
 
   // Perform PGO instrumentation.
-  MPM.addPass(PGOInstrumentationGen(IsCS));
+  MPM.addPass(PGOInstrumentationGen(IsCS ? PGOInstrumentationType::CSFDO
+                                         : PGOInstrumentationType::FDO));
   // Add the profile lowering pass.
   InstrProfOptions Options;
   if (!ProfileFile.empty())
@@ -1193,7 +1195,7 @@ PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level,
                       PGOOpt->ProfileFile, PGOOpt->ProfileRemappingFile,
                       PGOOpt->FS);
   } else if (IsCtxProfGen || IsCtxProfUse) {
-    MPM.addPass(PGOInstrumentationGen(false));
+    MPM.addPass(PGOInstrumentationGen(PGOInstrumentationType::CTXPROF));
     // In pre-link, we just want the instrumented IR. We use the contextual
     // profile in the post-thinlink phase.
     // The instrumentation will be removed in post-thinlink after IPO.
diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def
index 442c972fc616fcf..a11fc3755494abc 100644
--- a/llvm/lib/Passes/PassRegistry.def
+++ b/llvm/lib/Passes/PassRegistry.def
@@ -56,6 +56,8 @@ MODULE_PASS("constmerge", ConstantMergePass())
 MODULE_PASS("coro-cleanup", CoroCleanupPass())
 MODULE_PASS("coro-early", CoroEarlyPass())
 MODULE_PASS("cross-dso-cfi", CrossDSOCFIPass())
+MODULE_PASS("ctx-instr-gen",
+            PGOInstrumentationGen(PGOInstrumentationType::CTXPROF))
 MODULE_PASS("deadargelim", DeadArgumentEliminationPass())
 MODULE_PASS("debugify", NewPMDebugifyPass())
 MODULE_PASS("dfsan", DataFlowSanitizerPass())
diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
index 41618194d12ed7c..b3644031c5a44bc 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
@@ -110,7 +110,6 @@
 #include "llvm/Transforms/Instrumentation.h"
 #include "llvm/Transforms/Instrumentation/BlockCoverageInference.h"
 #include "llvm/Transforms/Instrumentation/CFGMST.h"
-#include "llvm/Transforms/Instrumentation/PGOCtxProfLowering.h"
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
 #include "llvm/Transforms/Utils/MisExpect.h"
 #include "llvm/Transforms/Utils/ModuleUtils.h"
@@ -321,7 +320,6 @@ static cl::opt<unsigned> PGOFunctionCriticalEdgeThreshold(
              " greater than this threshold."));
 
 extern cl::opt<unsigned> MaxNumVTableAnnotations;
-extern cl::opt<std::string> UseCtxProfile;
 
 namespace llvm {
 // Command line option to turn on CFG dot dump after profile annotation.
@@ -339,21 +337,43 @@ extern cl::opt<bool> EnableVTableProfileUse;
 extern cl::opt<InstrProfCorrelator::ProfCorrelatorKind> ProfileCorrelate;
 } // namespace llvm
 
-bool shouldInstrumentForCtxProf() {
-  return PGOCtxProfLoweringPass::isCtxIRPGOInstrEnabled() ||
-         !UseCtxProfile.empty();
-}
-bool shouldInstrumentEntryBB() {
-  return PGOInstrumentEntry || shouldInstrumentForCtxProf();
-}
+namespace {
+class FunctionInstrumenter final {
+  Module &M;
+  Function &F;
+  TargetLibraryInfo &TLI;
+  std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers;
+  BranchProbabilityInfo *const BPI;
+  BlockFrequencyInfo *const BFI;
 
-// FIXME(mtrofin): re-enable this for ctx profiling, for non-indirect calls. Ctx
-// profiling implicitly captures indirect call cases, but not other values.
-// Supporting other values is relatively straight-forward - just another counter
-// range within the context.
-bool isValueProfilingDisabled() {
-  return DisableValueProfiling || shouldInstrumentForCtxProf();
-}
+  const PGOInstrumentationType InstrumentationType;
+
+  // FIXME(mtrofin): re-enable this for ctx profiling, for non-indirect calls.
+  // Ctx profiling implicitly captures indirect call cases, but not other
+  // values. Supporting other values is relatively straight-forward - just
+  // another counter range within the context.
+  bool isValueProfilingDisabled() const {
+    return DisableValueProfiling ||
+           InstrumentationType == PGOInstrumentationType::CTXPROF;
+  }
+
+  bool shouldInstrumentEntryBB() const {
+    return PGOInstrumentEntry ||
+           InstrumentationType == PGOInstrumentationType::CTXPROF;
+  }
+
+public:
+  FunctionInstrumenter(
+      Module &M, Function &F, TargetLibraryInfo &TLI,
+      std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers,
+      BranchProbabilityInfo *BPI = nullptr, BlockFrequencyInfo *BFI = nullptr,
+      PGOInstrumentationType InstrumentationType = PGOInstrumentationType::FDO)
+      : M(M), F(F), TLI(TLI), ComdatMembers(ComdatMembers), BPI(BPI), BFI(BFI),
+        InstrumentationType(InstrumentationType) {}
+
+  void instrument();
+};
+} // namespace
 
 // Return a string describing the branch condition that can be
 // used in static branch probability heuristics:
@@ -395,13 +415,16 @@ static const char *ValueProfKindDescr[] = {
 
 // Create a COMDAT variable INSTR_PROF_RAW_VERSION_VAR to make the runtime
 // aware this is an ir_level profile so it can set the version flag.
-static GlobalVariable *createIRLevelProfileFlagVar(Module &M, bool IsCS) {
+static GlobalVariable *
+createIRLevelProfileFlagVar(Module &M,
+                            PGOInstrumentationType InstrumentationType) {
   const StringRef VarName(INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR));
   Type *IntTy64 = Type::getInt64Ty(M.getContext());
   uint64_t ProfileVersion = (INSTR_PROF_RAW_VERSION | VARIANT_MASK_IR_PROF);
-  if (IsCS)
+  if (InstrumentationType == PGOInstrumentationType::CSFDO)
     ProfileVersion |= VARIANT_MASK_CSIR_PROF;
-  if (shouldInstrumentEntryBB())
+  if (PGOInstrumentEntry ||
+      InstrumentationType == PGOInstrumentationType::CTXPROF)
     ProfileVersion |= VARIANT_MASK_INSTR_ENTRY;
   if (DebugInfoCorrelate || ProfileCorrelate == InstrProfCorrelator::DEBUG_INFO)
     ProfileVersion |= VARIANT_MASK_DBG_CORRELATE;
@@ -871,11 +894,7 @@ populateEHOperandBundle(VPCandidateInfo &Cand,
 
 // Visit all edge and instrument the edges not in MST, and do value profiling.
 // Critical edges will be split.
-static void instrumentOneFunc(
-    Function &F, Module *M, TargetLibraryInfo &TLI, BranchProbabilityInfo *BPI,
-    BlockFrequencyInfo *BFI,
-    std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers,
-    bool IsCS) {
+void FunctionInstrumenter::instrument() {
   if (!PGOBlockCoverage) {
     // Split indirectbr critical edges here before computing the MST rather than
     // later in getInstrBB() to avoid invalidating it.
@@ -883,19 +902,20 @@ static void instrumentOneFunc(
   }
 
   FuncPGOInstrumentation<PGOEdge, PGOBBInfo> FuncInfo(
-      F, TLI, ComdatMembers, true, BPI, BFI, IsCS, shouldInstrumentEntryBB(),
-      PGOBlockCoverage);
+      F, TLI, ComdatMembers, true, BPI, BFI,
+      InstrumentationType == PGOInstrumentationType::CSFDO,
+      shouldInstrumentEntryBB(), PGOBlockCoverage);
 
   auto Name = FuncInfo.FuncNameVar;
-  auto CFGHash = ConstantInt::get(Type::getInt64Ty(M->getContext()),
-                                  FuncInfo.FunctionHash);
+  auto CFGHash =
+      ConstantInt::get(Type::getInt64Ty(M.getContext()), FuncInfo.FunctionHash);
   if (PGOFunctionEntryCoverage) {
     auto &EntryBB = F.getEntryBlock();
     IRBuilder<> Builder(&EntryBB, EntryBB.getFirstInsertionPt());
     // llvm.instrprof.cover(i8* <name>, i64 <hash>, i32 <num-counters>,
     //                      i32 <index>)
     Builder.CreateCall(
-        Intrinsic::getDeclaration(M, Intrinsic::instrprof_cover),
+        Intrinsic::getDeclaration(&M, Intrinsic::instrprof_cover),
         {Name, CFGHash, Builder.getInt32(1), Builder.getInt32(0)});
     return;
   }
@@ -905,9 +925,9 @@ static void instrumentOneFunc(
   unsigned NumCounters =
       InstrumentBBs.size() + FuncInfo.SIVisitor.getNumOfSelectInsts();
 
-  if (shouldInstrumentForCtxProf()) {
+  if (InstrumentationType == PGOInstrumentationType::CTXPROF) {
     auto *CSIntrinsic =
-        Intrinsic::getDeclaration(M, Intrinsic::instrprof_callsite);
+        Intrinsic::getDeclaration(&M, Intrinsic::instrprof_callsite);
     // We want to count the instrumentable callsites, then instrument them. This
     // is because the llvm.instrprof.callsite intrinsic has an argument (like
     // the other instrprof intrinsics) capturing the total number of
@@ -950,7 +970,7 @@ static void instrumentOneFunc(
     // llvm.instrprof.timestamp(i8* <name>, i64 <hash>, i32 <num-counters>,
     //                          i32 <index>)
     Builder.CreateCall(
-        Intrinsic::getDeclaration(M, Intrinsic::instrprof_timestamp),
+        Intrinsic::getDeclaration(&M, Intrinsic::instrprof_timestamp),
         {Name, CFGHash, Builder.getInt32(NumCounters), Builder.getInt32(I)});
     I += PGOBlockCoverage ? 8 : 1;
   }
@@ -962,9 +982,9 @@ static void instrumentOneFunc(
     // llvm.instrprof.increment(i8* <name>, i64 <hash>, i32 <num-counters>,
     //                          i32 <index>)
     Builder.CreateCall(
-        Intrinsic::getDeclaration(M, PGOBlockCoverage
-                                         ? Intrinsic::instrprof_cover
-                                         : Intrinsic::instrprof_increment),
+        Intrinsic::getDeclaration(&M, PGOBlockCoverage
+                                          ? Intrinsic::instrprof_cover
+                                          : Intrinsic::instrprof_increment),
         {Name, CFGHash, Builder.getInt32(NumCounters), Builder.getInt32(I++)});
   }
 
@@ -1011,7 +1031,7 @@ static void instrumentOneFunc(
       SmallVector<OperandBundleDef, 1> OpBundles;
       populateEHOperandBundle(Cand, BlockColors, OpBundles);
       Builder.CreateCall(
-          Intrinsic::getDeclaration(M, Intrinsic::instrprof_value_profile),
+          Intrinsic::getDeclaration(&M, Intrinsic::instrprof_value_profile),
           {FuncInfo.FuncNameVar, Builder.getInt64(FuncInfo.FunctionHash),
            ToProfile, Builder.getInt32(Kind), Builder.getInt32(SiteIndex++)},
           OpBundles);
@@ -1746,7 +1766,7 @@ static uint32_t getMaxNumAnnotations(InstrProfValueKind ValueProfKind) {
 
 // Traverse all valuesites and annotate the instructions for all value kind.
 void PGOUseFunc::annotateValueSites() {
-  if (isValueProfilingDisabled())
+  if (DisableValueProfiling)
     return;
 
   // Create the PGOFuncName meta data.
@@ -1861,11 +1881,12 @@ static bool skipPGOGen(const Function &F) {
 static bool InstrumentAllFunctions(
     Module &M, function_ref<TargetLibraryInfo &(Function &)> LookupTLI,
     function_ref<BranchProbabilityInfo *(Function &)> LookupBPI,
-    function_ref<BlockFrequencyInfo *(Function &)> LookupBFI, bool IsCS) {
+    function_ref<BlockFrequencyInfo *(Function &)> LookupBFI,
+    PGOInstrumentationType InstrumentationType) {
   // For the context-sensitve instrumentation, we should have a separated pass
   // (before LTO/ThinLTO linking) to create these variables.
-  if (!IsCS && !shouldInstrumentForCtxProf())
-    createIRLevelProfileFlagVar(M, /*IsCS=*/false);
+  if (InstrumentationType == PGOInstrumentationType::FDO)
+    createIRLevelProfileFlagVar(M, InstrumentationType);
 
   Triple TT(M.getTargetTriple());
   LLVMContext &Ctx = M.getContext();
@@ -1884,7 +1905,9 @@ static bool InstrumentAllFunctions(
     auto &TLI = LookupTLI(F);
     auto *BPI = LookupBPI(F);
     auto *BFI = LookupBFI(F);
-    instrumentOneFunc(F, &M, TLI, BPI, BFI, ComdatMembers, IsCS);
+    FunctionInstrumenter FI(M, F, TLI, ComdatMembers, BPI, BFI,
+                            InstrumentationType);
+    FI.instrument();
   }
   return true;
 }
@@ -1894,7 +1917,8 @@ PGOInstrumentationGenCreateVar::run(Module &M, ModuleAnalysisManager &MAM) {
   createProfileFileNameVar(M, CSInstrName);
   // The variable in a comdat may be discarded by LTO. Ensure the declaration
   // will be retained.
-  appendToCompilerUsed(M, createIRLevelProfileFlagVar(M, /*IsCS=*/true));
+  appendToCompilerUsed(
+      M, createIRLevelProfileFlagVar(M, PGOInstrumentationType::CSFDO));
   if (ProfileSampling)
     createProfileSamplingVar(M);
   PreservedAnalyses PA;
@@ -1916,7 +1940,8 @@ PreservedAnalyses PGOInstrumentationGen::run(Module &M,
     return &FAM.getResult<BlockFrequencyAnalysis>(F);
   };
 
-  if (!InstrumentAllFunctions(M, LookupTLI, LookupBPI, LookupBFI, IsCS))
+  if (!InstrumentAllFunctions(M, LookupTLI, LookupBPI, LookupBFI,
+                              InstrumentationType))
     return PreservedAnalyses::all();
 
   return PreservedAnalyses::none();
@@ -2115,7 +2140,6 @@ static bool annotateAllFunctions(
   bool InstrumentFuncEntry = PGOReader->instrEntryBBEnabled();
   if (PGOInstrumentEntry.getNumOccurrences() > 0)
     InstrumentFuncEntry = PGOInstrumentEntry;
-  InstrumentFuncEntry |= shouldInstrumentForCtxProf();
 
   bool HasSingleByteCoverage = PGOReader->hasSingleByteCoverage();
   for (auto &F : M) {
diff --git a/llvm/test/Transforms/PGOProfile/ctx-instrumentation-invalid-roots.ll b/llvm/test/Transforms/PGOProfile/ctx-instrumentation-invalid-roots.ll
index 99c7762a67dfbd6..454780153b82368 100644
--- a/llvm/test/Transforms/PGOProfile/ctx-instrumentation-invalid-roots.ll
+++ b/llvm/test/Transforms/PGOProfile/ctx-instrumentation-invalid-roots.ll
@@ -1,4 +1,4 @@
-; RUN: not opt -passes=pgo-instr-gen,ctx-instr-lower -profile-context-root=good \
+; RUN: not opt -passes=ctx-instr-gen,ctx-instr-lower -profile-context-root=good \
 ; RUN:   -profile-context-root=bad \
 ; RUN:   -S < %s 2>&1 | FileCheck %s
 
diff --git a/llvm/test/Transforms/PGOProfile/ctx-instrumentation.ll b/llvm/test/Transforms/PGOProfile/ctx-instrumentation.ll
index a70f94e1521f0de..df4e467567c46ef 100644
--- a/llvm/test/Transforms/PGOProfile/ctx-instrumentation.ll
+++ b/llvm/test/Transforms/PGOProfile/ctx-instrumentation.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals all --version 4
-; RUN: opt -passes=pgo-instr-gen -profile-context-root=an_entrypoint \
+; RUN: opt -passes=ctx-instr-gen -profile-context-root=an_entrypoint \
 ; RUN:   -S < %s | FileCheck --check-prefix=INSTRUMENT %s
-; RUN: opt -passes=pgo-instr-gen,assign-guid,ctx-instr-lower -profile-context-root=an_entrypoint \
+; RUN: opt -passes=ctx-instr-gen,assign-guid,ctx-instr-lower -profile-context-root=an_entrypoint \
 ; RUN:   -profile-context-root=another_entrypoint_no_callees \
 ; RUN:   -S < %s | FileCheck --check-prefix=LOWERING %s
 

From aba3476111fbc06f652453c13c99ca029646df47 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke@igalia.com>
Date: Sat, 17 Aug 2024 00:49:27 +0800
Subject: [PATCH 201/441] [RISCV] Move vmv.v.v peephole from SelectionDAG to
 RISCVVectorPeephole (#100367)

This is split off from #71764, and moves only the vmv.v.v part of
performCombineVMergeAndVOps to work on MachineInstrs.

In retrospect trying to handle PseudoVMV_V_V and PseudoVMERGE_VVM in the
same function makes the code quite hard to read, so this just does it in
a separate peephole.

This turns out to be simpler since for PseudoVMV_V_V we don't need to
convert the Src instruction to a masked variant, and we don't need to
create a fake all ones mask.
---
 llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp   |  84 ++---------
 llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp | 141 +++++++++++++++++-
 2 files changed, 154 insertions(+), 71 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
index 0f7b90f5f5f1349..d95cb3cbe3e4aba 100644
--- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
@@ -3708,32 +3708,6 @@ static bool IsVMerge(SDNode *N) {
   return RISCV::getRVVMCOpcode(N->getMachineOpcode()) == RISCV::VMERGE_VVM;
 }
 
-static bool IsVMv(SDNode *N) {
-  return RISCV::getRVVMCOpcode(N->getMachineOpcode()) == RISCV::VMV_V_V;
-}
-
-static unsigned GetVMSetForLMul(RISCVII::VLMUL LMUL) {
-  switch (LMUL) {
-  case RISCVII::LMUL_F8:
-    return RISCV::PseudoVMSET_M_B1;
-  case RISCVII::LMUL_F4:
-    return RISCV::PseudoVMSET_M_B2;
-  case RISCVII::LMUL_F2:
-    return RISCV::PseudoVMSET_M_B4;
-  case RISCVII::LMUL_1:
-    return RISCV::PseudoVMSET_M_B8;
-  case RISCVII::LMUL_2:
-    return RISCV::PseudoVMSET_M_B16;
-  case RISCVII::LMUL_4:
-    return RISCV::PseudoVMSET_M_B32;
-  case RISCVII::LMUL_8:
-    return RISCV::PseudoVMSET_M_B64;
-  case RISCVII::LMUL_RESERVED:
-    llvm_unreachable("Unexpected LMUL");
-  }
-  llvm_unreachable("Unknown VLMUL enum");
-}
-
 // Try to fold away VMERGE_VVM instructions into their true operands:
 //
 // %true = PseudoVADD_VV ...
@@ -3748,35 +3722,22 @@ static unsigned GetVMSetForLMul(RISCVII::VLMUL LMUL) {
 // If %true is masked, then we can use its mask instead of vmerge's if vmerge's
 // mask is all ones.
 //
-// We can also fold a VMV_V_V into its true operand, since it is equivalent to a
-// VMERGE_VVM with an all ones mask.
-//
 // The resulting VL is the minimum of the two VLs.
 //
 // The resulting policy is the effective policy the vmerge would have had,
 // i.e. whether or not it's passthru operand was implicit-def.
 bool RISCVDAGToDAGISel::performCombineVMergeAndVOps(SDNode *N) {
   SDValue Passthru, False, True, VL, Mask, Glue;
-  // A vmv.v.v is equivalent to a vmerge with an all-ones mask.
-  if (IsVMv(N)) {
-    Passthru = N->getOperand(0);
-    False = N->getOperand(0);
-    True = N->getOperand(1);
-    VL = N->getOperand(2);
-    // A vmv.v.v won't have a Mask or Glue, instead we'll construct an all-ones
-    // mask later below.
-  } else {
-    assert(IsVMerge(N));
-    Passthru = N->getOperand(0);
-    False = N->getOperand(1);
-    True = N->getOperand(2);
-    Mask = N->getOperand(3);
-    VL = N->getOperand(4);
-    // We always have a glue node for the mask at v0.
-    Glue = N->getOperand(N->getNumOperands() - 1);
-  }
-  assert(!Mask || cast<RegisterSDNode>(Mask)->getReg() == RISCV::V0);
-  assert(!Glue || Glue.getValueType() == MVT::Glue);
+  assert(IsVMerge(N));
+  Passthru = N->getOperand(0);
+  False = N->getOperand(1);
+  True = N->getOperand(2);
+  Mask = N->getOperand(3);
+  VL = N->getOperand(4);
+  // We always have a glue node for the mask at v0.
+  Glue = N->getOperand(N->getNumOperands() - 1);
+  assert(cast<RegisterSDNode>(Mask)->getReg() == RISCV::V0);
+  assert(Glue.getValueType() == MVT::Glue);
 
   // If the EEW of True is different from vmerge's SEW, then we can't fold.
   if (True.getSimpleValueType() != N->getSimpleValueType(0))
@@ -3824,7 +3785,7 @@ bool RISCVDAGToDAGISel::performCombineVMergeAndVOps(SDNode *N) {
 
   // If True is masked then the vmerge must have either the same mask or an all
   // 1s mask, since we're going to keep the mask from True.
-  if (IsMasked && Mask) {
+  if (IsMasked) {
     // FIXME: Support mask agnostic True instruction which would have an
     // undef passthru operand.
     SDValue TrueMask =
@@ -3854,11 +3815,9 @@ bool RISCVDAGToDAGISel::performCombineVMergeAndVOps(SDNode *N) {
     SmallVector<const SDNode *, 4> LoopWorklist;
     SmallPtrSet<const SDNode *, 16> Visited;
     LoopWorklist.push_back(False.getNode());
-    if (Mask)
-      LoopWorklist.push_back(Mask.getNode());
+    LoopWorklist.push_back(Mask.getNode());
     LoopWorklist.push_back(VL.getNode());
-    if (Glue)
-      LoopWorklist.push_back(Glue.getNode());
+    LoopWorklist.push_back(Glue.getNode());
     if (SDNode::hasPredecessorHelper(True.getNode(), Visited, LoopWorklist))
       return false;
   }
@@ -3919,21 +3878,6 @@ bool RISCVDAGToDAGISel::performCombineVMergeAndVOps(SDNode *N) {
     Glue = True->getOperand(True->getNumOperands() - 1);
     assert(Glue.getValueType() == MVT::Glue);
   }
-  // If we end up using the vmerge mask the vmerge is actually a vmv.v.v, create
-  // an all-ones mask to use.
-  else if (IsVMv(N)) {
-    unsigned TSFlags = TII->get(N->getMachineOpcode()).TSFlags;
-    unsigned VMSetOpc = GetVMSetForLMul(RISCVII::getLMul(TSFlags));
-    ElementCount EC = N->getValueType(0).getVectorElementCount();
-    MVT MaskVT = MVT::getVectorVT(MVT::i1, EC);
-
-    SDValue AllOnesMask =
-        SDValue(CurDAG->getMachineNode(VMSetOpc, DL, MaskVT, VL, SEW), 0);
-    SDValue MaskCopy = CurDAG->getCopyToReg(CurDAG->getEntryNode(), DL,
-                                            RISCV::V0, AllOnesMask, SDValue());
-    Mask = CurDAG->getRegister(RISCV::V0, MaskVT);
-    Glue = MaskCopy.getValue(1);
-  }
 
   unsigned MaskedOpc = Info->MaskedPseudo;
 #ifndef NDEBUG
@@ -4012,7 +3956,7 @@ bool RISCVDAGToDAGISel::doPeepholeMergeVVMFold() {
     if (N->use_empty() || !N->isMachineOpcode())
       continue;
 
-    if (IsVMerge(N) || IsVMv(N))
+    if (IsVMerge(N))
       MadeChange |= performCombineVMergeAndVOps(N);
   }
   return MadeChange;
diff --git a/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp b/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
index 979677ee92332dd..2abed1ac984e35b 100644
--- a/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
+++ b/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
@@ -65,6 +65,7 @@ class RISCVVectorPeephole : public MachineFunctionPass {
   bool convertToWholeRegister(MachineInstr &MI) const;
   bool convertToUnmasked(MachineInstr &MI) const;
   bool convertVMergeToVMv(MachineInstr &MI) const;
+  bool foldVMV_V_V(MachineInstr &MI);
 
   bool isAllOnesMask(const MachineInstr *MaskDef) const;
   std::optional<unsigned> getConstant(const MachineOperand &VL) const;
@@ -324,6 +325,143 @@ bool RISCVVectorPeephole::convertToUnmasked(MachineInstr &MI) const {
   return true;
 }
 
+/// Given two VL operands, returns the one known to be the smallest or nullptr
+/// if unknown.
+static const MachineOperand *getKnownMinVL(const MachineOperand *LHS,
+                                           const MachineOperand *RHS) {
+  if (LHS->isReg() && RHS->isReg() && LHS->getReg().isVirtual() &&
+      LHS->getReg() == RHS->getReg())
+    return LHS;
+  if (LHS->isImm() && LHS->getImm() == RISCV::VLMaxSentinel)
+    return RHS;
+  if (RHS->isImm() && RHS->getImm() == RISCV::VLMaxSentinel)
+    return LHS;
+  if (!LHS->isImm() || !RHS->isImm())
+    return nullptr;
+  return LHS->getImm() <= RHS->getImm() ? LHS : RHS;
+}
+
+/// Check if it's safe to move From down to To, checking that no physical
+/// registers are clobbered.
+static bool isSafeToMove(const MachineInstr &From, const MachineInstr &To) {
+  assert(From.getParent() == To.getParent() && !From.hasImplicitDef());
+  SmallVector<Register> PhysUses;
+  for (const MachineOperand &MO : From.all_uses())
+    if (MO.getReg().isPhysical())
+      PhysUses.push_back(MO.getReg());
+  bool SawStore = false;
+  for (auto II = From.getIterator(); II != To.getIterator(); II++) {
+    for (Register PhysReg : PhysUses)
+      if (II->definesRegister(PhysReg, nullptr))
+        return false;
+    if (II->mayStore()) {
+      SawStore = true;
+      break;
+    }
+  }
+  return From.isSafeToMove(SawStore);
+}
+
+static unsigned getSEWLMULRatio(const MachineInstr &MI) {
+  RISCVII::VLMUL LMUL = RISCVII::getLMul(MI.getDesc().TSFlags);
+  unsigned Log2SEW = MI.getOperand(RISCVII::getSEWOpNum(MI.getDesc())).getImm();
+  return RISCVVType::getSEWLMULRatio(1 << Log2SEW, LMUL);
+}
+
+/// If a PseudoVMV_V_V is the only user of its input, fold its passthru and VL
+/// into it.
+///
+/// %x = PseudoVADD_V_V_M1 %passthru, %a, %b, %vl1, sew, policy
+/// %y = PseudoVMV_V_V_M1 %passthru, %x, %vl2, sew, policy
+///
+/// ->
+///
+/// %y = PseudoVADD_V_V_M1 %passthru, %a, %b, min(vl1, vl2), sew, policy
+bool RISCVVectorPeephole::foldVMV_V_V(MachineInstr &MI) {
+  if (RISCV::getRVVMCOpcode(MI.getOpcode()) != RISCV::VMV_V_V)
+    return false;
+
+  MachineOperand &Passthru = MI.getOperand(1);
+
+  if (!MRI->hasOneUse(MI.getOperand(2).getReg()))
+    return false;
+
+  MachineInstr *Src = MRI->getVRegDef(MI.getOperand(2).getReg());
+  if (!Src || Src->hasUnmodeledSideEffects() ||
+      Src->getParent() != MI.getParent() || Src->getNumDefs() != 1 ||
+      !RISCVII::isFirstDefTiedToFirstUse(Src->getDesc()) ||
+      !RISCVII::hasVLOp(Src->getDesc().TSFlags) ||
+      !RISCVII::hasVecPolicyOp(Src->getDesc().TSFlags))
+    return false;
+
+  // Src needs to have the same VLMAX as MI
+  if (getSEWLMULRatio(MI) != getSEWLMULRatio(*Src))
+    return false;
+
+  // Src needs to have the same passthru as VMV_V_V
+  MachineOperand &SrcPassthru = Src->getOperand(1);
+  if (SrcPassthru.getReg() != RISCV::NoRegister &&
+      SrcPassthru.getReg() != Passthru.getReg())
+    return false;
+
+  // Because Src and MI have the same passthru, we can use either AVL as long as
+  // it's the smaller of the two.
+  //
+  // (src pt, ..., vl=5)       x x x x x|. . .
+  // (vmv.v.v pt, src, vl=3)   x x x|. . . . .
+  // ->
+  // (src pt, ..., vl=3)       x x x|. . . . .
+  //
+  // (src pt, ..., vl=3)       x x x|. . . . .
+  // (vmv.v.v pt, src, vl=6)   x x x . . .|. .
+  // ->
+  // (src pt, ..., vl=3)       x x x|. . . . .
+  MachineOperand &SrcVL = Src->getOperand(RISCVII::getVLOpNum(Src->getDesc()));
+  const MachineOperand *MinVL = getKnownMinVL(&MI.getOperand(3), &SrcVL);
+  if (!MinVL)
+    return false;
+
+  bool VLChanged = !MinVL->isIdenticalTo(SrcVL);
+  bool ActiveElementsAffectResult = RISCVII::activeElementsAffectResult(
+      TII->get(RISCV::getRVVMCOpcode(Src->getOpcode())).TSFlags);
+
+  if (VLChanged && (ActiveElementsAffectResult || Src->mayRaiseFPException()))
+    return false;
+
+  // If Src ends up using MI's passthru/VL, move it so it can access it.
+  // TODO: We don't need to do this if they already dominate Src.
+  if (!SrcVL.isIdenticalTo(*MinVL) || !SrcPassthru.isIdenticalTo(Passthru)) {
+    if (!isSafeToMove(*Src, MI))
+      return false;
+    Src->moveBefore(&MI);
+  }
+
+  if (SrcPassthru.getReg() != Passthru.getReg()) {
+    SrcPassthru.setReg(Passthru.getReg());
+    // If Src is masked then its passthru needs to be in VRNoV0.
+    if (Passthru.getReg() != RISCV::NoRegister)
+      MRI->constrainRegClass(Passthru.getReg(),
+                             TII->getRegClass(Src->getDesc(), 1, TRI,
+                                              *Src->getParent()->getParent()));
+  }
+
+  if (MinVL->isImm())
+    SrcVL.ChangeToImmediate(MinVL->getImm());
+  else if (MinVL->isReg())
+    SrcVL.ChangeToRegister(MinVL->getReg(), false);
+
+  // Use a conservative tu,mu policy, RISCVInsertVSETVLI will relax it if
+  // passthru is undef.
+  Src->getOperand(RISCVII::getVecPolicyOpNum(Src->getDesc()))
+      .setImm(RISCVII::TAIL_UNDISTURBED_MASK_UNDISTURBED);
+
+  MRI->replaceRegWith(MI.getOperand(0).getReg(), Src->getOperand(0).getReg());
+  MI.eraseFromParent();
+  V0Defs.erase(&MI);
+
+  return true;
+}
+
 bool RISCVVectorPeephole::runOnMachineFunction(MachineFunction &MF) {
   if (skipFunction(MF.getFunction()))
     return false;
@@ -358,11 +496,12 @@ bool RISCVVectorPeephole::runOnMachineFunction(MachineFunction &MF) {
   }
 
   for (MachineBasicBlock &MBB : MF) {
-    for (MachineInstr &MI : MBB) {
+    for (MachineInstr &MI : make_early_inc_range(MBB)) {
       Changed |= convertToVLMAX(MI);
       Changed |= convertToUnmasked(MI);
       Changed |= convertToWholeRegister(MI);
       Changed |= convertVMergeToVMv(MI);
+      Changed |= foldVMV_V_V(MI);
     }
   }
 

From ab5102d61d1fc0b3b5f2dbaa268ffa5fef295cad Mon Sep 17 00:00:00 2001
From: vporpo <vporpodas@google.com>
Date: Fri, 16 Aug 2024 09:51:05 -0700
Subject: [PATCH 202/441] [SandboxIR] Implement AtomicRMWInst (#104529)

This patch implements sandboxir::AtomicRMWInst mirroring
llvm::AtomicRMWInst.
---
 llvm/include/llvm/SandboxIR/SandboxIR.h       |  76 ++++++++
 .../llvm/SandboxIR/SandboxIRValues.def        |   1 +
 llvm/lib/SandboxIR/SandboxIR.cpp              |  78 +++++++++
 llvm/unittests/SandboxIR/SandboxIRTest.cpp    | 163 ++++++++++++++++++
 llvm/unittests/SandboxIR/TrackerTest.cpp      |  55 ++++++
 5 files changed, 373 insertions(+)

diff --git a/llvm/include/llvm/SandboxIR/SandboxIR.h b/llvm/include/llvm/SandboxIR/SandboxIR.h
index 423dad854a91cbf..75c9d5017054b4a 100644
--- a/llvm/include/llvm/SandboxIR/SandboxIR.h
+++ b/llvm/include/llvm/SandboxIR/SandboxIR.h
@@ -132,6 +132,7 @@ class BitCastInst;
 class AllocaInst;
 class UnaryOperator;
 class BinaryOperator;
+class AtomicRMWInst;
 class AtomicCmpXchgInst;
 
 /// Iterator for the `Use` edges of a User's operands.
@@ -253,6 +254,7 @@ class Value {
   friend class GetElementPtrInst;  // For getting `Val`.
   friend class UnaryOperator;      // For getting `Val`.
   friend class BinaryOperator;     // For getting `Val`.
+  friend class AtomicRMWInst;      // For getting `Val`.
   friend class AtomicCmpXchgInst;  // For getting `Val`.
   friend class AllocaInst;         // For getting `Val`.
   friend class CastInst;           // For getting `Val`.
@@ -636,6 +638,7 @@ class Instruction : public sandboxir::User {
   friend class GetElementPtrInst;  // For getTopmostLLVMInstruction().
   friend class UnaryOperator;      // For getTopmostLLVMInstruction().
   friend class BinaryOperator;     // For getTopmostLLVMInstruction().
+  friend class AtomicRMWInst;      // For getTopmostLLVMInstruction().
   friend class AtomicCmpXchgInst;  // For getTopmostLLVMInstruction().
   friend class AllocaInst;         // For getTopmostLLVMInstruction().
   friend class CastInst;           // For getTopmostLLVMInstruction().
@@ -1559,6 +1562,77 @@ class BinaryOperator : public SingleLLVMInstructionImpl<llvm::BinaryOperator> {
   void swapOperands() { swapOperandsInternal(0, 1); }
 };
 
+class AtomicRMWInst : public SingleLLVMInstructionImpl<llvm::AtomicRMWInst> {
+  AtomicRMWInst(llvm::AtomicRMWInst *Atomic, Context &Ctx)
+      : SingleLLVMInstructionImpl(ClassID::AtomicRMW,
+                                  Instruction::Opcode::AtomicRMW, Atomic, Ctx) {
+  }
+  friend class Context; // For constructor.
+
+public:
+  using BinOp = llvm::AtomicRMWInst::BinOp;
+  BinOp getOperation() const {
+    return cast<llvm::AtomicRMWInst>(Val)->getOperation();
+  }
+  static StringRef getOperationName(BinOp Op) {
+    return llvm::AtomicRMWInst::getOperationName(Op);
+  }
+  static bool isFPOperation(BinOp Op) {
+    return llvm::AtomicRMWInst::isFPOperation(Op);
+  }
+  void setOperation(BinOp Op) {
+    cast<llvm::AtomicRMWInst>(Val)->setOperation(Op);
+  }
+  Align getAlign() const { return cast<llvm::AtomicRMWInst>(Val)->getAlign(); }
+  void setAlignment(Align Align);
+  bool isVolatile() const {
+    return cast<llvm::AtomicRMWInst>(Val)->isVolatile();
+  }
+  void setVolatile(bool V);
+  AtomicOrdering getOrdering() const {
+    return cast<llvm::AtomicRMWInst>(Val)->getOrdering();
+  }
+  void setOrdering(AtomicOrdering Ordering);
+  SyncScope::ID getSyncScopeID() const {
+    return cast<llvm::AtomicRMWInst>(Val)->getSyncScopeID();
+  }
+  void setSyncScopeID(SyncScope::ID SSID);
+  Value *getPointerOperand();
+  const Value *getPointerOperand() const {
+    return const_cast<AtomicRMWInst *>(this)->getPointerOperand();
+  }
+  Value *getValOperand();
+  const Value *getValOperand() const {
+    return const_cast<AtomicRMWInst *>(this)->getValOperand();
+  }
+  unsigned getPointerAddressSpace() const {
+    return cast<llvm::AtomicRMWInst>(Val)->getPointerAddressSpace();
+  }
+  bool isFloatingPointOperation() const {
+    return cast<llvm::AtomicRMWInst>(Val)->isFloatingPointOperation();
+  }
+  static bool classof(const Value *From) {
+    return From->getSubclassID() == ClassID::AtomicRMW;
+  }
+
+  static AtomicRMWInst *create(BinOp Op, Value *Ptr, Value *Val,
+                               MaybeAlign Align, AtomicOrdering Ordering,
+                               BBIterator WhereIt, BasicBlock *WhereBB,
+                               Context &Ctx,
+                               SyncScope::ID SSID = SyncScope::System,
+                               const Twine &Name = "");
+  static AtomicRMWInst *create(BinOp Op, Value *Ptr, Value *Val,
+                               MaybeAlign Align, AtomicOrdering Ordering,
+                               Instruction *InsertBefore, Context &Ctx,
+                               SyncScope::ID SSID = SyncScope::System,
+                               const Twine &Name = "");
+  static AtomicRMWInst *create(BinOp Op, Value *Ptr, Value *Val,
+                               MaybeAlign Align, AtomicOrdering Ordering,
+                               BasicBlock *InsertAtEnd, Context &Ctx,
+                               SyncScope::ID SSID = SyncScope::System,
+                               const Twine &Name = "");
+};
+
 class AtomicCmpXchgInst
     : public SingleLLVMInstructionImpl<llvm::AtomicCmpXchgInst> {
   AtomicCmpXchgInst(llvm::AtomicCmpXchgInst *Atomic, Context &Ctx)
@@ -2007,6 +2081,8 @@ class Context {
   friend UnaryOperator; // For createUnaryOperator()
   BinaryOperator *createBinaryOperator(llvm::BinaryOperator *I);
   friend BinaryOperator; // For createBinaryOperator()
+  AtomicRMWInst *createAtomicRMWInst(llvm::AtomicRMWInst *I);
+  friend AtomicRMWInst; // For createAtomicRMWInst()
   AtomicCmpXchgInst *createAtomicCmpXchgInst(llvm::AtomicCmpXchgInst *I);
   friend AtomicCmpXchgInst; // For createAtomicCmpXchgInst()
   AllocaInst *createAllocaInst(llvm::AllocaInst *I);
diff --git a/llvm/include/llvm/SandboxIR/SandboxIRValues.def b/llvm/include/llvm/SandboxIR/SandboxIRValues.def
index 7332316c85026c3..81a916be21e4a3b 100644
--- a/llvm/include/llvm/SandboxIR/SandboxIRValues.def
+++ b/llvm/include/llvm/SandboxIR/SandboxIRValues.def
@@ -68,6 +68,7 @@ DEF_INSTR(BinaryOperator, OPCODES(\
                          OP(Or)   \
                          OP(Xor)  \
                          ),                 BinaryOperator)
+DEF_INSTR(AtomicRMW,     OP(AtomicRMW),     AtomicRMWInst)
 DEF_INSTR(AtomicCmpXchg, OP(AtomicCmpXchg), AtomicCmpXchgInst)
 DEF_INSTR(Alloca,         OP(Alloca),         AllocaInst)
 DEF_INSTR(Cast,   OPCODES(\
diff --git a/llvm/lib/SandboxIR/SandboxIR.cpp b/llvm/lib/SandboxIR/SandboxIR.cpp
index 67262bdd0dea998..96c5af807d1001a 100644
--- a/llvm/lib/SandboxIR/SandboxIR.cpp
+++ b/llvm/lib/SandboxIR/SandboxIR.cpp
@@ -1385,6 +1385,74 @@ Value *BinaryOperator::createWithCopiedFlags(Instruction::Opcode Op, Value *LHS,
                                InsertAtEnd, Ctx, Name);
 }
 
+void AtomicRMWInst::setAlignment(Align Align) {
+  Ctx.getTracker()
+      .emplaceIfTracking<GenericSetter<&AtomicRMWInst::getAlign,
+                                       &AtomicRMWInst::setAlignment>>(this);
+  cast<llvm::AtomicRMWInst>(Val)->setAlignment(Align);
+}
+
+void AtomicRMWInst::setVolatile(bool V) {
+  Ctx.getTracker()
+      .emplaceIfTracking<GenericSetter<&AtomicRMWInst::isVolatile,
+                                       &AtomicRMWInst::setVolatile>>(this);
+  cast<llvm::AtomicRMWInst>(Val)->setVolatile(V);
+}
+
+void AtomicRMWInst::setOrdering(AtomicOrdering Ordering) {
+  Ctx.getTracker()
+      .emplaceIfTracking<GenericSetter<&AtomicRMWInst::getOrdering,
+                                       &AtomicRMWInst::setOrdering>>(this);
+  cast<llvm::AtomicRMWInst>(Val)->setOrdering(Ordering);
+}
+
+void AtomicRMWInst::setSyncScopeID(SyncScope::ID SSID) {
+  Ctx.getTracker()
+      .emplaceIfTracking<GenericSetter<&AtomicRMWInst::getSyncScopeID,
+                                       &AtomicRMWInst::setSyncScopeID>>(this);
+  cast<llvm::AtomicRMWInst>(Val)->setSyncScopeID(SSID);
+}
+
+Value *AtomicRMWInst::getPointerOperand() {
+  return Ctx.getValue(cast<llvm::AtomicRMWInst>(Val)->getPointerOperand());
+}
+
+Value *AtomicRMWInst::getValOperand() {
+  return Ctx.getValue(cast<llvm::AtomicRMWInst>(Val)->getValOperand());
+}
+
+AtomicRMWInst *AtomicRMWInst::create(BinOp Op, Value *Ptr, Value *Val,
+                                     MaybeAlign Align, AtomicOrdering Ordering,
+                                     BBIterator WhereIt, BasicBlock *WhereBB,
+                                     Context &Ctx, SyncScope::ID SSID,
+                                     const Twine &Name) {
+  auto &Builder = Ctx.getLLVMIRBuilder();
+  if (WhereIt == WhereBB->end())
+    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
+  else
+    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
+  auto *LLVMAtomicRMW =
+      Builder.CreateAtomicRMW(Op, Ptr->Val, Val->Val, Align, Ordering, SSID);
+  LLVMAtomicRMW->setName(Name);
+  return Ctx.createAtomicRMWInst(LLVMAtomicRMW);
+}
+
+AtomicRMWInst *AtomicRMWInst::create(BinOp Op, Value *Ptr, Value *Val,
+                                     MaybeAlign Align, AtomicOrdering Ordering,
+                                     Instruction *InsertBefore, Context &Ctx,
+                                     SyncScope::ID SSID, const Twine &Name) {
+  return create(Op, Ptr, Val, Align, Ordering, InsertBefore->getIterator(),
+                InsertBefore->getParent(), Ctx, SSID, Name);
+}
+
+AtomicRMWInst *AtomicRMWInst::create(BinOp Op, Value *Ptr, Value *Val,
+                                     MaybeAlign Align, AtomicOrdering Ordering,
+                                     BasicBlock *InsertAtEnd, Context &Ctx,
+                                     SyncScope::ID SSID, const Twine &Name) {
+  return create(Op, Ptr, Val, Align, Ordering, InsertAtEnd->end(), InsertAtEnd,
+                Ctx, SSID, Name);
+}
+
 void AtomicCmpXchgInst::setSyncScopeID(SyncScope::ID SSID) {
   Ctx.getTracker()
       .emplaceIfTracking<GenericSetter<&AtomicCmpXchgInst::getSyncScopeID,
@@ -1823,6 +1891,12 @@ Value *Context::getOrCreateValueInternal(llvm::Value *LLVMV, llvm::User *U) {
         new BinaryOperator(LLVMBinaryOperator, *this));
     return It->second.get();
   }
+  case llvm::Instruction::AtomicRMW: {
+    auto *LLVMAtomicRMW = cast<llvm::AtomicRMWInst>(LLVMV);
+    It->second =
+        std::unique_ptr<AtomicRMWInst>(new AtomicRMWInst(LLVMAtomicRMW, *this));
+    return It->second.get();
+  }
   case llvm::Instruction::AtomicCmpXchg: {
     auto *LLVMAtomicCmpXchg = cast<llvm::AtomicCmpXchgInst>(LLVMV);
     It->second = std::unique_ptr<AtomicCmpXchgInst>(
@@ -1954,6 +2028,10 @@ BinaryOperator *Context::createBinaryOperator(llvm::BinaryOperator *I) {
   auto NewPtr = std::unique_ptr<BinaryOperator>(new BinaryOperator(I, *this));
   return cast<BinaryOperator>(registerValue(std::move(NewPtr)));
 }
+AtomicRMWInst *Context::createAtomicRMWInst(llvm::AtomicRMWInst *I) {
+  auto NewPtr = std::unique_ptr<AtomicRMWInst>(new AtomicRMWInst(I, *this));
+  return cast<AtomicRMWInst>(registerValue(std::move(NewPtr)));
+}
 AtomicCmpXchgInst *
 Context::createAtomicCmpXchgInst(llvm::AtomicCmpXchgInst *I) {
   auto NewPtr =
diff --git a/llvm/unittests/SandboxIR/SandboxIRTest.cpp b/llvm/unittests/SandboxIR/SandboxIRTest.cpp
index 3df335985aa705a..e4563ab8f07ba6a 100644
--- a/llvm/unittests/SandboxIR/SandboxIRTest.cpp
+++ b/llvm/unittests/SandboxIR/SandboxIRTest.cpp
@@ -1932,6 +1932,169 @@ define void @foo(i8 %arg0, i8 %arg1, float %farg0, float %farg1) {
   }
 }
 
+TEST_F(SandboxIRTest, AtomicRMWInst) {
+  parseIR(C, R"IR(
+define void @foo(ptr %ptr, i8 %arg) {
+  %atomicrmw = atomicrmw add ptr %ptr, i8 %arg acquire, align 128
+  ret void
+}
+)IR");
+  llvm::Function &LLVMF = *M->getFunction("foo");
+  llvm::BasicBlock *LLVMBB = &*LLVMF.begin();
+  auto LLVMIt = LLVMBB->begin();
+  auto *LLVMRMW = cast<llvm::AtomicRMWInst>(&*LLVMIt++);
+
+  sandboxir::Context Ctx(C);
+  sandboxir::Function *F = Ctx.createFunction(&LLVMF);
+  auto *Ptr = F->getArg(0);
+  auto *Arg = F->getArg(1);
+  auto *BB = &*F->begin();
+  auto It = BB->begin();
+  auto *RMW = cast<sandboxir::AtomicRMWInst>(&*It++);
+  auto *Ret = cast<sandboxir::ReturnInst>(&*It++);
+
+  // Check getOperationName().
+  EXPECT_EQ(
+      sandboxir::AtomicRMWInst::getOperationName(
+          sandboxir::AtomicRMWInst::BinOp::Add),
+      llvm::AtomicRMWInst::getOperationName(llvm::AtomicRMWInst::BinOp::Add));
+  // Check isFPOperation().
+  EXPECT_EQ(
+      sandboxir::AtomicRMWInst::isFPOperation(
+          sandboxir::AtomicRMWInst::BinOp::Add),
+      llvm::AtomicRMWInst::isFPOperation(llvm::AtomicRMWInst::BinOp::Add));
+  EXPECT_FALSE(sandboxir::AtomicRMWInst::isFPOperation(
+      sandboxir::AtomicRMWInst::BinOp::Add));
+  EXPECT_TRUE(sandboxir::AtomicRMWInst::isFPOperation(
+      sandboxir::AtomicRMWInst::BinOp::FAdd));
+  // Check setOperation(), getOperation().
+  EXPECT_EQ(RMW->getOperation(), LLVMRMW->getOperation());
+  RMW->setOperation(sandboxir::AtomicRMWInst::BinOp::Sub);
+  EXPECT_EQ(RMW->getOperation(), sandboxir::AtomicRMWInst::BinOp::Sub);
+  RMW->setOperation(sandboxir::AtomicRMWInst::BinOp::Add);
+  // Check getAlign().
+  EXPECT_EQ(RMW->getAlign(), LLVMRMW->getAlign());
+  auto OrigAlign = RMW->getAlign();
+  Align NewAlign(256);
+  EXPECT_NE(NewAlign, OrigAlign);
+  RMW->setAlignment(NewAlign);
+  EXPECT_EQ(RMW->getAlign(), NewAlign);
+  RMW->setAlignment(OrigAlign);
+  EXPECT_EQ(RMW->getAlign(), OrigAlign);
+  // Check isVolatile(), setVolatile().
+  EXPECT_EQ(RMW->isVolatile(), LLVMRMW->isVolatile());
+  bool OrigV = RMW->isVolatile();
+  bool NewV = true;
+  EXPECT_NE(NewV, OrigV);
+  RMW->setVolatile(NewV);
+  EXPECT_EQ(RMW->isVolatile(), NewV);
+  RMW->setVolatile(OrigV);
+  EXPECT_EQ(RMW->isVolatile(), OrigV);
+  // Check getOrdering(), setOrdering().
+  EXPECT_EQ(RMW->getOrdering(), LLVMRMW->getOrdering());
+  auto OldOrdering = RMW->getOrdering();
+  auto NewOrdering = AtomicOrdering::Monotonic;
+  EXPECT_NE(NewOrdering, OldOrdering);
+  RMW->setOrdering(NewOrdering);
+  EXPECT_EQ(RMW->getOrdering(), NewOrdering);
+  RMW->setOrdering(OldOrdering);
+  EXPECT_EQ(RMW->getOrdering(), OldOrdering);
+  // Check getSyncScopeID(), setSyncScopeID().
+  EXPECT_EQ(RMW->getSyncScopeID(), LLVMRMW->getSyncScopeID());
+  auto OrigSSID = RMW->getSyncScopeID();
+  SyncScope::ID NewSSID = SyncScope::SingleThread;
+  EXPECT_NE(NewSSID, OrigSSID);
+  RMW->setSyncScopeID(NewSSID);
+  EXPECT_EQ(RMW->getSyncScopeID(), NewSSID);
+  RMW->setSyncScopeID(OrigSSID);
+  EXPECT_EQ(RMW->getSyncScopeID(), OrigSSID);
+  // Check getPointerOperand().
+  EXPECT_EQ(RMW->getPointerOperand(),
+            Ctx.getValue(LLVMRMW->getPointerOperand()));
+  // Check getValOperand().
+  EXPECT_EQ(RMW->getValOperand(), Ctx.getValue(LLVMRMW->getValOperand()));
+  // Check getPointerAddressSpace().
+  EXPECT_EQ(RMW->getPointerAddressSpace(), LLVMRMW->getPointerAddressSpace());
+  // Check isFloatingPointOperation().
+  EXPECT_EQ(RMW->isFloatingPointOperation(),
+            LLVMRMW->isFloatingPointOperation());
+
+  Align Align(1024);
+  auto Ordering = AtomicOrdering::Acquire;
+  auto SSID = SyncScope::System;
+  {
+    // Check create() WhereIt, WhereBB.
+    auto *NewI =
+        cast<sandboxir::AtomicRMWInst>(sandboxir::AtomicRMWInst::create(
+            sandboxir::AtomicRMWInst::BinOp::Sub, Ptr, Arg, Align, Ordering,
+            /*WhereIt=*/Ret->getIterator(),
+            /*WhereBB=*/Ret->getParent(), Ctx, SSID, "NewAtomicRMW1"));
+    // Check getOpcode().
+    EXPECT_EQ(NewI->getOpcode(), sandboxir::Instruction::Opcode::AtomicRMW);
+    // Check getAlign().
+    EXPECT_EQ(NewI->getAlign(), Align);
+    // Check getSuccessOrdering().
+    EXPECT_EQ(NewI->getOrdering(), Ordering);
+    // Check instr position.
+    EXPECT_EQ(NewI->getNextNode(), Ret);
+    // Check getPointerOperand().
+    EXPECT_EQ(NewI->getPointerOperand(), Ptr);
+    // Check getValOperand().
+    EXPECT_EQ(NewI->getValOperand(), Arg);
+#ifndef NDEBUG
+    // Check getName().
+    EXPECT_EQ(NewI->getName(), "NewAtomicRMW1");
+#endif // NDEBUG
+  }
+  {
+    // Check create() InsertBefore.
+    auto *NewI =
+        cast<sandboxir::AtomicRMWInst>(sandboxir::AtomicRMWInst::create(
+            sandboxir::AtomicRMWInst::BinOp::Sub, Ptr, Arg, Align, Ordering,
+            /*InsertBefore=*/Ret, Ctx, SSID, "NewAtomicRMW2"));
+    // Check getOpcode().
+    EXPECT_EQ(NewI->getOpcode(), sandboxir::Instruction::Opcode::AtomicRMW);
+    // Check getAlign().
+    EXPECT_EQ(NewI->getAlign(), Align);
+    // Check getSuccessOrdering().
+    EXPECT_EQ(NewI->getOrdering(), Ordering);
+    // Check instr position.
+    EXPECT_EQ(NewI->getNextNode(), Ret);
+    // Check getPointerOperand().
+    EXPECT_EQ(NewI->getPointerOperand(), Ptr);
+    // Check getValOperand().
+    EXPECT_EQ(NewI->getValOperand(), Arg);
+#ifndef NDEBUG
+    // Check getName().
+    EXPECT_EQ(NewI->getName(), "NewAtomicRMW2");
+#endif // NDEBUG
+  }
+  {
+    // Check create() InsertAtEnd.
+    auto *NewI =
+        cast<sandboxir::AtomicRMWInst>(sandboxir::AtomicRMWInst::create(
+            sandboxir::AtomicRMWInst::BinOp::Sub, Ptr, Arg, Align, Ordering,
+            /*InsertAtEnd=*/BB, Ctx, SSID, "NewAtomicRMW3"));
+    // Check getOpcode().
+    EXPECT_EQ(NewI->getOpcode(), sandboxir::Instruction::Opcode::AtomicRMW);
+    // Check getAlign().
+    EXPECT_EQ(NewI->getAlign(), Align);
+    // Check getSuccessOrdering().
+    EXPECT_EQ(NewI->getOrdering(), Ordering);
+    // Check instr position.
+    EXPECT_EQ(NewI->getParent(), BB);
+    EXPECT_EQ(NewI->getNextNode(), nullptr);
+    // Check getPointerOperand().
+    EXPECT_EQ(NewI->getPointerOperand(), Ptr);
+    // Check getValOperand().
+    EXPECT_EQ(NewI->getValOperand(), Arg);
+#ifndef NDEBUG
+    // Check getName().
+    EXPECT_EQ(NewI->getName(), "NewAtomicRMW3");
+#endif // NDEBUG
+  }
+}
+
 TEST_F(SandboxIRTest, AtomicCmpXchgInst) {
   parseIR(C, R"IR(
 define void @foo(ptr %ptr, i8 %cmp, i8 %new) {
diff --git a/llvm/unittests/SandboxIR/TrackerTest.cpp b/llvm/unittests/SandboxIR/TrackerTest.cpp
index c1f23c95cbfaed0..380c90e7f0f65f6 100644
--- a/llvm/unittests/SandboxIR/TrackerTest.cpp
+++ b/llvm/unittests/SandboxIR/TrackerTest.cpp
@@ -644,6 +644,61 @@ define void @foo(i8 %arg) {
   EXPECT_EQ(Invoke->getSuccessor(1), ExceptionBB);
 }
 
+TEST_F(TrackerTest, AtomicRMWSetters) {
+  parseIR(C, R"IR(
+define void @foo(ptr %ptr, i8 %arg) {
+  %atomicrmw = atomicrmw add ptr %ptr, i8 %arg acquire, align 128
+  ret void
+}
+)IR");
+  Function &LLVMF = *M->getFunction("foo");
+  sandboxir::Context Ctx(C);
+  auto &F = *Ctx.createFunction(&LLVMF);
+  auto *BB = &*F.begin();
+  auto It = BB->begin();
+  auto *RMW = cast<sandboxir::AtomicRMWInst>(&*It++);
+
+  // Check setAlignment().
+  Ctx.save();
+  auto OrigAlign = RMW->getAlign();
+  Align NewAlign(1024);
+  EXPECT_NE(NewAlign, OrigAlign);
+  RMW->setAlignment(NewAlign);
+  EXPECT_EQ(RMW->getAlign(), NewAlign);
+  Ctx.revert();
+  EXPECT_EQ(RMW->getAlign(), OrigAlign);
+
+  // Check setVolatile().
+  Ctx.save();
+  auto OrigIsVolatile = RMW->isVolatile();
+  bool NewIsVolatile = true;
+  EXPECT_NE(NewIsVolatile, OrigIsVolatile);
+  RMW->setVolatile(NewIsVolatile);
+  EXPECT_EQ(RMW->isVolatile(), NewIsVolatile);
+  Ctx.revert();
+  EXPECT_EQ(RMW->isVolatile(), OrigIsVolatile);
+
+  // Check setOrdering().
+  Ctx.save();
+  auto OrigOrdering = RMW->getOrdering();
+  auto NewOrdering = AtomicOrdering::SequentiallyConsistent;
+  EXPECT_NE(NewOrdering, OrigOrdering);
+  RMW->setOrdering(NewOrdering);
+  EXPECT_EQ(RMW->getOrdering(), NewOrdering);
+  Ctx.revert();
+  EXPECT_EQ(RMW->getOrdering(), OrigOrdering);
+
+  // Check setSyncScopeID().
+  Ctx.save();
+  auto OrigSSID = RMW->getSyncScopeID();
+  auto NewSSID = SyncScope::SingleThread;
+  EXPECT_NE(NewSSID, OrigSSID);
+  RMW->setSyncScopeID(NewSSID);
+  EXPECT_EQ(RMW->getSyncScopeID(), NewSSID);
+  Ctx.revert();
+  EXPECT_EQ(RMW->getSyncScopeID(), OrigSSID);
+}
+
 TEST_F(TrackerTest, AtomicCmpXchgSetters) {
   parseIR(C, R"IR(
 define void @foo(ptr %ptr, i8 %cmp, i8 %new) {

From f668708796b981733a5816f2efed0d5195af923d Mon Sep 17 00:00:00 2001
From: Michael Jones <michaelrj@google.com>
Date: Fri, 16 Aug 2024 09:51:29 -0700
Subject: [PATCH 203/441] [libc] Disable old headergen checks unless enabled
 (#104522)

Old Headergen needed extra build rules to ensure that it worked in
runtimes mode. This patch disables those checks if new headergen is
enabled. Also some new headers were not being properly built with
new headergen, and that's also fixed.
---
 libc/CMakeLists.txt          | 33 +++++++++++++++++----------------
 libc/include/CMakeLists.txt  | 14 ++++++++------
 llvm/runtimes/CMakeLists.txt |  3 ++-
 3 files changed, 27 insertions(+), 23 deletions(-)

diff --git a/libc/CMakeLists.txt b/libc/CMakeLists.txt
index dd45d6cc8cb6ab5..133686acf6fa62b 100644
--- a/libc/CMakeLists.txt
+++ b/libc/CMakeLists.txt
@@ -50,31 +50,32 @@ set(LIBC_NAMESPACE ${default_namespace}
   CACHE STRING "The namespace to use to enclose internal implementations. Must start with '__llvm_libc'."
 )
 
+option(LIBC_USE_NEW_HEADER_GEN "Generate header files using new headergen instead of the old one" ON)
 
-add_subdirectory(newhdrgen)
-
-
-if(LLVM_LIBC_FULL_BUILD OR LLVM_LIBC_GPU_BUILD)
-  if(NOT LIBC_HDRGEN_EXE)
-    # We need to set up hdrgen first since other targets depend on it.
-    add_subdirectory(utils/LibcTableGenUtil)
-    add_subdirectory(utils/HdrGen)
-    # Calling add_tablegen sets variables like LIBC_TABLEGEN_EXE in
-    # PARENT_SCOPE which get lost until saved in the cache.
-    set(LIBC_TABLEGEN_EXE "${LIBC_TABLEGEN_EXE}" CACHE INTERNAL "")
-    set(LIBC_TABLEGEN_TARGET "${LIBC_TABLEGEN_TARGET}" CACHE INTERNAL "")
-  else()
-    message(STATUS "Will use ${LIBC_HDRGEN_EXE} for libc header generation.")
+if(LIBC_USE_NEW_HEADER_GEN)
+  add_subdirectory(newhdrgen)
+else()
+  if(LLVM_LIBC_FULL_BUILD OR LLVM_LIBC_GPU_BUILD)
+    if(NOT LIBC_HDRGEN_EXE)
+      # We need to set up hdrgen first since other targets depend on it.
+      add_subdirectory(utils/LibcTableGenUtil)
+      add_subdirectory(utils/HdrGen)
+      # Calling add_tablegen sets variables like LIBC_TABLEGEN_EXE in
+      # PARENT_SCOPE which get lost until saved in the cache.
+      set(LIBC_TABLEGEN_EXE "${LIBC_TABLEGEN_EXE}" CACHE INTERNAL "")
+      set(LIBC_TABLEGEN_TARGET "${LIBC_TABLEGEN_TARGET}" CACHE INTERNAL "")
+    else()
+      message(STATUS "Will use ${LIBC_HDRGEN_EXE} for libc header generation.")
+    endif()
   endif()
 endif()
+
 # We will build the GPU utilities if we are not doing a runtimes build.
 option(LIBC_BUILD_GPU_LOADER "Always build the GPU loader utilities" OFF)
 if(LIBC_BUILD_GPU_LOADER OR (LLVM_LIBC_GPU_BUILD AND NOT LLVM_RUNTIMES_BUILD))
   add_subdirectory(utils/gpu)
 endif()
 
-option(LIBC_USE_NEW_HEADER_GEN "Generate header files using new headergen instead of the old one" ON)
-
 set(NEED_LIBC_HDRGEN FALSE)
 if(NOT LLVM_RUNTIMES_BUILD)
   if("libc" IN_LIST LLVM_ENABLE_RUNTIMES)
diff --git a/libc/include/CMakeLists.txt b/libc/include/CMakeLists.txt
index 2b6eb61782a6326..4e3ae7f801f4a08 100644
--- a/libc/include/CMakeLists.txt
+++ b/libc/include/CMakeLists.txt
@@ -420,19 +420,21 @@ add_header_macro(
     .llvm-libc-types.posix_spawn_file_actions_t
 )
 
-add_gen_header(
+add_header_macro(
   link
-  DEF_FILE link.h.def
-  GEN_HDR link.h
+  ../libc/newhdrgen/yaml/link.yaml
+  link.h.def
+  link.h
   DEPENDS
     .llvm_libc_common_h
     .llvm-libc-macros.link_macros
 )
 
-add_gen_header(
+add_header_macro(
   elf
-  DEF_FILE elf.h.def
-  GEN_HDR elf.h
+  ../libc/newhdrgen/yaml/elf.yaml
+  elf.h.def
+  elf.h
   DEPENDS
     .llvm-libc-macros.elf_macros
 )
diff --git a/llvm/runtimes/CMakeLists.txt b/llvm/runtimes/CMakeLists.txt
index 187c44fb9d04d2e..76202b2cfe04212 100644
--- a/llvm/runtimes/CMakeLists.txt
+++ b/llvm/runtimes/CMakeLists.txt
@@ -521,7 +521,8 @@ if(build_runtimes)
     endforeach()
   endif()
   if("libc" IN_LIST LLVM_ENABLE_PROJECTS AND
-      (LLVM_LIBC_FULL_BUILD OR LLVM_LIBC_GPU_BUILD))
+      (LLVM_LIBC_FULL_BUILD OR LLVM_LIBC_GPU_BUILD) AND 
+      (NOT LIBC_USE_NEW_HEADER_GEN))
     if(LIBC_HDRGEN_EXE)
       set(hdrgen_exe ${LIBC_HDRGEN_EXE})
     else()

From fbef911dc3ed5ab2c857736de9e68bec4c578410 Mon Sep 17 00:00:00 2001
From: Billy Zhu <billyzhu@modular.com>
Date: Fri, 16 Aug 2024 10:05:56 -0700
Subject: [PATCH 204/441] [bazel] Fix cyclic dependencies for macos (#104528)

Similar to https://github.com/llvm/llvm-project/pull/104481. Replace
more "Utility" dependencies with "UtilityHeaders" to avoid cyclic
dependency when building on macos.
---
 utils/bazel/llvm-project-overlay/lldb/BUILD.bazel | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel b/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel
index 7841045afa6d219..ee7ea7458b4488f 100644
--- a/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel
@@ -469,7 +469,7 @@ cc_library(
         "include/lldb/Host/posix/*.h",
     ]),
     strip_include_prefix = "include",
-    deps = [":Utility"],
+    deps = [":UtilityHeaders"],
 )
 
 cc_library(
@@ -484,7 +484,7 @@ cc_library(
         "@platforms//os:macos": [],
         "//conditions:default": ["@platforms//:incompatible"],
     }),
-    deps = [":Utility"],
+    deps = [":UtilityHeaders"],
 )
 
 objc_library(
@@ -502,7 +502,7 @@ objc_library(
         ":Headers",
         ":HostMacOSXHeaders",
         ":HostMacOSXPrivateHeaders",
-        ":Utility",
+        ":UtilityHeaders",
         "//llvm:Support",
         "//llvm:TargetParser",
     ],

From db279c72f2fea755c8e66cc1f0a69954764e9284 Mon Sep 17 00:00:00 2001
From: Helena Kotas <hekotas@microsoft.com>
Date: Fri, 16 Aug 2024 10:32:10 -0700
Subject: [PATCH 205/441] [HLSL] Change default linkage of HLSL functions to
 internal (#95331)

An HLSL function has internal linkage by default unless it is:
1. shader entry point function
2. marked with the `export` keyword
(https://github.com/llvm/llvm-project/issues/92812)
3. patch constant function (not implemented yet)

This PR adds a link-time pass `DXILFinalizeLinkage` that updates the
linkage of functions to make sure only shader entry points and exported
functions are visible from the module (have _program linkage_). All
other functions will be updated to have internal linkage.

Related spec update: microsoft/hlsl-specs#295

Fixes #llvm/llvm-project#92071
---
 llvm/lib/Target/DirectX/CMakeLists.txt        |  1 +
 .../Target/DirectX/DXILFinalizeLinkage.cpp    | 60 +++++++++++++++++
 llvm/lib/Target/DirectX/DXILFinalizeLinkage.h | 39 +++++++++++
 llvm/lib/Target/DirectX/DirectX.h             |  7 ++
 .../Target/DirectX/DirectXTargetMachine.cpp   |  2 +
 .../DirectX/conflicting-bitcast-insert.ll     |  4 +-
 llvm/test/CodeGen/DirectX/finalize_linkage.ll | 64 +++++++++++++++++++
 .../CodeGen/DirectX/omit-bitcast-insert.ll    |  6 +-
 8 files changed, 178 insertions(+), 5 deletions(-)
 create mode 100644 llvm/lib/Target/DirectX/DXILFinalizeLinkage.cpp
 create mode 100644 llvm/lib/Target/DirectX/DXILFinalizeLinkage.h
 create mode 100644 llvm/test/CodeGen/DirectX/finalize_linkage.ll

diff --git a/llvm/lib/Target/DirectX/CMakeLists.txt b/llvm/lib/Target/DirectX/CMakeLists.txt
index 4c70b3f9230edba..f7ae09957996b50 100644
--- a/llvm/lib/Target/DirectX/CMakeLists.txt
+++ b/llvm/lib/Target/DirectX/CMakeLists.txt
@@ -19,6 +19,7 @@ add_llvm_target(DirectXCodeGen
   DirectXSubtarget.cpp
   DirectXTargetMachine.cpp
   DXContainerGlobals.cpp
+  DXILFinalizeLinkage.cpp
   DXILIntrinsicExpansion.cpp
   DXILMetadata.cpp
   DXILOpBuilder.cpp
diff --git a/llvm/lib/Target/DirectX/DXILFinalizeLinkage.cpp b/llvm/lib/Target/DirectX/DXILFinalizeLinkage.cpp
new file mode 100644
index 000000000000000..c02eb768cdf49bb
--- /dev/null
+++ b/llvm/lib/Target/DirectX/DXILFinalizeLinkage.cpp
@@ -0,0 +1,60 @@
+//===- DXILFinalizeLinkage.cpp - Finalize linkage of functions ------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "DXILFinalizeLinkage.h"
+#include "DirectX.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/GlobalValue.h"
+#include "llvm/IR/Metadata.h"
+#include "llvm/IR/Module.h"
+
+#define DEBUG_TYPE "dxil-finalize-linkage"
+
+using namespace llvm;
+
+static bool finalizeLinkage(Module &M) {
+  SmallPtrSet<Function *, 8> EntriesAndExports;
+
+  // Find all entry points and export functions
+  for (Function &EF : M.functions()) {
+    if (!EF.hasFnAttribute("hlsl.shader") && !EF.hasFnAttribute("hlsl.export"))
+      continue;
+    EntriesAndExports.insert(&EF);
+  }
+
+  for (Function &F : M.functions()) {
+    if (F.getLinkage() == GlobalValue::ExternalLinkage &&
+        !EntriesAndExports.contains(&F)) {
+      F.setLinkage(GlobalValue::InternalLinkage);
+    }
+  }
+
+  return false;
+}
+
+PreservedAnalyses DXILFinalizeLinkage::run(Module &M,
+                                           ModuleAnalysisManager &AM) {
+  if (finalizeLinkage(M))
+    return PreservedAnalyses::none();
+  return PreservedAnalyses::all();
+}
+
+bool DXILFinalizeLinkageLegacy::runOnModule(Module &M) {
+  return finalizeLinkage(M);
+}
+
+char DXILFinalizeLinkageLegacy::ID = 0;
+
+INITIALIZE_PASS_BEGIN(DXILFinalizeLinkageLegacy, DEBUG_TYPE,
+                      "DXIL Finalize Linkage", false, false)
+INITIALIZE_PASS_END(DXILFinalizeLinkageLegacy, DEBUG_TYPE,
+                    "DXIL Finalize Linkage", false, false)
+
+ModulePass *llvm::createDXILFinalizeLinkageLegacyPass() {
+  return new DXILFinalizeLinkageLegacy();
+}
diff --git a/llvm/lib/Target/DirectX/DXILFinalizeLinkage.h b/llvm/lib/Target/DirectX/DXILFinalizeLinkage.h
new file mode 100644
index 000000000000000..aab1bc3f7a28e20
--- /dev/null
+++ b/llvm/lib/Target/DirectX/DXILFinalizeLinkage.h
@@ -0,0 +1,39 @@
+//===- DXILFinalizeLinkage.h - Finalize linkage of functions --------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// DXILFinalizeLinkage pass updates the linkage of functions to make sure only
+/// shader entry points and exported functions are visible from the module (have
+/// program linkage). All other functions will be updated to have internal
+/// linkage.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TARGET_DIRECTX_DXILFINALIZELINKAGE_H
+#define LLVM_TARGET_DIRECTX_DXILFINALIZELINKAGE_H
+
+#include "llvm/IR/PassManager.h"
+#include "llvm/Pass.h"
+
+namespace llvm {
+
+class DXILFinalizeLinkage : public PassInfoMixin<DXILFinalizeLinkage> {
+public:
+  PreservedAnalyses run(Module &M, ModuleAnalysisManager &);
+  static bool isRequired() { return true; }
+};
+
+class DXILFinalizeLinkageLegacy : public ModulePass {
+public:
+  DXILFinalizeLinkageLegacy() : ModulePass(ID) {}
+  bool runOnModule(Module &M) override;
+
+  static char ID; // Pass identification.
+};
+} // namespace llvm
+
+#endif // LLVM_TARGET_DIRECTX_DXILFINALIZELINKAGE_H
diff --git a/llvm/lib/Target/DirectX/DirectX.h b/llvm/lib/Target/DirectX/DirectX.h
index d056ae2bc488e7a..de6c6f2db5b2448 100644
--- a/llvm/lib/Target/DirectX/DirectX.h
+++ b/llvm/lib/Target/DirectX/DirectX.h
@@ -63,6 +63,13 @@ void initializeDXContainerGlobalsPass(PassRegistry &);
 
 /// Pass for generating DXContainer part globals.
 ModulePass *createDXContainerGlobalsPass();
+
+/// Initializer for DXILFinalizeLinkage pass.
+void initializeDXILFinalizeLinkageLegacyPass(PassRegistry &);
+
+/// Pass to finalize linkage of functions.
+ModulePass *createDXILFinalizeLinkageLegacyPass();
+
 } // namespace llvm
 
 #endif // LLVM_LIB_TARGET_DIRECTX_DIRECTX_H
diff --git a/llvm/lib/Target/DirectX/DirectXTargetMachine.cpp b/llvm/lib/Target/DirectX/DirectXTargetMachine.cpp
index 92bd69b69684f06..2c2b86e7193382e 100644
--- a/llvm/lib/Target/DirectX/DirectXTargetMachine.cpp
+++ b/llvm/lib/Target/DirectX/DirectXTargetMachine.cpp
@@ -48,6 +48,7 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeDirectXTarget() {
   initializeDXILTranslateMetadataPass(*PR);
   initializeDXILResourceMDWrapperPass(*PR);
   initializeShaderFlagsAnalysisWrapperPass(*PR);
+  initializeDXILFinalizeLinkageLegacyPass(*PR);
 }
 
 class DXILTargetObjectFile : public TargetLoweringObjectFile {
@@ -79,6 +80,7 @@ class DirectXPassConfig : public TargetPassConfig {
   void addCodeGenPrepare() override {
     addPass(createDXILIntrinsicExpansionLegacyPass());
     addPass(createDXILOpLoweringLegacyPass());
+    addPass(createDXILFinalizeLinkageLegacyPass());
     addPass(createDXILTranslateMetadataPass());
     addPass(createDXILPrepareModulePass());
   }
diff --git a/llvm/test/CodeGen/DirectX/conflicting-bitcast-insert.ll b/llvm/test/CodeGen/DirectX/conflicting-bitcast-insert.ll
index 01084f285cd72ac..8f5d3ae8641717f 100644
--- a/llvm/test/CodeGen/DirectX/conflicting-bitcast-insert.ll
+++ b/llvm/test/CodeGen/DirectX/conflicting-bitcast-insert.ll
@@ -7,7 +7,7 @@ define i64 @test(ptr %p) {
   ret i64 %v
 }
 
-; CHECK: define i64 @test(ptr %p) {
+; CHECK: define internal i64 @test(ptr %p) {
 ; CHECK-NEXT: %1 = bitcast ptr %p to ptr
 ; CHECK-NEXT: store i32 0, ptr %1, align 4
 ; CHECK-NEXT: %2 = bitcast ptr %p to ptr
@@ -19,7 +19,7 @@ define i64 @testGEP(ptr %p) {
   ret i64 %val
 }
 
-; CHECK: define i64 @testGEP(ptr %p) {
+; CHECK: define internal i64 @testGEP(ptr %p) {
 ; CHECK-NEXT:   %1 = bitcast ptr %p to ptr
 ; CHECK-NEXT:   %ptr = getelementptr i32, ptr %1, i32 4
 ; CHECK-NEXT:   %2 = bitcast ptr %p to ptr
diff --git a/llvm/test/CodeGen/DirectX/finalize_linkage.ll b/llvm/test/CodeGen/DirectX/finalize_linkage.ll
new file mode 100644
index 000000000000000..0ee8a5f44593ba1
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/finalize_linkage.ll
@@ -0,0 +1,64 @@
+; RUN: opt -S -dxil-finalize-linkage -mtriple=dxil-unknown-shadermodel6.5-compute %s | FileCheck %s
+; RUN: llc %s --filetype=asm -o - | FileCheck %s --check-prefixes=CHECK-LLC
+
+target triple = "dxilv1.5-pc-shadermodel6.5-compute"
+
+; DXILFinalizeLinkage changes linkage of all functions that are not
+; entry points or exported function to internal.
+
+; CHECK: define internal void @"?f1@@YAXXZ"()
+define void @"?f1@@YAXXZ"() #0 {
+entry:
+  ret void
+}
+
+; CHECK: define internal void @"?f2@@YAXXZ"()
+define void @"?f2@@YAXXZ"() #0 {
+entry:
+  ret void
+}
+
+; CHECK: define internal void @"?f3@@YAXXZ"()
+define void @"?f3@@YAXXZ"() #0 {
+entry:
+  ret void
+}
+
+; CHECK: define internal void @"?foo@@YAXXZ"()
+define void @"?foo@@YAXXZ"() #0 {
+entry:
+  call void @"?f2@@YAXXZ"() #3
+  ret void
+}
+
+; Exported function - do not change linkage
+; CHECK: define void @"?bar@@YAXXZ"()
+define void @"?bar@@YAXXZ"() #1 {
+entry:
+  call void @"?f3@@YAXXZ"() #3
+  ret void
+}
+
+; CHECK: define internal void @"?main@@YAXXZ"() #0
+define internal void @"?main@@YAXXZ"() #0 {
+entry:
+  call void @"?foo@@YAXXZ"() #3
+  call void @"?bar@@YAXXZ"() #3
+  ret void
+}
+
+; Entry point function - do not change linkage
+; CHECK: define void @main() #2
+define void @main() #2 {
+entry:
+  call void @"?main@@YAXXZ"()
+  ret void
+}
+
+attributes #0 = { convergent noinline nounwind optnone}
+attributes #1 = { convergent noinline nounwind optnone "hlsl.export"}
+attributes #2 = { convergent "hlsl.numthreads"="4,1,1" "hlsl.shader"="compute"}
+attributes #3 = { convergent }
+
+; Make sure "hlsl.export" attribute is stripped by llc
+; CHECK-LLC-NOT: "hlsl.export"
diff --git a/llvm/test/CodeGen/DirectX/omit-bitcast-insert.ll b/llvm/test/CodeGen/DirectX/omit-bitcast-insert.ll
index 73c33e8cfdd8648..6066a0033e45dee 100644
--- a/llvm/test/CodeGen/DirectX/omit-bitcast-insert.ll
+++ b/llvm/test/CodeGen/DirectX/omit-bitcast-insert.ll
@@ -6,7 +6,7 @@ define i64 @test(ptr %p) {
   ret i64 %v
 }
 
-; CHECK: define i64 @test(ptr %p) {
+; CHECK: define internal i64 @test(ptr %p) {
 ; CHECK-NEXT: %v = load i64, ptr %p, align 8
 ; CHECK-NEXT: ret i64 %v
 
@@ -16,7 +16,7 @@ define i64 @test2(ptr %p) {
   ret i64 %v
 }
 
-; CHECK: define i64 @test2(ptr %p) {
+; CHECK: define internal i64 @test2(ptr %p) {
 ; CHECK-NEXT: store i64 0, ptr %p
 ; CHECK-NEXT: %v = load i64, ptr %p, align 8
 ; CHECK-NEXT: ret i64 %v
@@ -27,6 +27,6 @@ define i32 @test3(ptr %0)  {
   ret i32 %3
 }
 
-; CHECK: define i32 @test3(ptr %0)  {
+; CHECK: define internal i32 @test3(ptr %0)  {
 ; CHECK-NEXT: %2 = getelementptr i32, ptr %0, i32 4
 ; CHECK-NEXT: %3 = load i32, ptr %2

From 907c7eb311077c5da434bf67858b1173a351d800 Mon Sep 17 00:00:00 2001
From: Shilei Tian <i@tianshilei.me>
Date: Fri, 16 Aug 2024 13:33:48 -0400
Subject: [PATCH 206/441] [Attributor] Enable `AAAddressSpace` in `OpenMPOpt`
 (#104363)

This reverts commit e592c2dcf5b7d2da6c2564f5d9990aa34079bad4.

We can finally reland the PR since the issue that caused the PR to be
reverted has been resolved in
https://github.com/llvm/llvm-project/pull/104051.
---
 llvm/lib/Transforms/IPO/OpenMPOpt.cpp         |  4 +++
 .../reduced/openmp_opt_constant_type_crash.ll |  1 +
 .../test/Transforms/OpenMP/barrier_removal.ll | 16 ++++++------
 .../heap-to-shared-missing-declarations.ll    |  2 +-
 .../Transforms/OpenMP/nested_parallelism.ll   |  4 +--
 .../OpenMP/spmdization_kernel_env_dep.ll      | 25 ++++++++++---------
 6 files changed, 28 insertions(+), 24 deletions(-)

diff --git a/llvm/lib/Transforms/IPO/OpenMPOpt.cpp b/llvm/lib/Transforms/IPO/OpenMPOpt.cpp
index 798bcb77328fc81..baf39c908706c6e 100644
--- a/llvm/lib/Transforms/IPO/OpenMPOpt.cpp
+++ b/llvm/lib/Transforms/IPO/OpenMPOpt.cpp
@@ -5569,6 +5569,8 @@ void OpenMPOpt::registerAAsForFunction(Attributor &A, const Function &F) {
       bool UsedAssumedInformation = false;
       A.getAssumedSimplified(IRPosition::value(*LI), /* AA */ nullptr,
                              UsedAssumedInformation, AA::Interprocedural);
+      A.getOrCreateAAFor<AAAddressSpace>(
+          IRPosition::value(*LI->getPointerOperand()));
       continue;
     }
     if (auto *CI = dyn_cast<CallBase>(&I)) {
@@ -5578,6 +5580,8 @@ void OpenMPOpt::registerAAsForFunction(Attributor &A, const Function &F) {
     }
     if (auto *SI = dyn_cast<StoreInst>(&I)) {
       A.getOrCreateAAFor<AAIsDead>(IRPosition::value(*SI));
+      A.getOrCreateAAFor<AAAddressSpace>(
+          IRPosition::value(*SI->getPointerOperand()));
       continue;
     }
     if (auto *FI = dyn_cast<FenceInst>(&I)) {
diff --git a/llvm/test/Transforms/Attributor/reduced/openmp_opt_constant_type_crash.ll b/llvm/test/Transforms/Attributor/reduced/openmp_opt_constant_type_crash.ll
index fda72a6e31a0c78..518ed97f42bc107 100644
--- a/llvm/test/Transforms/Attributor/reduced/openmp_opt_constant_type_crash.ll
+++ b/llvm/test/Transforms/Attributor/reduced/openmp_opt_constant_type_crash.ll
@@ -116,6 +116,7 @@ cond.end:                                         ; preds = %cond.true, %entry
 ; CHECK-LABEL: define {{[^@]+}}@_ZN6Kokkos4Impl14SubviewExtentsILj2ELj1EE3setIJLm0ELm0EEJiEEEbjjRKNS0_13ViewDimensionIJXspT_EEEENS0_5ALL_tEDpT0_.internalized
 ; CHECK-SAME: (ptr nocapture writeonly [[THIS:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = addrspacecast ptr [[THIS]] to ptr addrspace(5)
 ; CHECK-NEXT:    ret i1 false
 ;
 ;
diff --git a/llvm/test/Transforms/OpenMP/barrier_removal.ll b/llvm/test/Transforms/OpenMP/barrier_removal.ll
index 0925418cf54c725..47a5d5104aa8bd3 100644
--- a/llvm/test/Transforms/OpenMP/barrier_removal.ll
+++ b/llvm/test/Transforms/OpenMP/barrier_removal.ll
@@ -269,10 +269,9 @@ define void @neg_empty_2() "kernel" {
 define void @pos_constant_loads() "kernel" {
 ; CHECK-LABEL: define {{[^@]+}}@pos_constant_loads
 ; CHECK-SAME: () #[[ATTR4]] {
-; CHECK-NEXT:    [[ARG:%.*]] = load ptr addrspace(4), ptr addrspacecast (ptr addrspace(4) @GPtr4 to ptr), align 8
-; CHECK-NEXT:    [[B:%.*]] = load i32, ptr addrspacecast (ptr addrspace(4) @GC2 to ptr), align 4
-; CHECK-NEXT:    [[ARGC:%.*]] = addrspacecast ptr addrspace(4) [[ARG]] to ptr
-; CHECK-NEXT:    [[C:%.*]] = load i32, ptr [[ARGC]], align 4
+; CHECK-NEXT:    [[ARG:%.*]] = load ptr addrspace(4), ptr addrspace(4) @GPtr4, align 8
+; CHECK-NEXT:    [[B:%.*]] = load i32, ptr addrspace(4) @GC2, align 4
+; CHECK-NEXT:    [[C:%.*]] = load i32, ptr addrspace(4) [[ARG]], align 4
 ; CHECK-NEXT:    [[D:%.*]] = add i32 42, [[B]]
 ; CHECK-NEXT:    [[E:%.*]] = add i32 [[D]], [[C]]
 ; CHECK-NEXT:    call void @useI32(i32 [[E]])
@@ -303,7 +302,7 @@ define void @neg_loads() "kernel" {
 ; CHECK-NEXT:    [[ARG:%.*]] = load ptr, ptr @GPtr, align 8
 ; CHECK-NEXT:    [[A:%.*]] = load i32, ptr @G, align 4
 ; CHECK-NEXT:    call void @aligned_barrier()
-; CHECK-NEXT:    [[B:%.*]] = load i32, ptr addrspacecast (ptr addrspace(3) @GS to ptr), align 4
+; CHECK-NEXT:    [[B:%.*]] = load i32, ptr addrspace(3) @GS, align 4
 ; CHECK-NEXT:    call void @aligned_barrier()
 ; CHECK-NEXT:    [[C:%.*]] = load i32, ptr [[ARG]], align 4
 ; CHECK-NEXT:    call void @aligned_barrier()
@@ -335,9 +334,8 @@ define void @pos_priv_mem() "kernel" {
 ; CHECK-NEXT:    [[LOC:%.*]] = alloca i32, align 4, addrspace(5)
 ; CHECK-NEXT:    [[A:%.*]] = load i32, ptr @PG1, align 4
 ; CHECK-NEXT:    store i32 [[A]], ptr addrspace(5) [[LOC]], align 4
-; CHECK-NEXT:    [[B:%.*]] = load i32, ptr addrspacecast (ptr addrspace(5) @PG2 to ptr), align 4
-; CHECK-NEXT:    [[ARGC:%.*]] = addrspacecast ptr addrspace(5) [[ARG]] to ptr
-; CHECK-NEXT:    store i32 [[B]], ptr [[ARGC]], align 4
+; CHECK-NEXT:    [[B:%.*]] = load i32, ptr addrspace(5) @PG2, align 4
+; CHECK-NEXT:    store i32 [[B]], ptr addrspace(5) [[ARG]], align 4
 ; CHECK-NEXT:    [[V:%.*]] = load i32, ptr addrspace(5) [[LOC]], align 4
 ; CHECK-NEXT:    store i32 [[V]], ptr @PG1, align 4
 ; CHECK-NEXT:    ret void
@@ -370,7 +368,7 @@ define void @neg_mem() "kernel" {
 ; CHECK-NEXT:    store i32 [[A]], ptr [[ARG]], align 4
 ; CHECK-NEXT:    fence release
 ; CHECK-NEXT:    call void @aligned_barrier()
-; CHECK-NEXT:    [[B:%.*]] = load i32, ptr addrspacecast (ptr addrspace(1) @G2 to ptr), align 4
+; CHECK-NEXT:    [[B:%.*]] = load i32, ptr addrspace(1) @G2, align 4
 ; CHECK-NEXT:    store i32 [[B]], ptr @G1, align 4
 ; CHECK-NEXT:    fence acquire
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/OpenMP/heap-to-shared-missing-declarations.ll b/llvm/test/Transforms/OpenMP/heap-to-shared-missing-declarations.ll
index 7a5bba75f6c48c5..d81f34f3c427374 100644
--- a/llvm/test/Transforms/OpenMP/heap-to-shared-missing-declarations.ll
+++ b/llvm/test/Transforms/OpenMP/heap-to-shared-missing-declarations.ll
@@ -22,7 +22,7 @@ define internal void @func() {
 ; CHECK-LABEL: define {{[^@]+}}@func
 ; CHECK-SAME: () #[[ATTR1]] {
 ; CHECK-NEXT:  bb:
-; CHECK-NEXT:    [[I:%.*]] = load ptr, ptr null, align 4294967296
+; CHECK-NEXT:    [[I:%.*]] = load ptr, ptr addrspace(5) null, align 4294967296
 ; CHECK-NEXT:    store i64 0, ptr [[I]], align 8
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/OpenMP/nested_parallelism.ll b/llvm/test/Transforms/OpenMP/nested_parallelism.ll
index 4f4a87cbddfec15..5c4386b24a3d5a6 100644
--- a/llvm/test/Transforms/OpenMP/nested_parallelism.ll
+++ b/llvm/test/Transforms/OpenMP/nested_parallelism.ll
@@ -60,7 +60,7 @@ define weak_odr protected void @__omp_offloading_10302_bd7e0_main_l13(ptr %dyn,
 ; CHECK-NEXT:    br i1 [[TMP3]], label [[REGION_GUARDED_I:%.*]], label [[_Z3FOOI_INTERNALIZED_EXIT:%.*]]
 ; CHECK:       region.guarded.i:
 ; CHECK-NEXT:    [[I_ADDR_SROA_0_0_EXTRACT_TRUNC:%.*]] = trunc i64 [[I:%.*]] to i32
-; CHECK-NEXT:    store i32 [[I_ADDR_SROA_0_0_EXTRACT_TRUNC]], ptr addrspacecast (ptr addrspace(3) @i_shared to ptr), align 16
+; CHECK-NEXT:    store i32 [[I_ADDR_SROA_0_0_EXTRACT_TRUNC]], ptr addrspace(3) @i_shared, align 16
 ; CHECK-NEXT:    br label [[_Z3FOOI_INTERNALIZED_EXIT]]
 ; CHECK:       _Z3fooi.internalized.exit:
 ; CHECK-NEXT:    tail call void @__kmpc_barrier_simple_spmd(ptr nonnull @[[GLOB1]], i32 [[TMP2]]) #[[ATTR2]]
@@ -140,7 +140,7 @@ define weak_odr protected void @__omp_offloading_10302_bd7e0_main_l16(ptr %dyn,
 ; CHECK-NEXT:    [[I_ADDR_SROA_0_0_EXTRACT_TRUNC:%.*]] = trunc i64 [[I:%.*]] to i32
 ; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr nonnull [[CAPTURED_VARS_ADDRS_I]])
 ; CHECK-NEXT:    [[TMP1:%.*]] = tail call i32 @__kmpc_global_thread_num(ptr nonnull @[[GLOB1]]) #[[ATTR2]]
-; CHECK-NEXT:    store i32 [[I_ADDR_SROA_0_0_EXTRACT_TRUNC]], ptr addrspacecast (ptr addrspace(3) @i.i_shared to ptr), align 16
+; CHECK-NEXT:    store i32 [[I_ADDR_SROA_0_0_EXTRACT_TRUNC]], ptr addrspace(3) @i.i_shared, align 16
 ; CHECK-NEXT:    store ptr addrspacecast (ptr addrspace(3) @i.i_shared to ptr), ptr [[CAPTURED_VARS_ADDRS_I]], align 8
 ; CHECK-NEXT:    call void @__kmpc_parallel_51(ptr nonnull @[[GLOB1]], i32 [[TMP1]], i32 1, i32 -1, i32 -1, ptr nonnull @__omp_outlined__1, ptr nonnull @__omp_outlined__1_wrapper, ptr nonnull [[CAPTURED_VARS_ADDRS_I]], i64 1)
 ; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr nonnull [[CAPTURED_VARS_ADDRS_I]])
diff --git a/llvm/test/Transforms/OpenMP/spmdization_kernel_env_dep.ll b/llvm/test/Transforms/OpenMP/spmdization_kernel_env_dep.ll
index ce7b4f89b893ffc..52be16c41f872d9 100644
--- a/llvm/test/Transforms/OpenMP/spmdization_kernel_env_dep.ll
+++ b/llvm/test/Transforms/OpenMP/spmdization_kernel_env_dep.ll
@@ -27,21 +27,22 @@ define i32 @fputs() {
 define internal i32 @__kmpc_target_init(ptr %0, ptr %dyn) {
 ; AMDGPU-LABEL: define {{[^@]+}}@__kmpc_target_init
 ; AMDGPU-SAME: (ptr [[TMP0:%.*]], ptr [[DYN:%.*]]) #[[ATTR1:[0-9]+]] {
-; AMDGPU-NEXT:    [[TMP2:%.*]] = load i8, ptr getelementptr (i8, ptr addrspacecast (ptr addrspace(1) @__omp_offloading_10302_b20a40e_main_l4_kernel_environment to ptr), i64 2), align 2
-; AMDGPU-NEXT:    [[TMP3:%.*]] = and i8 [[TMP2]], 2
-; AMDGPU-NEXT:    [[TMP4:%.*]] = icmp ne i8 [[TMP3]], 0
-; AMDGPU-NEXT:    [[TMP5:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x() #[[ATTR3:[0-9]+]]
-; AMDGPU-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
-; AMDGPU-NEXT:    [[OR_COND:%.*]] = select i1 [[TMP4]], i1 [[TMP6]], i1 false
-; AMDGPU-NEXT:    br i1 [[OR_COND]], label [[TMP7:%.*]], label [[TMP8:%.*]]
-; AMDGPU:       7:
-; AMDGPU-NEXT:    store i8 0, ptr addrspace(3) null, align 2147483648
-; AMDGPU-NEXT:    br label [[TMP8]]
+; AMDGPU-NEXT:    [[TMP2:%.*]] = addrspacecast ptr getelementptr (i8, ptr addrspacecast (ptr addrspace(1) @__omp_offloading_10302_b20a40e_main_l4_kernel_environment to ptr), i64 2) to ptr addrspace(1)
+; AMDGPU-NEXT:    [[TMP3:%.*]] = load i8, ptr addrspace(1) [[TMP2]], align 2
+; AMDGPU-NEXT:    [[TMP4:%.*]] = and i8 [[TMP3]], 2
+; AMDGPU-NEXT:    [[TMP5:%.*]] = icmp ne i8 [[TMP4]], 0
+; AMDGPU-NEXT:    [[TMP6:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x() #[[ATTR3:[0-9]+]]
+; AMDGPU-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[TMP6]], 0
+; AMDGPU-NEXT:    [[OR_COND:%.*]] = select i1 [[TMP5]], i1 [[TMP7]], i1 false
+; AMDGPU-NEXT:    br i1 [[OR_COND]], label [[TMP8:%.*]], label [[TMP9:%.*]]
 ; AMDGPU:       8:
-; AMDGPU-NEXT:    br label [[TMP10:%.*]]
+; AMDGPU-NEXT:    store i8 0, ptr addrspace(3) null, align 2147483648
+; AMDGPU-NEXT:    br label [[TMP9]]
 ; AMDGPU:       9:
-; AMDGPU-NEXT:    unreachable
+; AMDGPU-NEXT:    br label [[TMP11:%.*]]
 ; AMDGPU:       10:
+; AMDGPU-NEXT:    unreachable
+; AMDGPU:       11:
 ; AMDGPU-NEXT:    ret i32 0
 ;
   %2 = getelementptr %struct.ConfigurationEnvironmentTy.8, ptr %0, i64 0, i32 2

From b81697718f49b0f353882e178f25981f47b2aa43 Mon Sep 17 00:00:00 2001
From: Egor Zhdan <e_zhdan@apple.com>
Date: Fri, 16 Aug 2024 18:48:39 +0100
Subject: [PATCH 207/441] [APINotes] Support fields of C/C++ structs

This allows annotating fields of C/C++ structs using API Notes.

Previously API Notes supported Objective-C properties, but not fields.

rdar://131548377
---
 clang/include/clang/APINotes/APINotesReader.h |   7 +
 clang/include/clang/APINotes/APINotesWriter.h |   8 ++
 clang/include/clang/APINotes/Types.h          |   6 +
 clang/lib/APINotes/APINotesFormat.h           |  20 ++-
 clang/lib/APINotes/APINotesReader.cpp         | 130 ++++++++++++++++++
 clang/lib/APINotes/APINotesWriter.cpp         |  73 ++++++++++
 clang/lib/APINotes/APINotesYAMLCompiler.cpp   |  64 +++++++--
 clang/lib/Sema/SemaAPINotes.cpp               |  20 +++
 .../APINotes/Inputs/Headers/Fields.apinotes   |  23 ++++
 clang/test/APINotes/Inputs/Headers/Fields.h   |  21 +++
 .../APINotes/Inputs/Headers/module.modulemap  |   5 +
 clang/test/APINotes/fields.cpp                |  17 +++
 12 files changed, 381 insertions(+), 13 deletions(-)
 create mode 100644 clang/test/APINotes/Inputs/Headers/Fields.apinotes
 create mode 100644 clang/test/APINotes/Inputs/Headers/Fields.h
 create mode 100644 clang/test/APINotes/fields.cpp

diff --git a/clang/include/clang/APINotes/APINotesReader.h b/clang/include/clang/APINotes/APINotesReader.h
index 03657352c49a59e..baf63340640242a 100644
--- a/clang/include/clang/APINotes/APINotesReader.h
+++ b/clang/include/clang/APINotes/APINotesReader.h
@@ -141,6 +141,13 @@ class APINotesReader {
                                                  ObjCSelectorRef Selector,
                                                  bool IsInstanceMethod);
 
+  /// Look for information regarding the given field of a C struct.
+  ///
+  /// \param Name The name of the field.
+  ///
+  /// \returns information about the field, if known.
+  VersionedInfo<FieldInfo> lookupField(ContextID CtxID, llvm::StringRef Name);
+
   /// Look for information regarding the given C++ method in the given C++ tag
   /// context.
   ///
diff --git a/clang/include/clang/APINotes/APINotesWriter.h b/clang/include/clang/APINotes/APINotesWriter.h
index e0fe5eacef72582..3cc16c3d959faf6 100644
--- a/clang/include/clang/APINotes/APINotesWriter.h
+++ b/clang/include/clang/APINotes/APINotesWriter.h
@@ -86,6 +86,14 @@ class APINotesWriter {
   void addCXXMethod(ContextID CtxID, llvm::StringRef Name,
                     const CXXMethodInfo &Info, llvm::VersionTuple SwiftVersion);
 
+  /// Add information about a specific C record field.
+  ///
+  /// \param CtxID The context in which this field resides, i.e. a C/C++ tag.
+  /// \param Name The name of the field.
+  /// \param Info Information about this field.
+  void addField(ContextID CtxID, llvm::StringRef Name, const FieldInfo &Info,
+                llvm::VersionTuple SwiftVersion);
+
   /// Add information about a global variable.
   ///
   /// \param Name The name of this global variable.
diff --git a/clang/include/clang/APINotes/Types.h b/clang/include/clang/APINotes/Types.h
index f972d0cf26640d3..89889910d1a0730 100644
--- a/clang/include/clang/APINotes/Types.h
+++ b/clang/include/clang/APINotes/Types.h
@@ -656,6 +656,12 @@ class GlobalFunctionInfo : public FunctionInfo {
   GlobalFunctionInfo() {}
 };
 
+/// Describes API notes data for a C/C++ record field.
+class FieldInfo : public VariableInfo {
+public:
+  FieldInfo() {}
+};
+
 /// Describes API notes data for a C++ method.
 class CXXMethodInfo : public FunctionInfo {
 public:
diff --git a/clang/lib/APINotes/APINotesFormat.h b/clang/lib/APINotes/APINotesFormat.h
index fba5f4e8907dae8..d724a9ea471b54a 100644
--- a/clang/lib/APINotes/APINotesFormat.h
+++ b/clang/lib/APINotes/APINotesFormat.h
@@ -24,7 +24,7 @@ const uint16_t VERSION_MAJOR = 0;
 /// API notes file minor version number.
 ///
 /// When the format changes IN ANY WAY, this number should be incremented.
-const uint16_t VERSION_MINOR = 29; // SwiftConformsTo
+const uint16_t VERSION_MINOR = 30; // fields
 
 const uint8_t kSwiftCopyable = 1;
 const uint8_t kSwiftNonCopyable = 2;
@@ -72,6 +72,10 @@ enum BlockID {
   /// used in other tables.
   OBJC_SELECTOR_BLOCK_ID,
 
+  /// The fields data block, which maps names fields of C records to
+  /// information about the field.
+  FIELD_BLOCK_ID,
+
   /// The global variables data block, which maps global variable names to
   /// information about the global variable.
   GLOBAL_VARIABLE_BLOCK_ID,
@@ -199,6 +203,20 @@ using CXXMethodDataLayout =
                          >;
 } // namespace cxx_method_block
 
+namespace field_block {
+enum {
+  FIELD_DATA = 1,
+};
+
+using FieldDataLayout =
+    llvm::BCRecordLayout<FIELD_DATA,      // record ID
+                         llvm::BCVBR<16>, // table offset within the blob (see
+                                          // below)
+                         llvm::BCBlob     // map from C (context id, name)
+                                          // tuples to C field information
+                         >;
+} // namespace field_block
+
 namespace objc_selector_block {
 enum {
   OBJC_SELECTOR_DATA = 1,
diff --git a/clang/lib/APINotes/APINotesReader.cpp b/clang/lib/APINotes/APINotesReader.cpp
index c05fdffe4a071b8..da8a86f7adb16e0 100644
--- a/clang/lib/APINotes/APINotesReader.cpp
+++ b/clang/lib/APINotes/APINotesReader.cpp
@@ -299,6 +299,28 @@ class ObjCPropertyTableInfo
   }
 };
 
+/// Used to deserialize the on-disk C record field table.
+class FieldTableInfo
+    : public VersionedTableInfo<FieldTableInfo, SingleDeclTableKey, FieldInfo> {
+public:
+  static internal_key_type ReadKey(const uint8_t *Data, unsigned Length) {
+    auto CtxID = endian::readNext<uint32_t, llvm::endianness::little>(Data);
+    auto NameID = endian::readNext<uint32_t, llvm::endianness::little>(Data);
+    return {CtxID, NameID};
+  }
+
+  hash_value_type ComputeHash(internal_key_type Key) {
+    return static_cast<size_t>(Key.hashValue());
+  }
+
+  static FieldInfo readUnversioned(internal_key_type Key,
+                                   const uint8_t *&Data) {
+    FieldInfo Info;
+    ReadVariableInfo(Data, Info);
+    return Info;
+  }
+};
+
 /// Read serialized ParamInfo.
 void ReadParamInfo(const uint8_t *&Data, ParamInfo &Info) {
   ReadVariableInfo(Data, Info);
@@ -653,6 +675,12 @@ class APINotesReader::Implementation {
   /// The Objective-C property table.
   std::unique_ptr<SerializedObjCPropertyTable> ObjCPropertyTable;
 
+  using SerializedFieldTable =
+      llvm::OnDiskIterableChainedHashTable<FieldTableInfo>;
+
+  /// The C record field table.
+  std::unique_ptr<SerializedFieldTable> FieldTable;
+
   using SerializedObjCMethodTable =
       llvm::OnDiskIterableChainedHashTable<ObjCMethodTableInfo>;
 
@@ -720,6 +748,8 @@ class APINotesReader::Implementation {
                            llvm::SmallVectorImpl<uint64_t> &Scratch);
   bool readCXXMethodBlock(llvm::BitstreamCursor &Cursor,
                           llvm::SmallVectorImpl<uint64_t> &Scratch);
+  bool readFieldBlock(llvm::BitstreamCursor &Cursor,
+                      llvm::SmallVectorImpl<uint64_t> &Scratch);
   bool readObjCSelectorBlock(llvm::BitstreamCursor &Cursor,
                              llvm::SmallVectorImpl<uint64_t> &Scratch);
   bool readGlobalVariableBlock(llvm::BitstreamCursor &Cursor,
@@ -1252,6 +1282,81 @@ bool APINotesReader::Implementation::readCXXMethodBlock(
   return false;
 }
 
+bool APINotesReader::Implementation::readFieldBlock(
+    llvm::BitstreamCursor &Cursor, llvm::SmallVectorImpl<uint64_t> &Scratch) {
+  if (Cursor.EnterSubBlock(FIELD_BLOCK_ID))
+    return true;
+
+  llvm::Expected<llvm::BitstreamEntry> MaybeNext = Cursor.advance();
+  if (!MaybeNext) {
+    // FIXME this drops the error on the floor.
+    consumeError(MaybeNext.takeError());
+    return false;
+  }
+  llvm::BitstreamEntry Next = MaybeNext.get();
+  while (Next.Kind != llvm::BitstreamEntry::EndBlock) {
+    if (Next.Kind == llvm::BitstreamEntry::Error)
+      return true;
+
+    if (Next.Kind == llvm::BitstreamEntry::SubBlock) {
+      // Unknown sub-block, possibly for use by a future version of the
+      // API notes format.
+      if (Cursor.SkipBlock())
+        return true;
+
+      MaybeNext = Cursor.advance();
+      if (!MaybeNext) {
+        // FIXME this drops the error on the floor.
+        consumeError(MaybeNext.takeError());
+        return false;
+      }
+      Next = MaybeNext.get();
+      continue;
+    }
+
+    Scratch.clear();
+    llvm::StringRef BlobData;
+    llvm::Expected<unsigned> MaybeKind =
+        Cursor.readRecord(Next.ID, Scratch, &BlobData);
+    if (!MaybeKind) {
+      // FIXME this drops the error on the floor.
+      consumeError(MaybeKind.takeError());
+      return false;
+    }
+    unsigned Kind = MaybeKind.get();
+    switch (Kind) {
+    case field_block::FIELD_DATA: {
+      // Already saw field table.
+      if (FieldTable)
+        return true;
+
+      uint32_t tableOffset;
+      field_block::FieldDataLayout::readRecord(Scratch, tableOffset);
+      auto base = reinterpret_cast<const uint8_t *>(BlobData.data());
+
+      FieldTable.reset(SerializedFieldTable::Create(
+          base + tableOffset, base + sizeof(uint32_t), base));
+      break;
+    }
+
+    default:
+      // Unknown record, possibly for use by a future version of the
+      // module format.
+      break;
+    }
+
+    MaybeNext = Cursor.advance();
+    if (!MaybeNext) {
+      // FIXME this drops the error on the floor.
+      consumeError(MaybeNext.takeError());
+      return false;
+    }
+    Next = MaybeNext.get();
+  }
+
+  return false;
+}
+
 bool APINotesReader::Implementation::readObjCSelectorBlock(
     llvm::BitstreamCursor &Cursor, llvm::SmallVectorImpl<uint64_t> &Scratch) {
   if (Cursor.EnterSubBlock(OBJC_SELECTOR_BLOCK_ID))
@@ -1812,6 +1917,14 @@ APINotesReader::APINotesReader(llvm::MemoryBuffer *InputBuffer,
       }
       break;
 
+    case FIELD_BLOCK_ID:
+      if (!HasValidControlBlock ||
+          Implementation->readFieldBlock(Cursor, Scratch)) {
+        Failed = true;
+        return;
+      }
+      break;
+
     case OBJC_SELECTOR_BLOCK_ID:
       if (!HasValidControlBlock ||
           Implementation->readObjCSelectorBlock(Cursor, Scratch)) {
@@ -2031,6 +2144,23 @@ auto APINotesReader::lookupObjCMethod(ContextID CtxID, ObjCSelectorRef Selector,
   return {Implementation->SwiftVersion, *Known};
 }
 
+auto APINotesReader::lookupField(ContextID CtxID, llvm::StringRef Name)
+    -> VersionedInfo<FieldInfo> {
+  if (!Implementation->FieldTable)
+    return std::nullopt;
+
+  std::optional<IdentifierID> NameID = Implementation->getIdentifier(Name);
+  if (!NameID)
+    return std::nullopt;
+
+  auto Known = Implementation->FieldTable->find(
+      SingleDeclTableKey(CtxID.Value, *NameID));
+  if (Known == Implementation->FieldTable->end())
+    return std::nullopt;
+
+  return {Implementation->SwiftVersion, *Known};
+}
+
 auto APINotesReader::lookupCXXMethod(ContextID CtxID, llvm::StringRef Name)
     -> VersionedInfo<CXXMethodInfo> {
   if (!Implementation->CXXMethodTable)
diff --git a/clang/lib/APINotes/APINotesWriter.cpp b/clang/lib/APINotes/APINotesWriter.cpp
index cf3a0bee393eee4..c452677983bb360 100644
--- a/clang/lib/APINotes/APINotesWriter.cpp
+++ b/clang/lib/APINotes/APINotesWriter.cpp
@@ -62,6 +62,13 @@ class APINotesWriter::Implementation {
       llvm::SmallVector<std::pair<VersionTuple, ObjCPropertyInfo>, 1>>
       ObjCProperties;
 
+  /// Information about C record fields.
+  ///
+  /// Indexed by the context ID and name ID.
+  llvm::DenseMap<SingleDeclTableKey,
+                 llvm::SmallVector<std::pair<VersionTuple, FieldInfo>, 1>>
+      Fields;
+
   /// Information about Objective-C methods.
   ///
   /// Indexed by the context ID, selector ID, and Boolean (stored as a char)
@@ -158,6 +165,7 @@ class APINotesWriter::Implementation {
   void writeObjCPropertyBlock(llvm::BitstreamWriter &Stream);
   void writeObjCMethodBlock(llvm::BitstreamWriter &Stream);
   void writeCXXMethodBlock(llvm::BitstreamWriter &Stream);
+  void writeFieldBlock(llvm::BitstreamWriter &Stream);
   void writeObjCSelectorBlock(llvm::BitstreamWriter &Stream);
   void writeGlobalVariableBlock(llvm::BitstreamWriter &Stream);
   void writeGlobalFunctionBlock(llvm::BitstreamWriter &Stream);
@@ -190,6 +198,7 @@ void APINotesWriter::Implementation::writeToStream(llvm::raw_ostream &OS) {
     writeObjCPropertyBlock(Stream);
     writeObjCMethodBlock(Stream);
     writeCXXMethodBlock(Stream);
+    writeFieldBlock(Stream);
     writeObjCSelectorBlock(Stream);
     writeGlobalVariableBlock(Stream);
     writeGlobalFunctionBlock(Stream);
@@ -858,6 +867,62 @@ void APINotesWriter::Implementation::writeCXXMethodBlock(
   }
 }
 
+namespace {
+/// Used to serialize the on-disk C field table.
+class FieldTableInfo
+    : public VersionedTableInfo<FieldTableInfo, SingleDeclTableKey, FieldInfo> {
+public:
+  unsigned getKeyLength(key_type_ref) {
+    return sizeof(uint32_t) + sizeof(uint32_t);
+  }
+
+  void EmitKey(raw_ostream &OS, key_type_ref Key, unsigned) {
+    llvm::support::endian::Writer writer(OS, llvm::endianness::little);
+    writer.write<uint32_t>(Key.parentContextID);
+    writer.write<uint32_t>(Key.nameID);
+  }
+
+  hash_value_type ComputeHash(key_type_ref key) {
+    return static_cast<size_t>(key.hashValue());
+  }
+
+  unsigned getUnversionedInfoSize(const FieldInfo &FI) {
+    return getVariableInfoSize(FI);
+  }
+
+  void emitUnversionedInfo(raw_ostream &OS, const FieldInfo &FI) {
+    emitVariableInfo(OS, FI);
+  }
+};
+} // namespace
+
+void APINotesWriter::Implementation::writeFieldBlock(
+    llvm::BitstreamWriter &Stream) {
+  llvm::BCBlockRAII Scope(Stream, FIELD_BLOCK_ID, 3);
+
+  if (Fields.empty())
+    return;
+
+  {
+    llvm::SmallString<4096> HashTableBlob;
+    uint32_t Offset;
+    {
+      llvm::OnDiskChainedHashTableGenerator<FieldTableInfo> Generator;
+      for (auto &FD : Fields)
+        Generator.insert(FD.first, FD.second);
+
+      llvm::raw_svector_ostream BlobStream(HashTableBlob);
+      // Make sure that no bucket is at offset 0
+      llvm::support::endian::write<uint32_t>(BlobStream, 0,
+                                             llvm::endianness::little);
+      Offset = Generator.Emit(BlobStream);
+    }
+
+    field_block::FieldDataLayout FieldData(Stream);
+    FieldData.emit(Scratch, Offset, HashTableBlob);
+  }
+}
+
 namespace {
 /// Used to serialize the on-disk Objective-C selector table.
 class ObjCSelectorTableInfo {
@@ -1423,6 +1488,14 @@ void APINotesWriter::addCXXMethod(ContextID CtxID, llvm::StringRef Name,
   Implementation->CXXMethods[Key].push_back({SwiftVersion, Info});
 }
 
+void APINotesWriter::addField(ContextID CtxID, llvm::StringRef Name,
+                              const FieldInfo &Info,
+                              VersionTuple SwiftVersion) {
+  IdentifierID NameID = Implementation->getIdentifier(Name);
+  SingleDeclTableKey Key(CtxID.Value, NameID);
+  Implementation->Fields[Key].push_back({SwiftVersion, Info});
+}
+
 void APINotesWriter::addGlobalVariable(std::optional<Context> Ctx,
                                        llvm::StringRef Name,
                                        const GlobalVariableInfo &Info,
diff --git a/clang/lib/APINotes/APINotesYAMLCompiler.cpp b/clang/lib/APINotes/APINotesYAMLCompiler.cpp
index 2205686c4d15c3f..16fd59244086fd4 100644
--- a/clang/lib/APINotes/APINotesYAMLCompiler.cpp
+++ b/clang/lib/APINotes/APINotesYAMLCompiler.cpp
@@ -405,6 +405,38 @@ template <> struct ScalarEnumerationTraits<EnumConvenienceAliasKind> {
 } // namespace yaml
 } // namespace llvm
 
+namespace {
+struct Field {
+  StringRef Name;
+  std::optional<NullabilityKind> Nullability;
+  AvailabilityItem Availability;
+  std::optional<bool> SwiftPrivate;
+  StringRef SwiftName;
+  StringRef Type;
+};
+
+typedef std::vector<Field> FieldsSeq;
+} // namespace
+
+LLVM_YAML_IS_SEQUENCE_VECTOR(Field)
+
+namespace llvm {
+namespace yaml {
+template <> struct MappingTraits<Field> {
+  static void mapping(IO &IO, Field &F) {
+    IO.mapRequired("Name", F.Name);
+    IO.mapOptional("Nullability", F.Nullability, std::nullopt);
+    IO.mapOptional("Availability", F.Availability.Mode,
+                   APIAvailability::Available);
+    IO.mapOptional("AvailabilityMsg", F.Availability.Msg, StringRef(""));
+    IO.mapOptional("SwiftPrivate", F.SwiftPrivate);
+    IO.mapOptional("SwiftName", F.SwiftName, StringRef(""));
+    IO.mapOptional("Type", F.Type, StringRef(""));
+  }
+};
+} // namespace yaml
+} // namespace llvm
+
 namespace {
 struct Tag;
 typedef std::vector<Tag> TagsSeq;
@@ -425,6 +457,7 @@ struct Tag {
   std::optional<EnumConvenienceAliasKind> EnumConvenienceKind;
   std::optional<bool> SwiftCopyable;
   FunctionsSeq Methods;
+  FieldsSeq Fields;
 
   /// Tags that are declared within the current tag. Only the tags that have
   /// corresponding API Notes will be listed.
@@ -463,6 +496,7 @@ template <> struct MappingTraits<Tag> {
     IO.mapOptional("EnumKind", T.EnumConvenienceKind);
     IO.mapOptional("SwiftCopyable", T.SwiftCopyable);
     IO.mapOptional("Methods", T.Methods);
+    IO.mapOptional("Fields", T.Fields);
     IO.mapOptional("Tags", T.Tags);
   }
 };
@@ -793,6 +827,16 @@ class YAMLConverter {
                          SwiftVersion);
   }
 
+  template <typename T>
+  void convertVariable(const T &Entity, VariableInfo &VI) {
+    convertAvailability(Entity.Availability, VI, Entity.Name);
+    VI.setSwiftPrivate(Entity.SwiftPrivate);
+    VI.SwiftName = std::string(Entity.SwiftName);
+    if (Entity.Nullability)
+      VI.setNullabilityAudited(*Entity.Nullability);
+    VI.setType(std::string(Entity.Type));
+  }
+
   void convertContext(std::optional<ContextID> ParentContextID, const Class &C,
                       ContextKind Kind, VersionTuple SwiftVersion) {
     // Write the class.
@@ -848,14 +892,9 @@ class YAMLConverter {
 
       // Translate from Property into ObjCPropertyInfo.
       ObjCPropertyInfo PI;
-      convertAvailability(Property.Availability, PI, Property.Name);
-      PI.setSwiftPrivate(Property.SwiftPrivate);
-      PI.SwiftName = std::string(Property.SwiftName);
-      if (Property.Nullability)
-        PI.setNullabilityAudited(*Property.Nullability);
+      convertVariable(Property, PI);
       if (Property.SwiftImportAsAccessors)
         PI.setSwiftImportAsAccessors(*Property.SwiftImportAsAccessors);
-      PI.setType(std::string(Property.Type));
 
       // Add both instance and class properties with this name.
       if (Property.Kind) {
@@ -970,6 +1009,12 @@ class YAMLConverter {
                                       CI, SwiftVersion);
     Context TagCtx(TagCtxID, ContextKind::Tag);
 
+    for (const auto &Field : T.Fields) {
+      FieldInfo FI;
+      convertVariable(Field, FI);
+      Writer.addField(TagCtxID, Field.Name, FI, SwiftVersion);
+    }
+
     for (const auto &CXXMethod : T.Methods) {
       CXXMethodInfo MI;
       convertFunction(CXXMethod, MI);
@@ -1037,12 +1082,7 @@ class YAMLConverter {
       }
 
       GlobalVariableInfo GVI;
-      convertAvailability(Global.Availability, GVI, Global.Name);
-      GVI.setSwiftPrivate(Global.SwiftPrivate);
-      GVI.SwiftName = std::string(Global.SwiftName);
-      if (Global.Nullability)
-        GVI.setNullabilityAudited(*Global.Nullability);
-      GVI.setType(std::string(Global.Type));
+      convertVariable(Global, GVI);
       Writer.addGlobalVariable(Ctx, Global.Name, GVI, SwiftVersion);
     }
 
diff --git a/clang/lib/Sema/SemaAPINotes.cpp b/clang/lib/Sema/SemaAPINotes.cpp
index 65b56bd1c8efc73..d0236d08c98e68d 100644
--- a/clang/lib/Sema/SemaAPINotes.cpp
+++ b/clang/lib/Sema/SemaAPINotes.cpp
@@ -433,6 +433,15 @@ static void ProcessAPINotes(Sema &S, VarDecl *D,
                   metadata);
 }
 
+/// Process API notes for a C field.
+static void ProcessAPINotes(Sema &S, FieldDecl *D,
+                            const api_notes::FieldInfo &Info,
+                            VersionedInfoMetadata metadata) {
+  // Handle common entity information.
+  ProcessAPINotes(S, D, static_cast<const api_notes::VariableInfo &>(Info),
+                  metadata);
+}
+
 /// Process API notes for an Objective-C property.
 static void ProcessAPINotes(Sema &S, ObjCPropertyDecl *D,
                             const api_notes::ObjCPropertyInfo &Info,
@@ -1062,6 +1071,17 @@ void Sema::ProcessAPINotes(Decl *D) {
       }
     }
 
+    if (auto Field = dyn_cast<FieldDecl>(D)) {
+      if (!Field->isUnnamedBitField() && !Field->isAnonymousStructOrUnion()) {
+        for (auto Reader : APINotes.findAPINotes(D->getLocation())) {
+          if (auto Context = UnwindTagContext(TagContext, APINotes)) {
+            auto Info = Reader->lookupField(Context->id, Field->getName());
+            ProcessVersionedAPINotes(*this, Field, Info);
+          }
+        }
+      }
+    }
+
     if (auto Tag = dyn_cast<TagDecl>(D)) {
       for (auto Reader : APINotes.findAPINotes(D->getLocation())) {
         if (auto Context = UnwindTagContext(TagContext, APINotes)) {
diff --git a/clang/test/APINotes/Inputs/Headers/Fields.apinotes b/clang/test/APINotes/Inputs/Headers/Fields.apinotes
new file mode 100644
index 000000000000000..931da52ba29d183
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Headers/Fields.apinotes
@@ -0,0 +1,23 @@
+---
+Name: Fields
+Tags:
+- Name: IntWrapper
+  Fields:
+  - Name: value
+    Availability: none
+    AvailabilityMsg: "oh no"
+- Name: Outer
+  Tags:
+  - Name: Inner
+    Fields:
+    - Name: value
+      Availability: none
+      AvailabilityMsg: "oh no 2"
+  Methods:
+  - Name: value
+    Availability: none
+    AvailabilityMsg: "this should have no effect"  
+Functions:
+- Name: value
+  Availability: none
+  AvailabilityMsg: "this should have no effect"
diff --git a/clang/test/APINotes/Inputs/Headers/Fields.h b/clang/test/APINotes/Inputs/Headers/Fields.h
new file mode 100644
index 000000000000000..dbd342da4778922
--- /dev/null
+++ b/clang/test/APINotes/Inputs/Headers/Fields.h
@@ -0,0 +1,21 @@
+struct IntWrapper {
+  int value;
+
+  bool : 1;
+
+  enum {
+    one,
+    two,
+    three
+  };
+  union {
+    int a;
+    char b;
+  };
+};
+
+struct Outer {
+  struct Inner {
+    int value;
+  };
+};
diff --git a/clang/test/APINotes/Inputs/Headers/module.modulemap b/clang/test/APINotes/Inputs/Headers/module.modulemap
index faf6042c78d576e..7d304863398ecfc 100644
--- a/clang/test/APINotes/Inputs/Headers/module.modulemap
+++ b/clang/test/APINotes/Inputs/Headers/module.modulemap
@@ -12,6 +12,11 @@ module ExportAs {
   export *
 }
 
+module Fields {
+  header "Fields.h"
+  export *
+}
+
 module HeaderLib {
   header "HeaderLib.h"
 }
diff --git a/clang/test/APINotes/fields.cpp b/clang/test/APINotes/fields.cpp
new file mode 100644
index 000000000000000..8dea2229b4e0a27
--- /dev/null
+++ b/clang/test/APINotes/fields.cpp
@@ -0,0 +1,17 @@
+// RUN: rm -rf %t && mkdir -p %t
+// RUN: %clang_cc1 -fmodules -fblocks -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache/Fields -fdisable-module-hash -fapinotes-modules -fsyntax-only -I %S/Inputs/Headers -F %S/Inputs/Frameworks %s -x c++
+// RUN: %clang_cc1 -fmodules -fblocks -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache/Fields -fdisable-module-hash -fapinotes-modules -I %S/Inputs/Headers -F %S/Inputs/Frameworks %s -ast-dump -ast-dump-filter IntWrapper::value -x c++ | FileCheck --check-prefix=CHECK-FIELD %s
+// RUN: %clang_cc1 -fmodules -fblocks -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache/Fields -fdisable-module-hash -fapinotes-modules -I %S/Inputs/Headers -F %S/Inputs/Frameworks %s -ast-dump -ast-dump-filter Outer::Inner::value -x c++ | FileCheck --check-prefix=CHECK-DEEP-FIELD %s
+
+#include "Fields.h"
+
+// CHECK-FIELD: Dumping IntWrapper::value:
+// CHECK-FIELD-NEXT: FieldDecl {{.+}} value
+// CHECK-FIELD: UnavailableAttr {{.+}} <<invalid sloc>> "oh no"
+
+// CHECK-DEEP-FIELD: Dumping Outer::Inner::value:
+// CHECK-DEEP-FIELD-NEXT: FieldDecl {{.+}} value
+// CHECK-DEEP-FIELD: UnavailableAttr {{.+}} <<invalid sloc>> "oh no 2"
+
+// CHECK-FIELD-NOT: this should have no effect
+// CHECK-DEEP-FIELD-NOT: this should have no effect

From 9f89d31d5185015f8eea9c0f3d35e7ba9d353e67 Mon Sep 17 00:00:00 2001
From: Kendal Harland <3987220+kendalharland@users.noreply.github.com>
Date: Fri, 16 Aug 2024 10:54:25 -0700
Subject: [PATCH 208/441] [lldb][test] Mark gtest cases as XFAIL if the test
 suite is XFAIL (#102986)

When a test case inside of a gtest suite fails, we report a failure
which causes the entire `ninja check-lldb` invocation to fail, even if
the outer test case is marked as XFAIL - each test case result is
reported as its own lit test run. This PR updates lit so it checks
whether each test case's parent test suite is XFAIL before setting the
status to FAIL.

This is especially problematic because the failing tests can't manually
be marked as XFAIL, due to
https://github.com/llvm/llvm-project/issues/102264.

Fixes https://github.com/llvm/llvm-project/issues/102265

### Repro instructions

1. Modify any gtest test case to generate a failure.
2. Mark the outer lit test with XFAIL using either `--xfail-tests` flag
or `LIT_XFAIL` env var.
3. Run the tests
4. Observe the lit test is XFAIL as expected, but the failed child test
cases show up as separate failures.

Co-authored-by: kendal <kendal@thebrowser.company>
---
 llvm/utils/lit/lit/formats/googletest.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/llvm/utils/lit/lit/formats/googletest.py b/llvm/utils/lit/lit/formats/googletest.py
index 8037094a910678a..6c29cca891581fa 100644
--- a/llvm/utils/lit/lit/formats/googletest.py
+++ b/llvm/utils/lit/lit/formats/googletest.py
@@ -334,7 +334,11 @@ def remove_gtest(tests):
                             returnCode = lit.Test.SKIPPED
                         elif "failures" in testinfo:
                             has_failure_in_shard = True
-                            returnCode = lit.Test.FAIL
+                            returnCode = (
+                                lit.Test.XFAIL
+                                if test.isExpectedToFail()
+                                else lit.Test.FAIL
+                            )
                             output = header
                             for fail in testinfo["failures"]:
                                 output += fail["failure"] + "\n"

From 0551926fda739c1023a41978efb55047013e70d6 Mon Sep 17 00:00:00 2001
From: Shilei Tian <i@tianshilei.me>
Date: Fri, 16 Aug 2024 13:59:46 -0400
Subject: [PATCH 209/441] [Clang][OMPX] Add the code generation for multi-dim
 `thread_limit` clause (#102717)

---
 clang/lib/CodeGen/CGOpenMPRuntime.cpp         | 33 ++++++++++-------
 clang/test/OpenMP/target_teams_codegen.cpp    | 12 +++---
 .../llvm/Frontend/OpenMP/OMPIRBuilder.h       | 26 ++++++-------
 llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp     | 37 +++++++++++--------
 4 files changed, 61 insertions(+), 47 deletions(-)

diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
index 8c5e4aa9c037e2b..34120486996fb17 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -9588,15 +9588,18 @@ static void genMapInfo(const OMPExecutableDirective &D, CodeGenFunction &CGF,
   genMapInfo(MEHandler, CGF, CombinedInfo, OMPBuilder, MappedVarSet);
 }
 
-static void emitNumTeamsForBareTargetDirective(
-    CodeGenFunction &CGF, const OMPExecutableDirective &D,
-    llvm::SmallVectorImpl<llvm::Value *> &NumTeams) {
-  const auto *C = D.getSingleClause<OMPNumTeamsClause>();
-  assert(!C->varlist_empty() && "ompx_bare requires explicit num_teams");
-  CodeGenFunction::RunCleanupsScope NumTeamsScope(CGF);
-  for (auto *E : C->getNumTeams()) {
+template <typename ClauseTy>
+static void
+emitClauseForBareTargetDirective(CodeGenFunction &CGF,
+                                 const OMPExecutableDirective &D,
+                                 llvm::SmallVectorImpl<llvm::Value *> &Values) {
+  const auto *C = D.getSingleClause<ClauseTy>();
+  assert(!C->varlist_empty() &&
+         "ompx_bare requires explicit num_teams and thread_limit");
+  CodeGenFunction::RunCleanupsScope Scope(CGF);
+  for (auto *E : C->varlist()) {
     llvm::Value *V = CGF.EmitScalarExpr(E);
-    NumTeams.push_back(
+    Values.push_back(
         CGF.Builder.CreateIntCast(V, CGF.Int32Ty, /*isSigned=*/true));
   }
 }
@@ -9672,14 +9675,18 @@ static void emitTargetCallKernelLaunch(
 
     bool IsBare = D.hasClausesOfKind<OMPXBareClause>();
     SmallVector<llvm::Value *, 3> NumTeams;
-    if (IsBare)
-      emitNumTeamsForBareTargetDirective(CGF, D, NumTeams);
-    else
+    SmallVector<llvm::Value *, 3> NumThreads;
+    if (IsBare) {
+      emitClauseForBareTargetDirective<OMPNumTeamsClause>(CGF, D, NumTeams);
+      emitClauseForBareTargetDirective<OMPThreadLimitClause>(CGF, D,
+                                                             NumThreads);
+    } else {
       NumTeams.push_back(OMPRuntime->emitNumTeamsForTargetDirective(CGF, D));
+      NumThreads.push_back(
+          OMPRuntime->emitNumThreadsForTargetDirective(CGF, D));
+    }
 
     llvm::Value *DeviceID = emitDeviceID(Device, CGF);
-    llvm::Value *NumThreads =
-        OMPRuntime->emitNumThreadsForTargetDirective(CGF, D);
     llvm::Value *RTLoc = OMPRuntime->emitUpdateLocation(CGF, D.getBeginLoc());
     llvm::Value *NumIterations =
         OMPRuntime->emitTargetNumIterationsCall(CGF, D, SizeEmitter);
diff --git a/clang/test/OpenMP/target_teams_codegen.cpp b/clang/test/OpenMP/target_teams_codegen.cpp
index 9cab8eef1488330..13d44e127201bda 100644
--- a/clang/test/OpenMP/target_teams_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_codegen.cpp
@@ -127,13 +127,13 @@ int foo(int n) {
     aa += 1;
   }
 
-  #pragma omp target teams ompx_bare num_teams(1, 2) thread_limit(1)
+  #pragma omp target teams ompx_bare num_teams(1, 2) thread_limit(1, 2)
   {
     a += 1;
     aa += 1;
   }
 
-  #pragma omp target teams ompx_bare num_teams(1, 2, 3) thread_limit(1)
+  #pragma omp target teams ompx_bare num_teams(1, 2, 3) thread_limit(1, 2, 3)
   {
     a += 1;
     aa += 1;
@@ -667,7 +667,7 @@ int bar(int n){
 // CHECK1-NEXT:    [[TMP144:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS29]], i32 0, i32 10
 // CHECK1-NEXT:    store [3 x i32] [i32 1, i32 2, i32 0], ptr [[TMP144]], align 4
 // CHECK1-NEXT:    [[TMP145:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS29]], i32 0, i32 11
-// CHECK1-NEXT:    store [3 x i32] [i32 1, i32 0, i32 0], ptr [[TMP145]], align 4
+// CHECK1-NEXT:    store [3 x i32] [i32 1, i32 2, i32 0], ptr [[TMP145]], align 4
 // CHECK1-NEXT:    [[TMP146:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS29]], i32 0, i32 12
 // CHECK1-NEXT:    store i32 0, ptr [[TMP146]], align 4
 // CHECK1-NEXT:    [[TMP147:%.*]] = call i32 @__tgt_target_kernel(ptr @[[GLOB1]], i64 -1, i32 1, i32 1, ptr @.{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3fooi_l130.region_id, ptr [[KERNEL_ARGS29]])
@@ -720,7 +720,7 @@ int bar(int n){
 // CHECK1-NEXT:    [[TMP171:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS37]], i32 0, i32 10
 // CHECK1-NEXT:    store [3 x i32] [i32 1, i32 2, i32 3], ptr [[TMP171]], align 4
 // CHECK1-NEXT:    [[TMP172:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS37]], i32 0, i32 11
-// CHECK1-NEXT:    store [3 x i32] [i32 1, i32 0, i32 0], ptr [[TMP172]], align 4
+// CHECK1-NEXT:    store [3 x i32] [i32 1, i32 2, i32 3], ptr [[TMP172]], align 4
 // CHECK1-NEXT:    [[TMP173:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS37]], i32 0, i32 12
 // CHECK1-NEXT:    store i32 0, ptr [[TMP173]], align 4
 // CHECK1-NEXT:    [[TMP174:%.*]] = call i32 @__tgt_target_kernel(ptr @[[GLOB1]], i64 -1, i32 1, i32 1, ptr @.{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3fooi_l136.region_id, ptr [[KERNEL_ARGS37]])
@@ -2458,7 +2458,7 @@ int bar(int n){
 // CHECK3-NEXT:    [[TMP142:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS29]], i32 0, i32 10
 // CHECK3-NEXT:    store [3 x i32] [i32 1, i32 2, i32 0], ptr [[TMP142]], align 4
 // CHECK3-NEXT:    [[TMP143:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS29]], i32 0, i32 11
-// CHECK3-NEXT:    store [3 x i32] [i32 1, i32 0, i32 0], ptr [[TMP143]], align 4
+// CHECK3-NEXT:    store [3 x i32] [i32 1, i32 2, i32 0], ptr [[TMP143]], align 4
 // CHECK3-NEXT:    [[TMP144:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS29]], i32 0, i32 12
 // CHECK3-NEXT:    store i32 0, ptr [[TMP144]], align 4
 // CHECK3-NEXT:    [[TMP145:%.*]] = call i32 @__tgt_target_kernel(ptr @[[GLOB1]], i64 -1, i32 1, i32 1, ptr @.{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3fooi_l130.region_id, ptr [[KERNEL_ARGS29]])
@@ -2511,7 +2511,7 @@ int bar(int n){
 // CHECK3-NEXT:    [[TMP169:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS37]], i32 0, i32 10
 // CHECK3-NEXT:    store [3 x i32] [i32 1, i32 2, i32 3], ptr [[TMP169]], align 4
 // CHECK3-NEXT:    [[TMP170:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS37]], i32 0, i32 11
-// CHECK3-NEXT:    store [3 x i32] [i32 1, i32 0, i32 0], ptr [[TMP170]], align 4
+// CHECK3-NEXT:    store [3 x i32] [i32 1, i32 2, i32 3], ptr [[TMP170]], align 4
 // CHECK3-NEXT:    [[TMP171:%.*]] = getelementptr inbounds nuw [[STRUCT___TGT_KERNEL_ARGUMENTS]], ptr [[KERNEL_ARGS37]], i32 0, i32 12
 // CHECK3-NEXT:    store i32 0, ptr [[TMP171]], align 4
 // CHECK3-NEXT:    [[TMP172:%.*]] = call i32 @__tgt_target_kernel(ptr @[[GLOB1]], i64 -1, i32 1, i32 1, ptr @.{{__omp_offloading_[0-9a-z]+_[0-9a-z]+}}__Z3fooi_l136.region_id, ptr [[KERNEL_ARGS37]])
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
index 9e4e7ebf2a5703e..4be0159fb1dd9f2 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
@@ -2195,7 +2195,7 @@ class OpenMPIRBuilder {
     /// The number of teams.
     ArrayRef<Value *> NumTeams;
     /// The number of threads.
-    Value *NumThreads = nullptr;
+    ArrayRef<Value *> NumThreads;
     /// The size of the dynamic shared memory.
     Value *DynCGGroupMem = nullptr;
     /// True if the kernel has 'no wait' clause.
@@ -2205,7 +2205,8 @@ class OpenMPIRBuilder {
     TargetKernelArgs() {}
     TargetKernelArgs(unsigned NumTargetItems, TargetDataRTArgs RTArgs,
                      Value *NumIterations, ArrayRef<Value *> NumTeams,
-                     Value *NumThreads, Value *DynCGGroupMem, bool HasNoWait)
+                     ArrayRef<Value *> NumThreads, Value *DynCGGroupMem,
+                     bool HasNoWait)
         : NumTargetItems(NumTargetItems), RTArgs(RTArgs),
           NumIterations(NumIterations), NumTeams(NumTeams),
           NumThreads(NumThreads), DynCGGroupMem(DynCGGroupMem),
@@ -2852,17 +2853,16 @@ class OpenMPIRBuilder {
   /// instructions for passed in target arguments where neccessary
   /// \param Dependencies A vector of DependData objects that carry
   // dependency information as passed in the depend clause
-  InsertPointTy createTarget(const LocationDescription &Loc,
-                             bool IsOffloadEntry,
-                             OpenMPIRBuilder::InsertPointTy AllocaIP,
-                             OpenMPIRBuilder::InsertPointTy CodeGenIP,
-                             TargetRegionEntryInfo &EntryInfo,
-                             ArrayRef<int32_t> NumTeams, int32_t NumThreads,
-                             SmallVectorImpl<Value *> &Inputs,
-                             GenMapInfoCallbackTy GenMapInfoCB,
-                             TargetBodyGenCallbackTy BodyGenCB,
-                             TargetGenArgAccessorsCallbackTy ArgAccessorFuncCB,
-                             SmallVector<DependData> Dependencies = {});
+  InsertPointTy
+  createTarget(const LocationDescription &Loc, bool IsOffloadEntry,
+               OpenMPIRBuilder::InsertPointTy AllocaIP,
+               OpenMPIRBuilder::InsertPointTy CodeGenIP,
+               TargetRegionEntryInfo &EntryInfo, ArrayRef<int32_t> NumTeams,
+               ArrayRef<int32_t> NumThreads, SmallVectorImpl<Value *> &Inputs,
+               GenMapInfoCallbackTy GenMapInfoCB,
+               TargetBodyGenCallbackTy BodyGenCB,
+               TargetGenArgAccessorsCallbackTy ArgAccessorFuncCB,
+               SmallVector<DependData> Dependencies = {});
 
   /// Returns __kmpc_for_static_init_* runtime function for the specified
   /// size \a IVSize and sign \a IVSigned. Will create a distribute call
diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index f9b070e6f1eae4e..bba3c5f3229f5cf 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -501,15 +501,20 @@ void OpenMPIRBuilder::getKernelArgsVector(TargetKernelArgs &KernelArgs,
   Value *ZeroArray = Constant::getNullValue(ArrayType::get(Int32Ty, MaxDim));
   Value *Flags = Builder.getInt64(KernelArgs.HasNoWait);
 
-  assert(!KernelArgs.NumTeams.empty());
+  assert(!KernelArgs.NumTeams.empty() && !KernelArgs.NumThreads.empty());
 
   Value *NumTeams3D =
       Builder.CreateInsertValue(ZeroArray, KernelArgs.NumTeams[0], {0});
-  for (unsigned I = 1; I < std::min(KernelArgs.NumTeams.size(), MaxDim); ++I)
+  Value *NumThreads3D =
+      Builder.CreateInsertValue(ZeroArray, KernelArgs.NumThreads[0], {0});
+  for (unsigned I :
+       seq<unsigned>(1, std::min(KernelArgs.NumTeams.size(), MaxDim)))
     NumTeams3D =
         Builder.CreateInsertValue(NumTeams3D, KernelArgs.NumTeams[I], {I});
-  Value *NumThreads3D =
-      Builder.CreateInsertValue(ZeroArray, KernelArgs.NumThreads, {0});
+  for (unsigned I :
+       seq<unsigned>(1, std::min(KernelArgs.NumThreads.size(), MaxDim)))
+    NumThreads3D =
+        Builder.CreateInsertValue(NumThreads3D, KernelArgs.NumThreads[I], {I});
 
   ArgsVector = {Version,
                 PointerNum,
@@ -1114,9 +1119,9 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitKernelLaunch(
   // __tgt_target_teams() launches a GPU kernel with the requested number
   // of teams and threads so no additional calls to the runtime are required.
   // Check the error code and execute the host version if required.
-  Builder.restoreIP(emitTargetKernel(Builder, AllocaIP, Return, RTLoc, DeviceID,
-                                     Args.NumTeams.front(), Args.NumThreads,
-                                     OutlinedFnID, ArgsVector));
+  Builder.restoreIP(emitTargetKernel(
+      Builder, AllocaIP, Return, RTLoc, DeviceID, Args.NumTeams.front(),
+      Args.NumThreads.front(), OutlinedFnID, ArgsVector));
 
   BasicBlock *OffloadFailedBlock =
       BasicBlock::Create(Builder.getContext(), "omp_offload.failed");
@@ -7075,8 +7080,8 @@ void OpenMPIRBuilder::emitOffloadingArraysAndArgs(
 static void emitTargetCall(
     OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder,
     OpenMPIRBuilder::InsertPointTy AllocaIP, Function *OutlinedFn,
-    Constant *OutlinedFnID, ArrayRef<int32_t> NumTeams, int32_t NumThreads,
-    SmallVectorImpl<Value *> &Args,
+    Constant *OutlinedFnID, ArrayRef<int32_t> NumTeams,
+    ArrayRef<int32_t> NumThreads, SmallVectorImpl<Value *> &Args,
     OpenMPIRBuilder::GenMapInfoCallbackTy GenMapInfoCB,
     SmallVector<llvm::OpenMPIRBuilder::DependData> Dependencies = {}) {
   // Generate a function call to the host fallback implementation of the target
@@ -7123,13 +7128,15 @@ static void emitTargetCall(
                                          /*ForEndCall=*/false);
 
   SmallVector<Value *, 3> NumTeamsC;
+  SmallVector<Value *, 3> NumThreadsC;
   for (auto V : NumTeams)
     NumTeamsC.push_back(llvm::ConstantInt::get(Builder.getInt32Ty(), V));
+  for (auto V : NumThreads)
+    NumThreadsC.push_back(llvm::ConstantInt::get(Builder.getInt32Ty(), V));
 
   unsigned NumTargetItems = Info.NumberOfPtrs;
   // TODO: Use correct device ID
   Value *DeviceID = Builder.getInt64(OMP_DEVICEID_UNDEF);
-  Value *NumThreadsVal = Builder.getInt32(NumThreads);
   uint32_t SrcLocStrSize;
   Constant *SrcLocStr = OMPBuilder.getOrCreateDefaultSrcLocStr(SrcLocStrSize);
   Value *RTLoc = OMPBuilder.getOrCreateIdent(SrcLocStr, SrcLocStrSize,
@@ -7140,8 +7147,8 @@ static void emitTargetCall(
   Value *DynCGGroupMem = Builder.getInt32(0);
 
   OpenMPIRBuilder::TargetKernelArgs KArgs(NumTargetItems, RTArgs, NumIterations,
-                                          NumTeamsC, NumThreadsVal,
-                                          DynCGGroupMem, HasNoWait);
+                                          NumTeamsC, NumThreadsC, DynCGGroupMem,
+                                          HasNoWait);
 
   // The presence of certain clauses on the target directive require the
   // explicit generation of the target task.
@@ -7159,11 +7166,11 @@ static void emitTargetCall(
 OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createTarget(
     const LocationDescription &Loc, bool IsOffloadEntry, InsertPointTy AllocaIP,
     InsertPointTy CodeGenIP, TargetRegionEntryInfo &EntryInfo,
-    ArrayRef<int32_t> NumTeams, int32_t NumThreads,
+    ArrayRef<int32_t> NumTeams, ArrayRef<int32_t> NumThreads,
     SmallVectorImpl<Value *> &Args, GenMapInfoCallbackTy GenMapInfoCB,
     OpenMPIRBuilder::TargetBodyGenCallbackTy CBFunc,
     OpenMPIRBuilder::TargetGenArgAccessorsCallbackTy ArgAccessorFuncCB,
-    SmallVector<DependData> Dependenciess) {
+    SmallVector<DependData> Dependencies) {
 
   if (!updateToLocation(Loc))
     return InsertPointTy();
@@ -7184,7 +7191,7 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createTarget(
   // that represents the target region. Do that now.
   if (!Config.isTargetDevice())
     emitTargetCall(*this, Builder, AllocaIP, OutlinedFn, OutlinedFnID, NumTeams,
-                   NumThreads, Args, GenMapInfoCB, Dependenciess);
+                   NumThreads, Args, GenMapInfoCB, Dependencies);
   return Builder.saveIP();
 }
 

From 89a1f144f3890d9bd8ab30537b36f8c18038719b Mon Sep 17 00:00:00 2001
From: Peter Rong <peterrong96@gmail.com>
Date: Fri, 16 Aug 2024 11:00:11 -0700
Subject: [PATCH 210/441] [LLD, MachO] Default objc_relative_method_lists on
 MacOS10.16+/iOS14+ (#104519)

This patch makes `-objc_relative_method_lists` default on MacOS
10.16+/iOS 14+. Manual override still work if command line argument is
provided.

To test this change, many explict arguments are removed from the test
files. Some explict `-objc_no_objc_relative_method_lists` are also added
for tests that don't support this yet.

This commit tries to revive #101360, which exposes a bug that breaks CI.
#104081 has fixed that bug.
---
 lld/MachO/Driver.cpp                          | 98 +++++++++++--------
 lld/test/MachO/objc-category-conflicts.s      | 16 +--
 .../objc-category-merging-complete-test.s     | 10 +-
 ...jc-category-merging-erase-objc-name-test.s |  2 +-
 .../MachO/objc-category-merging-minimal.s     |  6 +-
 lld/test/MachO/objc-category-merging-swift.s  |  2 +-
 .../MachO/objc-relative-method-lists-simple.s | 16 ++-
 7 files changed, 85 insertions(+), 65 deletions(-)

diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp
index f3d2a93914f7176..6a1ff96ed656975 100644
--- a/lld/MachO/Driver.cpp
+++ b/lld/MachO/Driver.cpp
@@ -1042,20 +1042,36 @@ static bool shouldAdhocSignByDefault(Architecture arch, PlatformType platform) {
          platform == PLATFORM_XROS_SIMULATOR;
 }
 
-static bool dataConstDefault(const InputArgList &args) {
-  static const std::array<std::pair<PlatformType, VersionTuple>, 6> minVersion =
-      {{{PLATFORM_MACOS, VersionTuple(10, 15)},
-        {PLATFORM_IOS, VersionTuple(13, 0)},
-        {PLATFORM_TVOS, VersionTuple(13, 0)},
-        {PLATFORM_WATCHOS, VersionTuple(6, 0)},
-        {PLATFORM_XROS, VersionTuple(1, 0)},
-        {PLATFORM_BRIDGEOS, VersionTuple(4, 0)}}};
-  PlatformType platform = removeSimulator(config->platformInfo.target.Platform);
-  auto it = llvm::find_if(minVersion,
+template <std::size_t N>
+using MinVersions = std::array<std::pair<PlatformType, VersionTuple>, N>;
+
+/// Returns true if the platform is greater than the min version.
+/// Returns false if the platform does not exist.
+template <std::size_t N>
+static bool greaterEqMinVersion(const MinVersions<N> &minVersions,
+                                bool ignoreSimulator) {
+  PlatformType platform = config->platformInfo.target.Platform;
+  if (ignoreSimulator)
+    platform = removeSimulator(platform);
+  auto it = llvm::find_if(minVersions,
                           [&](const auto &p) { return p.first == platform; });
-  if (it != minVersion.end())
-    if (config->platformInfo.target.MinDeployment < it->second)
-      return false;
+  if (it != minVersions.end())
+    if (config->platformInfo.target.MinDeployment >= it->second)
+      return true;
+  return false;
+}
+
+static bool dataConstDefault(const InputArgList &args) {
+  static const MinVersions<6> minVersion = {{
+      {PLATFORM_MACOS, VersionTuple(10, 15)},
+      {PLATFORM_IOS, VersionTuple(13, 0)},
+      {PLATFORM_TVOS, VersionTuple(13, 0)},
+      {PLATFORM_WATCHOS, VersionTuple(6, 0)},
+      {PLATFORM_XROS, VersionTuple(1, 0)},
+      {PLATFORM_BRIDGEOS, VersionTuple(4, 0)},
+  }};
+  if (!greaterEqMinVersion(minVersion, true))
+    return false;
 
   switch (config->outputType) {
   case MH_EXECUTE:
@@ -1106,30 +1122,18 @@ static bool shouldEmitChainedFixups(const InputArgList &args) {
   if (requested)
     return true;
 
-  static const std::array<std::pair<PlatformType, VersionTuple>, 9> minVersion =
-      {{
-          {PLATFORM_IOS, VersionTuple(13, 4)},
-          {PLATFORM_IOSSIMULATOR, VersionTuple(16, 0)},
-          {PLATFORM_MACOS, VersionTuple(13, 0)},
-          {PLATFORM_TVOS, VersionTuple(14, 0)},
-          {PLATFORM_TVOSSIMULATOR, VersionTuple(15, 0)},
-          {PLATFORM_WATCHOS, VersionTuple(7, 0)},
-          {PLATFORM_WATCHOSSIMULATOR, VersionTuple(8, 0)},
-          {PLATFORM_XROS, VersionTuple(1, 0)},
-          {PLATFORM_XROS_SIMULATOR, VersionTuple(1, 0)},
-      }};
-  PlatformType platform = config->platformInfo.target.Platform;
-  auto it = llvm::find_if(minVersion,
-                          [&](const auto &p) { return p.first == platform; });
-
-  // We don't know the versions for other platforms, so default to disabled.
-  if (it == minVersion.end())
-    return false;
-
-  if (it->second > config->platformInfo.target.MinDeployment)
-    return false;
-
-  return true;
+  static const MinVersions<9> minVersion = {{
+      {PLATFORM_IOS, VersionTuple(13, 4)},
+      {PLATFORM_IOSSIMULATOR, VersionTuple(16, 0)},
+      {PLATFORM_MACOS, VersionTuple(13, 0)},
+      {PLATFORM_TVOS, VersionTuple(14, 0)},
+      {PLATFORM_TVOSSIMULATOR, VersionTuple(15, 0)},
+      {PLATFORM_WATCHOS, VersionTuple(7, 0)},
+      {PLATFORM_WATCHOSSIMULATOR, VersionTuple(8, 0)},
+      {PLATFORM_XROS, VersionTuple(1, 0)},
+      {PLATFORM_XROS_SIMULATOR, VersionTuple(1, 0)},
+  }};
+  return greaterEqMinVersion(minVersion, false);
 }
 
 static bool shouldEmitRelativeMethodLists(const InputArgList &args) {
@@ -1140,12 +1144,20 @@ static bool shouldEmitRelativeMethodLists(const InputArgList &args) {
   if (arg && arg->getOption().getID() == OPT_no_objc_relative_method_lists)
     return false;
 
-  // TODO: If no flag is specified, don't default to false, but instead:
-  //   - default false on   <   ios14
-  //   - default true  on   >=  ios14
-  // For now, until this feature is confirmed stable, default to false if no
-  // flag is explicitly specified
-  return false;
+  // If no flag is specified, enable this on newer versions by default.
+  // The min versions is taken from
+  // ld64(https://github.com/apple-oss-distributions/ld64/blob/47f477cb721755419018f7530038b272e9d0cdea/src/ld/ld.hpp#L310)
+  // to mimic to operation of ld64
+  // [here](https://github.com/apple-oss-distributions/ld64/blob/47f477cb721755419018f7530038b272e9d0cdea/src/ld/Options.cpp#L6085-L6101)
+  static const MinVersions<6> minVersion = {{
+      {PLATFORM_MACOS, VersionTuple(10, 16)},
+      {PLATFORM_IOS, VersionTuple(14, 0)},
+      {PLATFORM_WATCHOS, VersionTuple(7, 0)},
+      {PLATFORM_TVOS, VersionTuple(14, 0)},
+      {PLATFORM_BRIDGEOS, VersionTuple(5, 0)},
+      {PLATFORM_XROS, VersionTuple(1, 0)},
+  }};
+  return greaterEqMinVersion(minVersion, true);
 }
 
 void SymbolPatterns::clear() {
diff --git a/lld/test/MachO/objc-category-conflicts.s b/lld/test/MachO/objc-category-conflicts.s
index eb4c0adfd9e934b..3fb81d7caf092c6 100644
--- a/lld/test/MachO/objc-category-conflicts.s
+++ b/lld/test/MachO/objc-category-conflicts.s
@@ -10,31 +10,31 @@
 # RUN: llvm-mc -filetype=obj -triple=x86_64-apple-macos11.0 -I %t %t/cat2.s --defsym MAKE_LOAD_METHOD=1 -o %t/cat2-with-load.o
 # RUN: llvm-mc -filetype=obj -triple=x86_64-apple-macos11.0 -I %t %t/klass.s --defsym MAKE_LOAD_METHOD=1 -o %t/klass-with-load.o
 # RUN: llvm-mc -filetype=obj -triple=x86_64-apple-macos11.0 -I %t %t/klass-with-no-rodata.s -o %t/klass-with-no-rodata.o
-# RUN: %lld -dylib -lobjc %t/klass.o -o %t/libklass.dylib
+# RUN: %lld -no_objc_relative_method_lists -dylib -lobjc %t/klass.o -o %t/libklass.dylib
 
-# RUN: %no-fatal-warnings-lld --check-category-conflicts -dylib -lobjc %t/klass.o %t/cat1.o %t/cat2.o -o \
+# RUN: %no-fatal-warnings-lld -no_objc_relative_method_lists --check-category-conflicts -dylib -lobjc %t/klass.o %t/cat1.o %t/cat2.o -o \
 # RUN:   /dev/null 2>&1 | FileCheck %s --check-prefixes=CATCLS,CATCAT
-# RUN: %no-fatal-warnings-lld --check-category-conflicts -dylib -lobjc %t/libklass.dylib %t/cat1.o \
+# RUN: %no-fatal-warnings-lld -no_objc_relative_method_lists --check-category-conflicts -dylib -lobjc %t/libklass.dylib %t/cat1.o \
 # RUN:   %t/cat2.o -o /dev/null 2>&1 | FileCheck %s --check-prefix=CATCAT
 
-# RUN: %no-fatal-warnings-lld --check-category-conflicts -dylib -lobjc %t/klass_w_sym.o %t/cat1_w_sym.o %t/cat2_w_sym.o -o \
+# RUN: %no-fatal-warnings-lld -no_objc_relative_method_lists --check-category-conflicts -dylib -lobjc %t/klass_w_sym.o %t/cat1_w_sym.o %t/cat2_w_sym.o -o \
 # RUN:   /dev/null 2>&1 | FileCheck %s --check-prefixes=CATCLS_W_SYM,CATCAT_W_SYM
-# RUN: %no-fatal-warnings-lld --check-category-conflicts -dylib -lobjc %t/libklass.dylib %t/cat1_w_sym.o \
+# RUN: %no-fatal-warnings-lld -no_objc_relative_method_lists --check-category-conflicts -dylib -lobjc %t/libklass.dylib %t/cat1_w_sym.o \
 # RUN:   %t/cat2_w_sym.o -o /dev/null 2>&1 | FileCheck %s --check-prefix=CATCAT_W_SYM
 
 ## Check that we don't emit spurious warnings around the +load method while
 ## still emitting the other warnings. Note that we have made separate
 ## `*-with-load.s` files for ease of comparison with ld64; ld64 will not warn
 ## at all if multiple +load methods are present.
-# RUN: %no-fatal-warnings-lld --check-category-conflicts -dylib -lobjc %t/klass-with-load.o \
+# RUN: %no-fatal-warnings-lld -no_objc_relative_method_lists --check-category-conflicts -dylib -lobjc %t/klass-with-load.o \
 # RUN:   %t/cat1-with-load.o %t/cat2-with-load.o -o /dev/null 2>&1 | \
 # RUN:   FileCheck %s --check-prefixes=CATCLS,CATCAT --implicit-check-not '+load'
 
 ## Regression test: Check that we don't crash.
-# RUN: %no-fatal-warnings-lld --check-category-conflicts -dylib -lobjc %t/klass-with-no-rodata.o -o /dev/null
+# RUN: %no-fatal-warnings-lld -no_objc_relative_method_lists --check-category-conflicts -dylib -lobjc %t/klass-with-no-rodata.o -o /dev/null
 
 ## Check that we don't emit any warnings without --check-category-conflicts.
-# RUN: %no-fatal-warnings-lld -dylib -lobjc %t/klass.o %t/cat1.o %t/cat2.o -o \
+# RUN: %no-fatal-warnings-lld -no_objc_relative_method_lists -dylib -lobjc %t/klass.o %t/cat1.o %t/cat2.o -o \
 # RUN:   /dev/null 2>&1 | FileCheck %s --implicit-check-not 'warning' --allow-empty
 
 # CATCLS:      warning: method '+s1' has conflicting definitions:
diff --git a/lld/test/MachO/objc-category-merging-complete-test.s b/lld/test/MachO/objc-category-merging-complete-test.s
index cb112073eb87103..3befd683e3a15ef 100644
--- a/lld/test/MachO/objc-category-merging-complete-test.s
+++ b/lld/test/MachO/objc-category-merging-complete-test.s
@@ -7,10 +7,10 @@
 # RUN: %lld -arch arm64 a64_file1.o -o a64_file1.dylib -dylib
 
 # RUN: llvm-mc -filetype=obj -triple=arm64-apple-macos -o a64_file2.o a64_file2.s
-# RUN: %lld -arch arm64 -o a64_file2_no_merge.exe a64_file1.dylib a64_file2.o
-# RUN: %lld -arch arm64 -o a64_file2_no_merge_v2.exe a64_file1.dylib a64_file2.o -no_objc_category_merging
-# RUN: %lld -arch arm64 -o a64_file2_no_merge_v3.exe a64_file1.dylib a64_file2.o -objc_category_merging -no_objc_category_merging
-# RUN: %lld -arch arm64 -o a64_file2_merge.exe -objc_category_merging a64_file1.dylib a64_file2.o
+# RUN: %lld -no_objc_relative_method_lists -arch arm64 -o a64_file2_no_merge.exe a64_file1.dylib a64_file2.o
+# RUN: %lld -no_objc_relative_method_lists -arch arm64 -o a64_file2_no_merge_v2.exe a64_file1.dylib a64_file2.o -no_objc_category_merging
+# RUN: %lld -no_objc_relative_method_lists -arch arm64 -o a64_file2_no_merge_v3.exe a64_file1.dylib a64_file2.o -objc_category_merging -no_objc_category_merging
+# RUN: %lld -no_objc_relative_method_lists -arch arm64 -o a64_file2_merge.exe -objc_category_merging a64_file1.dylib a64_file2.o
 
 # RUN: llvm-objdump --objc-meta-data --macho a64_file2_no_merge.exe | FileCheck %s --check-prefixes=NO_MERGE_CATS
 # RUN: llvm-objdump --objc-meta-data --macho a64_file2_no_merge_v2.exe | FileCheck %s --check-prefixes=NO_MERGE_CATS
@@ -18,7 +18,7 @@
 # RUN: llvm-objdump --objc-meta-data --macho a64_file2_merge.exe | FileCheck %s --check-prefixes=MERGE_CATS
 
 ############ Test merging multiple categories into the base class ############
-# RUN: %lld -arch arm64 -o a64_file2_merge_into_class.exe -objc_category_merging a64_file1.o a64_file2.o
+# RUN: %lld -no_objc_relative_method_lists -arch arm64 -o a64_file2_merge_into_class.exe -objc_category_merging a64_file1.o a64_file2.o
 # RUN: llvm-objdump --objc-meta-data --macho a64_file2_merge_into_class.exe | FileCheck %s --check-prefixes=MERGE_CATS_CLS
 
 
diff --git a/lld/test/MachO/objc-category-merging-erase-objc-name-test.s b/lld/test/MachO/objc-category-merging-erase-objc-name-test.s
index aeb2395b3a85805..9e4046a1447e703 100644
--- a/lld/test/MachO/objc-category-merging-erase-objc-name-test.s
+++ b/lld/test/MachO/objc-category-merging-erase-objc-name-test.s
@@ -5,7 +5,7 @@
 ; delete the 'MyTestProtocol' name
 
 ; RUN: llvm-mc -filetype=obj -triple=arm64-apple-macos -o %T/erase-objc-name.o %s
-; RUN: %lld -arch arm64 -dylib -o %T/erase-objc-name.dylib %T/erase-objc-name.o -objc_category_merging
+; RUN: %lld -no_objc_relative_method_lists -arch arm64 -dylib -o %T/erase-objc-name.dylib %T/erase-objc-name.o -objc_category_merging
 ; RUN: llvm-objdump --objc-meta-data --macho %T/erase-objc-name.dylib | FileCheck %s --check-prefixes=MERGE_CATS
 
 ; === Check merge categories enabled ===
diff --git a/lld/test/MachO/objc-category-merging-minimal.s b/lld/test/MachO/objc-category-merging-minimal.s
index b94799a57a4d852..088a4d0f30417f6 100644
--- a/lld/test/MachO/objc-category-merging-minimal.s
+++ b/lld/test/MachO/objc-category-merging-minimal.s
@@ -9,7 +9,7 @@
 ## Create our main testing dylib - linking against the fake dylib above
 # RUN: llvm-mc -filetype=obj -triple=arm64-apple-macos -o merge_cat_minimal.o merge_cat_minimal.s
 # RUN: %lld -arch arm64 -dylib -o merge_cat_minimal_no_merge.dylib a64_fakedylib.dylib merge_cat_minimal.o
-# RUN: %lld -objc_relative_method_lists -arch arm64 -dylib -o merge_cat_minimal_merge.dylib -objc_category_merging a64_fakedylib.dylib merge_cat_minimal.o
+# RUN: %lld -arch arm64 -dylib -o merge_cat_minimal_merge.dylib -objc_category_merging a64_fakedylib.dylib merge_cat_minimal.o
 
 ## Now verify that the flag caused category merging to happen appropriatelly
 # RUN: llvm-objdump --objc-meta-data --macho merge_cat_minimal_no_merge.dylib | FileCheck %s --check-prefixes=NO_MERGE_CATS
@@ -17,7 +17,7 @@
 
 ############ Test merging multiple categories into the base class ############
 # RUN: llvm-mc -filetype=obj -triple=arm64-apple-macos -o merge_base_class_minimal.o merge_base_class_minimal.s
-# RUN: %lld -arch arm64 -dylib -objc_relative_method_lists -o merge_base_class_minimal_yes_merge.dylib -objc_category_merging merge_base_class_minimal.o merge_cat_minimal.o
+# RUN: %lld -arch arm64 -dylib -o merge_base_class_minimal_yes_merge.dylib -objc_category_merging merge_base_class_minimal.o merge_cat_minimal.o
 # RUN: %lld -arch arm64 -dylib -o merge_base_class_minimal_no_merge.dylib merge_base_class_minimal.o merge_cat_minimal.o
 
 # RUN: llvm-objdump --objc-meta-data --macho merge_base_class_minimal_no_merge.dylib  | FileCheck %s --check-prefixes=NO_MERGE_INTO_BASE
@@ -25,7 +25,7 @@
 
 ############ Test merging swift category into the base class ############
 # RUN: llvm-mc -filetype=obj -triple=arm64-apple-macos -o MyBaseClassSwiftExtension.o MyBaseClassSwiftExtension.s
-# RUN: %lld -arch arm64 -dylib -o merge_base_class_swift_minimal_yes_merge.dylib -objc_category_merging MyBaseClassSwiftExtension.o merge_base_class_minimal.o
+# RUN: %lld -no_objc_relative_method_lists -arch arm64 -dylib -o merge_base_class_swift_minimal_yes_merge.dylib -objc_category_merging MyBaseClassSwiftExtension.o merge_base_class_minimal.o
 # RUN: llvm-objdump --objc-meta-data --macho merge_base_class_swift_minimal_yes_merge.dylib | FileCheck %s --check-prefixes=YES_MERGE_INTO_BASE_SWIFT
 
 #### Check merge categories enabled ###
diff --git a/lld/test/MachO/objc-category-merging-swift.s b/lld/test/MachO/objc-category-merging-swift.s
index 28025b0bc577a88..7a084d9310d6c62 100644
--- a/lld/test/MachO/objc-category-merging-swift.s
+++ b/lld/test/MachO/objc-category-merging-swift.s
@@ -5,7 +5,7 @@
 ## Apply category merging to swiftc code just make sure we can handle addends
 ## and don't erase category names for swift -- in order to not crash
 # RUN: llvm-mc -filetype=obj -triple=arm64-apple-macos -o cat_swift.o %s
-# RUN: %lld -arch arm64 -dylib -o cat_swift.dylib cat_swift.o -objc_category_merging
+# RUN: %lld -arch arm64 -dylib -o cat_swift.dylib cat_swift.o -objc_category_merging -no_objc_relative_method_lists
 # RUN: llvm-objdump --objc-meta-data --macho cat_swift.dylib | FileCheck %s --check-prefixes=CHECK-MERGE
 
 ; CHECK-MERGE:      Contents of (__DATA_CONST,__objc_classlist) section
diff --git a/lld/test/MachO/objc-relative-method-lists-simple.s b/lld/test/MachO/objc-relative-method-lists-simple.s
index 9f54b5ad828a076..c8646f59603b3f3 100644
--- a/lld/test/MachO/objc-relative-method-lists-simple.s
+++ b/lld/test/MachO/objc-relative-method-lists-simple.s
@@ -3,20 +3,28 @@
 # RUN: rm -rf %t; split-file %s %t && cd %t
 
 ## Compile a64_rel_dylib.o
-# RUN: llvm-mc -filetype=obj -triple=arm64-apple-macos -o a64_rel_dylib.o a64_simple_class.s
+# RUN: llvm-mc -filetype=obj -triple=arm64-apple-macos10.15 -o a64_rel_dylib.o a64_simple_class.s
 
 ## Test arm64 + relative method lists
-# RUN: %no-lsystem-lld a64_rel_dylib.o -o a64_rel_dylib.dylib -map a64_rel_dylib.map -dylib -arch arm64 -objc_relative_method_lists
+# RUN: %no-lsystem-lld a64_rel_dylib.o -o a64_rel_dylib.dylib -map a64_rel_dylib.map -dylib -arch arm64
 # RUN: llvm-objdump --macho --objc-meta-data a64_rel_dylib.dylib  | FileCheck %s --check-prefix=CHK_REL
 
 ## Test arm64 + relative method lists + dead-strip
-# RUN: %no-lsystem-lld a64_rel_dylib.o -o a64_rel_dylib.dylib -map a64_rel_dylib.map -dylib -arch arm64 -objc_relative_method_lists -dead_strip
+# RUN: %no-lsystem-lld a64_rel_dylib.o -o a64_rel_dylib.dylib -map a64_rel_dylib.map -dylib -arch arm64 -dead_strip
 # RUN: llvm-objdump --macho --objc-meta-data a64_rel_dylib.dylib  | FileCheck %s --check-prefix=CHK_REL
 
 ## Test arm64 + traditional method lists (no relative offsets)
 # RUN: %no-lsystem-lld a64_rel_dylib.o -o a64_rel_dylib.dylib -map a64_rel_dylib.map -dylib -arch arm64 -no_objc_relative_method_lists
 # RUN: llvm-objdump --macho --objc-meta-data a64_rel_dylib.dylib  | FileCheck %s --check-prefix=CHK_NO_REL
 
+## Test arm64 + relative method lists by explicitly adding `-objc_relative_method_lists`.
+# RUN: %lld a64_rel_dylib.o -o a64_rel_dylib.dylib -map a64_rel_dylib.map -dylib -arch arm64 -platform_version macOS 10.15 10.15 -objc_relative_method_lists
+# RUN: llvm-objdump --macho --objc-meta-data a64_rel_dylib.dylib  | FileCheck %s --check-prefix=CHK_REL
+
+## Test arm64 + no relative method lists by default.
+# RUN: %lld a64_rel_dylib.o -o a64_rel_dylib.dylib -map a64_rel_dylib.map -dylib -arch arm64 -platform_version macOS 10.15 10.15
+# RUN: llvm-objdump --macho --objc-meta-data a64_rel_dylib.dylib  | FileCheck %s --check-prefix=CHK_NO_REL
+
 
 CHK_REL:       Contents of (__DATA_CONST,__objc_classlist) section
 CHK_REL-NEXT:  _OBJC_CLASS_$_MyClass
@@ -125,7 +133,7 @@ CHK_NO_REL-NEXT:		       imp +[MyClass class_method_02]
 .include "objc-macros.s"
 
 .section	__TEXT,__text,regular,pure_instructions
-.build_version macos, 11, 0
+.build_version macos, 10, 15
 
 .objc_selector_def "-[MyClass instance_method_00]"
 .objc_selector_def "-[MyClass instance_method_01]"

From 97919864ec9227001f7bcb5594197b304ace9482 Mon Sep 17 00:00:00 2001
From: Michael Jones <michaelrj@google.com>
Date: Fri, 16 Aug 2024 11:22:33 -0700
Subject: [PATCH 211/441] Revert "[libc] Disable old headergen checks unless
 enabled" (#104627)

Reverts llvm/llvm-project#104522

Caused crashes on Fuchsia
---
 libc/CMakeLists.txt          | 33 ++++++++++++++++-----------------
 libc/include/CMakeLists.txt  | 14 ++++++--------
 llvm/runtimes/CMakeLists.txt |  3 +--
 3 files changed, 23 insertions(+), 27 deletions(-)

diff --git a/libc/CMakeLists.txt b/libc/CMakeLists.txt
index 133686acf6fa62b..dd45d6cc8cb6ab5 100644
--- a/libc/CMakeLists.txt
+++ b/libc/CMakeLists.txt
@@ -50,32 +50,31 @@ set(LIBC_NAMESPACE ${default_namespace}
   CACHE STRING "The namespace to use to enclose internal implementations. Must start with '__llvm_libc'."
 )
 
-option(LIBC_USE_NEW_HEADER_GEN "Generate header files using new headergen instead of the old one" ON)
 
-if(LIBC_USE_NEW_HEADER_GEN)
-  add_subdirectory(newhdrgen)
-else()
-  if(LLVM_LIBC_FULL_BUILD OR LLVM_LIBC_GPU_BUILD)
-    if(NOT LIBC_HDRGEN_EXE)
-      # We need to set up hdrgen first since other targets depend on it.
-      add_subdirectory(utils/LibcTableGenUtil)
-      add_subdirectory(utils/HdrGen)
-      # Calling add_tablegen sets variables like LIBC_TABLEGEN_EXE in
-      # PARENT_SCOPE which get lost until saved in the cache.
-      set(LIBC_TABLEGEN_EXE "${LIBC_TABLEGEN_EXE}" CACHE INTERNAL "")
-      set(LIBC_TABLEGEN_TARGET "${LIBC_TABLEGEN_TARGET}" CACHE INTERNAL "")
-    else()
-      message(STATUS "Will use ${LIBC_HDRGEN_EXE} for libc header generation.")
-    endif()
+add_subdirectory(newhdrgen)
+
+
+if(LLVM_LIBC_FULL_BUILD OR LLVM_LIBC_GPU_BUILD)
+  if(NOT LIBC_HDRGEN_EXE)
+    # We need to set up hdrgen first since other targets depend on it.
+    add_subdirectory(utils/LibcTableGenUtil)
+    add_subdirectory(utils/HdrGen)
+    # Calling add_tablegen sets variables like LIBC_TABLEGEN_EXE in
+    # PARENT_SCOPE which get lost until saved in the cache.
+    set(LIBC_TABLEGEN_EXE "${LIBC_TABLEGEN_EXE}" CACHE INTERNAL "")
+    set(LIBC_TABLEGEN_TARGET "${LIBC_TABLEGEN_TARGET}" CACHE INTERNAL "")
+  else()
+    message(STATUS "Will use ${LIBC_HDRGEN_EXE} for libc header generation.")
   endif()
 endif()
-
 # We will build the GPU utilities if we are not doing a runtimes build.
 option(LIBC_BUILD_GPU_LOADER "Always build the GPU loader utilities" OFF)
 if(LIBC_BUILD_GPU_LOADER OR (LLVM_LIBC_GPU_BUILD AND NOT LLVM_RUNTIMES_BUILD))
   add_subdirectory(utils/gpu)
 endif()
 
+option(LIBC_USE_NEW_HEADER_GEN "Generate header files using new headergen instead of the old one" ON)
+
 set(NEED_LIBC_HDRGEN FALSE)
 if(NOT LLVM_RUNTIMES_BUILD)
   if("libc" IN_LIST LLVM_ENABLE_RUNTIMES)
diff --git a/libc/include/CMakeLists.txt b/libc/include/CMakeLists.txt
index 4e3ae7f801f4a08..2b6eb61782a6326 100644
--- a/libc/include/CMakeLists.txt
+++ b/libc/include/CMakeLists.txt
@@ -420,21 +420,19 @@ add_header_macro(
     .llvm-libc-types.posix_spawn_file_actions_t
 )
 
-add_header_macro(
+add_gen_header(
   link
-  ../libc/newhdrgen/yaml/link.yaml
-  link.h.def
-  link.h
+  DEF_FILE link.h.def
+  GEN_HDR link.h
   DEPENDS
     .llvm_libc_common_h
     .llvm-libc-macros.link_macros
 )
 
-add_header_macro(
+add_gen_header(
   elf
-  ../libc/newhdrgen/yaml/elf.yaml
-  elf.h.def
-  elf.h
+  DEF_FILE elf.h.def
+  GEN_HDR elf.h
   DEPENDS
     .llvm-libc-macros.elf_macros
 )
diff --git a/llvm/runtimes/CMakeLists.txt b/llvm/runtimes/CMakeLists.txt
index 76202b2cfe04212..187c44fb9d04d2e 100644
--- a/llvm/runtimes/CMakeLists.txt
+++ b/llvm/runtimes/CMakeLists.txt
@@ -521,8 +521,7 @@ if(build_runtimes)
     endforeach()
   endif()
   if("libc" IN_LIST LLVM_ENABLE_PROJECTS AND
-      (LLVM_LIBC_FULL_BUILD OR LLVM_LIBC_GPU_BUILD) AND 
-      (NOT LIBC_USE_NEW_HEADER_GEN))
+      (LLVM_LIBC_FULL_BUILD OR LLVM_LIBC_GPU_BUILD))
     if(LIBC_HDRGEN_EXE)
       set(hdrgen_exe ${LIBC_HDRGEN_EXE})
     else()

From 8becb80c43d567d12126a539514040e93892ef1a Mon Sep 17 00:00:00 2001
From: Ahmed Bougacha <ahmed@bougacha.org>
Date: Fri, 16 Aug 2024 11:27:21 -0700
Subject: [PATCH 212/441] [AArch64] Remove apple-a7-sysreg. (#102709)

This feature provided CPM_IOACC_CTL_EL3, a lone system register that has
been carried over since the original ARM64 implementation, where it was
the only processor-specific register in a long list of architectural
sysregs. We don't need it here.

It's been used as a generic processor-specific sysreg in tests, but the
functionality they target is now better covered in other more exhaustive
tests.
---
 llvm/lib/Target/AArch64/AArch64Features.td        |   3 ---
 llvm/lib/Target/AArch64/AArch64Processors.td      |   2 +-
 llvm/lib/Target/AArch64/AArch64SystemOperands.td  |   5 -----
 .../CodeGen/AArch64/print-mrs-system-register.ll  |  11 -----------
 .../MC/AArch64/arm64-target-specific-sysreg.s     |  10 ----------
 .../AArch64/Inputs/print-mrs.obj.macho-aarch64    | Bin 348 -> 0 bytes
 .../MachO/AArch64/macho-print-mrs.test            |   3 ---
 7 files changed, 1 insertion(+), 33 deletions(-)
 delete mode 100644 llvm/test/CodeGen/AArch64/print-mrs-system-register.ll
 delete mode 100644 llvm/test/MC/AArch64/arm64-target-specific-sysreg.s
 delete mode 100644 llvm/test/tools/llvm-objdump/MachO/AArch64/Inputs/print-mrs.obj.macho-aarch64
 delete mode 100644 llvm/test/tools/llvm-objdump/MachO/AArch64/macho-print-mrs.test

diff --git a/llvm/lib/Target/AArch64/AArch64Features.td b/llvm/lib/Target/AArch64/AArch64Features.td
index 8ec13f17cf0a0d4..f157b1053f59fa0 100644
--- a/llvm/lib/Target/AArch64/AArch64Features.td
+++ b/llvm/lib/Target/AArch64/AArch64Features.td
@@ -721,9 +721,6 @@ def FeatureTaggedGlobals : SubtargetFeature<"tagged-globals",
     "true", "Use an instruction sequence for taking the address of a global "
     "that allows a memory tag in the upper address bits">;
 
-def FeatureAppleA7SysReg  : SubtargetFeature<"apple-a7-sysreg", "HasAppleA7SysReg", "true",
-  "Apple A7 (the CPU formerly known as Cyclone)">;
-
 def FeatureEL2VMSA : SubtargetFeature<"el2vmsa", "HasEL2VMSA", "true",
   "Enable Exception Level 2 Virtual Memory System Architecture">;
 
diff --git a/llvm/lib/Target/AArch64/AArch64Processors.td b/llvm/lib/Target/AArch64/AArch64Processors.td
index 52b5c8a0903ea67..83f7fa761e00ea5 100644
--- a/llvm/lib/Target/AArch64/AArch64Processors.td
+++ b/llvm/lib/Target/AArch64/AArch64Processors.td
@@ -835,7 +835,7 @@ def ProcessorFeatures {
                                      FeatureFullFP16, FeatureCRC, FeatureLSE, FeatureRAS, FeatureRDM,
                                      FeatureFPARMv8];
   list<SubtargetFeature> AppleA7  = [HasV8_0aOps, FeatureSHA2, FeatureAES, FeatureFPARMv8,
-                                     FeatureNEON,FeaturePerfMon, FeatureAppleA7SysReg];
+                                     FeatureNEON,FeaturePerfMon];
   list<SubtargetFeature> AppleA10 = [HasV8_0aOps, FeatureSHA2, FeatureAES, FeatureFPARMv8,
                                      FeatureNEON, FeaturePerfMon, FeatureCRC,
                                      FeatureRDM, FeaturePAN, FeatureLOR, FeatureVH];
diff --git a/llvm/lib/Target/AArch64/AArch64SystemOperands.td b/llvm/lib/Target/AArch64/AArch64SystemOperands.td
index cd6e6bffda10bca..7476ab852a923bd 100644
--- a/llvm/lib/Target/AArch64/AArch64SystemOperands.td
+++ b/llvm/lib/Target/AArch64/AArch64SystemOperands.td
@@ -1747,11 +1747,6 @@ foreach n = 0-31 in {
 let Requires = [{ {AArch64::FeatureSPE_EEF} }] in
 def : RWSysReg<"PMSNEVFR_EL1",      0b11, 0b000, 0b1001, 0b1001, 0b001>;
 
-// Cyclone specific system registers
-//                                 Op0    Op1     CRn     CRm    Op2
-let Requires = [{ {AArch64::FeatureAppleA7SysReg} }] in
-def : RWSysReg<"CPM_IOACC_CTL_EL3", 0b11, 0b111, 0b1111, 0b0010, 0b000>;
-
 // Scalable Matrix Extension (SME)
 //                                 Op0   Op1    CRn     CRm     Op2
 let Requires = [{ {AArch64::FeatureSME} }] in {
diff --git a/llvm/test/CodeGen/AArch64/print-mrs-system-register.ll b/llvm/test/CodeGen/AArch64/print-mrs-system-register.ll
deleted file mode 100644
index 3411ed6161bdbdf..000000000000000
--- a/llvm/test/CodeGen/AArch64/print-mrs-system-register.ll
+++ /dev/null
@@ -1,11 +0,0 @@
-; RUN: llc -mtriple=arm64-apple-darwin %s -o - | FileCheck %s
-
-; CHECK: mrs x0, CPM_IOACC_CTL_EL3
-
-define void @foo1() #0 {
-entry:
-  tail call void asm sideeffect "mrs x0, cpm_ioacc_ctl_el3", ""()
-  ret void
-}
-
-attributes #0 = { "target-cpu"="cyclone" }
diff --git a/llvm/test/MC/AArch64/arm64-target-specific-sysreg.s b/llvm/test/MC/AArch64/arm64-target-specific-sysreg.s
deleted file mode 100644
index 05cea3ac2da55f4..000000000000000
--- a/llvm/test/MC/AArch64/arm64-target-specific-sysreg.s
+++ /dev/null
@@ -1,10 +0,0 @@
-// RUN: not llvm-mc -triple arm64 -mcpu=generic -show-encoding < %s 2>&1 | \
-// RUN:   FileCheck %s --check-prefix=CHECK-GENERIC
-//
-// RUN: llvm-mc -triple arm64 -mcpu=cyclone -show-encoding < %s 2>&1 | \
-// RUN:   FileCheck %s --check-prefix=CHECK-CYCLONE
-
-msr CPM_IOACC_CTL_EL3, x0
-
-// CHECK-GENERIC: error: expected writable system register or pstate
-// CHECK-CYCLONE: msr CPM_IOACC_CTL_EL3, x0   // encoding: [0x00,0xf2,0x1f,0xd5]
diff --git a/llvm/test/tools/llvm-objdump/MachO/AArch64/Inputs/print-mrs.obj.macho-aarch64 b/llvm/test/tools/llvm-objdump/MachO/AArch64/Inputs/print-mrs.obj.macho-aarch64
deleted file mode 100644
index 06cb13d07dafc5fa60ab0981ef0d3a22a371e835..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 348
zcmX^A>+L@t1_nk3AOI1}Fopt%!N4F1#52Gw>;zCR7$|^EU|@jp*@2=U3>9XGk1t8B
zD1oR2k@4{%t`Q*!K8yv^4>FGlCJ3Tg7#fhdAUO#j)`6<?0n!3MJ~t2t05J%F?1o_!
uv%s=M5})j^GB7j*9bk^X2J#Od$UrCnG5L}B4Do6C`9=&mCAkF#ASD3PO%vDv

diff --git a/llvm/test/tools/llvm-objdump/MachO/AArch64/macho-print-mrs.test b/llvm/test/tools/llvm-objdump/MachO/AArch64/macho-print-mrs.test
deleted file mode 100644
index cd40f0e849c5e8b..000000000000000
--- a/llvm/test/tools/llvm-objdump/MachO/AArch64/macho-print-mrs.test
+++ /dev/null
@@ -1,3 +0,0 @@
-RUN: llvm-objdump -d -m --no-show-raw-insn %p/Inputs/print-mrs.obj.macho-aarch64 | FileCheck %s
-
-CHECK: 0:  mrs x0, CPM_IOACC_CTL_EL3

From f999b321d7317fa9e59ce597bd3d63b7035fe774 Mon Sep 17 00:00:00 2001
From: Justin Bogner <mail@justinbogner.com>
Date: Fri, 16 Aug 2024 20:28:34 +0200
Subject: [PATCH 213/441] [DirectX] Add missing Analysis usage to
 DXILResourceMDWrapper

This analysis can't be used with other analyses if this isn't set.

Pull Request: https://github.com/llvm/llvm-project/pull/104244
---
 llvm/lib/Target/DirectX/DXILResourceAnalysis.h | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/llvm/lib/Target/DirectX/DXILResourceAnalysis.h b/llvm/lib/Target/DirectX/DXILResourceAnalysis.h
index 3a2b8a9fd39d591..26d9237d51b49cb 100644
--- a/llvm/lib/Target/DirectX/DXILResourceAnalysis.h
+++ b/llvm/lib/Target/DirectX/DXILResourceAnalysis.h
@@ -57,6 +57,10 @@ class DXILResourceMDWrapper : public ModulePass {
   /// Calculate the DXILResource for the module.
   bool runOnModule(Module &M) override;
 
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.setPreservesAll();
+  }
+
   void print(raw_ostream &O, const Module *M = nullptr) const override;
 };
 } // namespace llvm

From 535b209cb3b485a66083d7c4386c16c2df0000a8 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Fri, 16 Aug 2024 11:35:50 -0700
Subject: [PATCH 214/441] [SelectionDAGISel] Use getSignedConstant for
 OPC_EmitInteger.

---
 llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
index 3e517a51862de32..46c24299d1ae97d 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
@@ -3784,7 +3784,9 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch,
       if (Opcode >= OPC_EmitInteger && Opcode <= OPC_EmitInteger64)
         Val = decodeSignRotatedValue(Val);
       RecordedNodes.push_back(std::pair<SDValue, SDNode *>(
-          CurDAG->getTargetConstant(Val, SDLoc(NodeToMatch), VT), nullptr));
+          CurDAG->getSignedConstant(Val, SDLoc(NodeToMatch), VT,
+                                    /*isTarget=*/true),
+          nullptr));
       continue;
     }
     case OPC_EmitRegister:

From 51ede55ee2acd3383d990c5fc8882d4b021b87b4 Mon Sep 17 00:00:00 2001
From: Justin Bogner <mail@justinbogner.com>
Date: Fri, 16 Aug 2024 20:36:41 +0200
Subject: [PATCH 215/441] Re-Apply "[DXIL][Analysis] Implement enough of
 DXILResourceAnalysis for buffers" (#104517)

Some build configs allow `llvm_unreachable` in a constexpr context, but
not all, so these functions that map a fully covered enum to a string
can't be constexpr. This version fixes that by dropping constexpr from
those functions.

This reverts commit fcc318ff7960d7de8cbac56eb4f32b44b5261677, reapplying
28d577ecefa1557f5dea5566bf33b885c563d14b.

Original message follows:

This implements the DXILResourceAnalysis pass for `dx.TypedBuffer` and
`dx.RawBuffer` types. This should be sufficient to lower
`dx.handle.fromBinding` for this set of types, but it leaves a number of
TODOs around for other resource types.

This also includes a straightforward `print` method in `ResourceInfo` to
make the analysis testable. This is deliberately different than the
printer in `lib/Target/DirectX/DXILResource.cpp`, which attempts to
print bindings in a format compatible with the comments `dxc` prints. We
will eventually want to make that functionality driven by this analysis
pass, but it isn't sufficient for testing so we need both.
---
 llvm/include/llvm/Analysis/DXILResource.h     |  13 +-
 llvm/include/llvm/IR/IntrinsicsDirectX.td     |  10 +
 llvm/lib/Analysis/DXILResource.cpp            | 380 +++++++++++++++++-
 .../DXILResource/buffer-frombinding.ll        | 126 ++++++
 4 files changed, 521 insertions(+), 8 deletions(-)
 create mode 100644 llvm/test/Analysis/DXILResource/buffer-frombinding.ll

diff --git a/llvm/include/llvm/Analysis/DXILResource.h b/llvm/include/llvm/Analysis/DXILResource.h
index 7db088df260f304..3ba0ae5de61d515 100644
--- a/llvm/include/llvm/Analysis/DXILResource.h
+++ b/llvm/include/llvm/Analysis/DXILResource.h
@@ -18,19 +18,20 @@
 namespace llvm {
 class CallInst;
 class MDTuple;
+class TargetExtType;
 
 namespace dxil {
 
 class ResourceInfo {
   struct ResourceBinding {
-    uint32_t UniqueID;
+    uint32_t RecordID;
     uint32_t Space;
     uint32_t LowerBound;
     uint32_t Size;
 
     bool operator==(const ResourceBinding &RHS) const {
-      return std::tie(UniqueID, Space, LowerBound, Size) ==
-             std::tie(RHS.UniqueID, RHS.Space, RHS.LowerBound, RHS.Size);
+      return std::tie(RecordID, Space, LowerBound, Size) ==
+             std::tie(RHS.RecordID, RHS.Space, RHS.LowerBound, RHS.Size);
     }
     bool operator!=(const ResourceBinding &RHS) const {
       return !(*this == RHS);
@@ -128,9 +129,9 @@ class ResourceInfo {
   bool isFeedback() const;
   bool isMultiSample() const;
 
-  void bind(uint32_t UniqueID, uint32_t Space, uint32_t LowerBound,
+  void bind(uint32_t RecordID, uint32_t Space, uint32_t LowerBound,
             uint32_t Size) {
-    Binding.UniqueID = UniqueID;
+    Binding.RecordID = RecordID;
     Binding.Space = Space;
     Binding.LowerBound = LowerBound;
     Binding.Size = Size;
@@ -215,6 +216,8 @@ class ResourceInfo {
 
   ResourceBinding getBinding() const { return Binding; }
   std::pair<uint32_t, uint32_t> getAnnotateProps() const;
+
+  void print(raw_ostream &OS) const;
 };
 
 } // namespace dxil
diff --git a/llvm/include/llvm/IR/IntrinsicsDirectX.td b/llvm/include/llvm/IR/IntrinsicsDirectX.td
index 904801e6e9e95f0..c9102aa3dd972bc 100644
--- a/llvm/include/llvm/IR/IntrinsicsDirectX.td
+++ b/llvm/include/llvm/IR/IntrinsicsDirectX.td
@@ -20,6 +20,16 @@ def int_dx_flattened_thread_id_in_group : Intrinsic<[llvm_i32_ty], [], [IntrNoMe
 def int_dx_create_handle : ClangBuiltin<"__builtin_hlsl_create_handle">,
     Intrinsic<[ llvm_ptr_ty ], [llvm_i8_ty], [IntrWillReturn]>;
 
+// Create resource handle given binding information. Returns a `target("dx.")`
+// type appropriate for the kind of resource given a register space ID, lower
+// bound and range size of the binding, as well as an index and an indicator
+// whether that index may be non-uniform.
+def int_dx_handle_fromBinding
+    : DefaultAttrsIntrinsic<
+          [llvm_any_ty],
+          [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i1_ty],
+          [IntrNoMem]>;
+
 def int_dx_all : DefaultAttrsIntrinsic<[llvm_i1_ty], [llvm_any_ty]>;
 def int_dx_any : DefaultAttrsIntrinsic<[llvm_i1_ty], [llvm_any_ty]>;
 def int_dx_clamp : DefaultAttrsIntrinsic<[llvm_any_ty], [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>]>;
diff --git a/llvm/lib/Analysis/DXILResource.cpp b/llvm/lib/Analysis/DXILResource.cpp
index 9941a55d46d6459..1b5b051c9db29eb 100644
--- a/llvm/lib/Analysis/DXILResource.cpp
+++ b/llvm/lib/Analysis/DXILResource.cpp
@@ -8,9 +8,14 @@
 
 #include "llvm/Analysis/DXILResource.h"
 #include "llvm/ADT/APInt.h"
+#include "llvm/IR/Constants.h"
 #include "llvm/IR/DerivedTypes.h"
+#include "llvm/IR/DiagnosticInfo.h"
 #include "llvm/IR/Instructions.h"
+#include "llvm/IR/Intrinsics.h"
+#include "llvm/IR/IntrinsicsDirectX.h"
 #include "llvm/IR/Metadata.h"
+#include "llvm/IR/Module.h"
 #include "llvm/InitializePasses.h"
 
 #define DEBUG_TYPE "dxil-resource"
@@ -18,6 +23,131 @@
 using namespace llvm;
 using namespace dxil;
 
+static StringRef getResourceClassName(ResourceClass RC) {
+  switch (RC) {
+  case ResourceClass::SRV:
+    return "SRV";
+  case ResourceClass::UAV:
+    return "UAV";
+  case ResourceClass::CBuffer:
+    return "CBuffer";
+  case ResourceClass::Sampler:
+    return "Sampler";
+  }
+  llvm_unreachable("Unhandled ResourceClass");
+}
+
+static StringRef getResourceKindName(ResourceKind RK) {
+  switch (RK) {
+  case ResourceKind::Texture1D:
+    return "Texture1D";
+  case ResourceKind::Texture2D:
+    return "Texture2D";
+  case ResourceKind::Texture2DMS:
+    return "Texture2DMS";
+  case ResourceKind::Texture3D:
+    return "Texture3D";
+  case ResourceKind::TextureCube:
+    return "TextureCube";
+  case ResourceKind::Texture1DArray:
+    return "Texture1DArray";
+  case ResourceKind::Texture2DArray:
+    return "Texture2DArray";
+  case ResourceKind::Texture2DMSArray:
+    return "Texture2DMSArray";
+  case ResourceKind::TextureCubeArray:
+    return "TextureCubeArray";
+  case ResourceKind::TypedBuffer:
+    return "TypedBuffer";
+  case ResourceKind::RawBuffer:
+    return "RawBuffer";
+  case ResourceKind::StructuredBuffer:
+    return "StructuredBuffer";
+  case ResourceKind::CBuffer:
+    return "CBuffer";
+  case ResourceKind::Sampler:
+    return "Sampler";
+  case ResourceKind::TBuffer:
+    return "TBuffer";
+  case ResourceKind::RTAccelerationStructure:
+    return "RTAccelerationStructure";
+  case ResourceKind::FeedbackTexture2D:
+    return "FeedbackTexture2D";
+  case ResourceKind::FeedbackTexture2DArray:
+    return "FeedbackTexture2DArray";
+  case ResourceKind::NumEntries:
+  case ResourceKind::Invalid:
+    return "<invalid>";
+  }
+  llvm_unreachable("Unhandled ResourceKind");
+}
+
+static StringRef getElementTypeName(ElementType ET) {
+  switch (ET) {
+  case ElementType::I1:
+    return "i1";
+  case ElementType::I16:
+    return "i16";
+  case ElementType::U16:
+    return "u16";
+  case ElementType::I32:
+    return "i32";
+  case ElementType::U32:
+    return "u32";
+  case ElementType::I64:
+    return "i64";
+  case ElementType::U64:
+    return "u64";
+  case ElementType::F16:
+    return "f16";
+  case ElementType::F32:
+    return "f32";
+  case ElementType::F64:
+    return "f64";
+  case ElementType::SNormF16:
+    return "snorm_f16";
+  case ElementType::UNormF16:
+    return "unorm_f16";
+  case ElementType::SNormF32:
+    return "snorm_f32";
+  case ElementType::UNormF32:
+    return "unorm_f32";
+  case ElementType::SNormF64:
+    return "snorm_f64";
+  case ElementType::UNormF64:
+    return "unorm_f64";
+  case ElementType::PackedS8x32:
+    return "p32i8";
+  case ElementType::PackedU8x32:
+    return "p32u8";
+  case ElementType::Invalid:
+    return "<invalid>";
+  }
+  llvm_unreachable("Unhandled ElementType");
+}
+
+static StringRef getSamplerTypeName(SamplerType ST) {
+  switch (ST) {
+  case SamplerType::Default:
+    return "Default";
+  case SamplerType::Comparison:
+    return "Comparison";
+  case SamplerType::Mono:
+    return "Mono";
+  }
+  llvm_unreachable("Unhandled SamplerType");
+}
+
+static StringRef getSamplerFeedbackTypeName(SamplerFeedbackType SFT) {
+  switch (SFT) {
+  case SamplerFeedbackType::MinMip:
+    return "MinMip";
+  case SamplerFeedbackType::MipRegionUsed:
+    return "MipRegionUsed";
+  }
+  llvm_unreachable("Unhandled SamplerFeedbackType");
+}
+
 bool ResourceInfo::isUAV() const { return RC == ResourceClass::UAV; }
 
 bool ResourceInfo::isCBuffer() const { return RC == ResourceClass::CBuffer; }
@@ -240,7 +370,7 @@ MDTuple *ResourceInfo::getAsMetadata(LLVMContext &Ctx) const {
         Constant::getIntegerValue(I1Ty, APInt(1, V)));
   };
 
-  MDVals.push_back(getIntMD(Binding.UniqueID));
+  MDVals.push_back(getIntMD(Binding.RecordID));
   MDVals.push_back(ValueAsMetadata::get(Symbol));
   MDVals.push_back(MDString::get(Ctx, Name));
   MDVals.push_back(getIntMD(Binding.Space));
@@ -330,6 +460,248 @@ std::pair<uint32_t, uint32_t> ResourceInfo::getAnnotateProps() const {
   return {Word0, Word1};
 }
 
+void ResourceInfo::print(raw_ostream &OS) const {
+  OS << "  Symbol: ";
+  Symbol->printAsOperand(OS);
+  OS << "\n";
+
+  OS << "  Name: \"" << Name << "\"\n"
+     << "  Binding:\n"
+     << "    Record ID: " << Binding.RecordID << "\n"
+     << "    Space: " << Binding.Space << "\n"
+     << "    Lower Bound: " << Binding.LowerBound << "\n"
+     << "    Size: " << Binding.Size << "\n"
+     << "  Class: " << getResourceClassName(RC) << "\n"
+     << "  Kind: " << getResourceKindName(Kind) << "\n";
+
+  if (isCBuffer()) {
+    OS << "  CBuffer size: " << CBufferSize << "\n";
+  } else if (isSampler()) {
+    OS << "  Sampler Type: " << getSamplerTypeName(SamplerTy) << "\n";
+  } else {
+    if (isUAV()) {
+      OS << "  Globally Coherent: " << UAVFlags.GloballyCoherent << "\n"
+         << "  HasCounter: " << UAVFlags.HasCounter << "\n"
+         << "  IsROV: " << UAVFlags.IsROV << "\n";
+    }
+    if (isMultiSample())
+      OS << "  Sample Count: " << MultiSample.Count << "\n";
+
+    if (isStruct()) {
+      OS << "  Buffer Stride: " << Struct.Stride << "\n";
+      OS << "  Alignment: " << Struct.AlignLog2 << "\n";
+    } else if (isTyped()) {
+      OS << "  Element Type: " << getElementTypeName(Typed.ElementTy) << "\n"
+         << "  Element Count: " << Typed.ElementCount << "\n";
+    } else if (isFeedback())
+      OS << "  Feedback Type: " << getSamplerFeedbackTypeName(Feedback.Type)
+         << "\n";
+  }
+}
+
+//===----------------------------------------------------------------------===//
+// ResourceMapper
+
+static dxil::ElementType toDXILElementType(Type *Ty, bool IsSigned) {
+  // TODO: Handle unorm, snorm, and packed.
+  Ty = Ty->getScalarType();
+
+  if (Ty->isIntegerTy()) {
+    switch (Ty->getIntegerBitWidth()) {
+    case 16:
+      return IsSigned ? ElementType::I16 : ElementType::U16;
+    case 32:
+      return IsSigned ? ElementType::I32 : ElementType::U32;
+    case 64:
+      return IsSigned ? ElementType::I64 : ElementType::U64;
+    case 1:
+    default:
+      return ElementType::Invalid;
+    }
+  } else if (Ty->isFloatTy()) {
+    return ElementType::F32;
+  } else if (Ty->isDoubleTy()) {
+    return ElementType::F64;
+  } else if (Ty->isHalfTy()) {
+    return ElementType::F16;
+  }
+
+  return ElementType::Invalid;
+}
+
+namespace {
+
+class ResourceMapper {
+  Module &M;
+  LLVMContext &Context;
+  DXILResourceMap &Resources;
+
+  // In DXC, Record ID is unique per resource type. Match that.
+  uint32_t NextUAV = 0;
+  uint32_t NextSRV = 0;
+  uint32_t NextCBuf = 0;
+  uint32_t NextSmp = 0;
+
+public:
+  ResourceMapper(Module &M,
+                 MapVector<CallInst *, dxil::ResourceInfo> &Resources)
+      : M(M), Context(M.getContext()), Resources(Resources) {}
+
+  void diagnoseHandle(CallInst *CI, const Twine &Msg,
+                      DiagnosticSeverity Severity = DS_Error) {
+    std::string S;
+    raw_string_ostream SS(S);
+    CI->printAsOperand(SS);
+    DiagnosticInfoUnsupported Diag(*CI->getFunction(), Msg + ": " + SS.str(),
+                                   CI->getDebugLoc(), Severity);
+    Context.diagnose(Diag);
+  }
+
+  ResourceInfo *mapBufferType(CallInst *CI, TargetExtType *HandleTy,
+                              bool IsTyped) {
+    if (HandleTy->getNumTypeParameters() != 1 ||
+        HandleTy->getNumIntParameters() != (IsTyped ? 3 : 2)) {
+      diagnoseHandle(CI, Twine("Invalid buffer target type"));
+      return nullptr;
+    }
+
+    Type *ElTy = HandleTy->getTypeParameter(0);
+    unsigned IsWriteable = HandleTy->getIntParameter(0);
+    unsigned IsROV = HandleTy->getIntParameter(1);
+    bool IsSigned = IsTyped && HandleTy->getIntParameter(2);
+
+    ResourceClass RC = IsWriteable ? ResourceClass::UAV : ResourceClass::SRV;
+    ResourceKind Kind;
+    if (IsTyped)
+      Kind = ResourceKind::TypedBuffer;
+    else if (ElTy->isIntegerTy(8))
+      Kind = ResourceKind::RawBuffer;
+    else
+      Kind = ResourceKind::StructuredBuffer;
+
+    // TODO: We need to lower to a typed pointer, can we smuggle the type
+    // through?
+    Value *Symbol = UndefValue::get(PointerType::getUnqual(Context));
+    // TODO: We don't actually keep track of the name right now...
+    StringRef Name = "";
+
+    auto [It, Success] = Resources.try_emplace(CI, RC, Kind, Symbol, Name);
+    assert(Success && "Mapping the same CallInst again?");
+    (void)Success;
+    // We grab a pointer into the map's storage, which isn't generally safe.
+    // Since we're just using this to fill in the info the map won't mutate and
+    // the pointer stays valid for as long as we need it to.
+    ResourceInfo *RI = &(It->second);
+
+    if (RI->isUAV())
+      // TODO: We need analysis for GloballyCoherent and HasCounter
+      RI->setUAV(false, false, IsROV);
+
+    if (RI->isTyped()) {
+      dxil::ElementType ET = toDXILElementType(ElTy, IsSigned);
+      uint32_t Count = 1;
+      if (auto *VTy = dyn_cast<FixedVectorType>(ElTy))
+        Count = VTy->getNumElements();
+      RI->setTyped(ET, Count);
+    } else if (RI->isStruct()) {
+      const DataLayout &DL = M.getDataLayout();
+
+      // This mimics what DXC does. Notably, we only ever set the alignment if
+      // the type is actually a struct type.
+      uint32_t Stride = DL.getTypeAllocSize(ElTy);
+      MaybeAlign Alignment;
+      if (auto *STy = dyn_cast<StructType>(ElTy))
+        Alignment = DL.getStructLayout(STy)->getAlignment();
+      RI->setStruct(Stride, Alignment);
+    }
+
+    return RI;
+  }
+
+  ResourceInfo *mapHandleIntrin(CallInst *CI) {
+    FunctionType *FTy = CI->getFunctionType();
+    Type *RetTy = FTy->getReturnType();
+    auto *HandleTy = dyn_cast<TargetExtType>(RetTy);
+    if (!HandleTy) {
+      diagnoseHandle(CI, "dx.handle.fromBinding requires target type");
+      return nullptr;
+    }
+
+    StringRef TypeName = HandleTy->getName();
+    if (TypeName == "dx.TypedBuffer") {
+      return mapBufferType(CI, HandleTy, /*IsTyped=*/true);
+    } else if (TypeName == "dx.RawBuffer") {
+      return mapBufferType(CI, HandleTy, /*IsTyped=*/false);
+    } else if (TypeName == "dx.CBuffer") {
+      // TODO: implement
+      diagnoseHandle(CI, "dx.CBuffer handles are not implemented yet");
+      return nullptr;
+    } else if (TypeName == "dx.Sampler") {
+      // TODO: implement
+      diagnoseHandle(CI, "dx.Sampler handles are not implemented yet");
+      return nullptr;
+    } else if (TypeName == "dx.Texture") {
+      // TODO: implement
+      diagnoseHandle(CI, "dx.Texture handles are not implemented yet");
+      return nullptr;
+    }
+
+    diagnoseHandle(CI, "Invalid target(dx) type");
+    return nullptr;
+  }
+
+  ResourceInfo *mapHandleFromBinding(CallInst *CI) {
+    assert(CI->getIntrinsicID() == Intrinsic::dx_handle_fromBinding &&
+           "Must be dx.handle.fromBinding intrinsic");
+
+    ResourceInfo *RI = mapHandleIntrin(CI);
+    if (!RI)
+      return nullptr;
+
+    uint32_t NextID;
+    if (RI->isCBuffer())
+      NextID = NextCBuf++;
+    else if (RI->isSampler())
+      NextID = NextSmp++;
+    else if (RI->isUAV())
+      NextID = NextUAV++;
+    else
+      NextID = NextSRV++;
+
+    uint32_t Space = cast<ConstantInt>(CI->getArgOperand(0))->getZExtValue();
+    uint32_t LowerBound =
+        cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
+    uint32_t Size = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
+
+    RI->bind(NextID, Space, LowerBound, Size);
+
+    return RI;
+  }
+
+  void mapResources() {
+    for (Function &F : M.functions()) {
+      if (!F.isDeclaration())
+        continue;
+      LLVM_DEBUG(dbgs() << "Function: " << F.getName() << "\n");
+      Intrinsic::ID ID = F.getIntrinsicID();
+      switch (ID) {
+      default:
+        // TODO: handle `dx.op` functions.
+        continue;
+      case Intrinsic::dx_handle_fromBinding:
+        for (User *U : F.users()) {
+          LLVM_DEBUG(dbgs() << "  Visiting: " << *U << "\n");
+          if (CallInst *CI = dyn_cast<CallInst>(U))
+            mapHandleFromBinding(CI);
+        }
+        break;
+      }
+    }
+  }
+};
+
+} // namespace
+
 //===----------------------------------------------------------------------===//
 // DXILResourceAnalysis and DXILResourcePrinterPass
 
@@ -339,6 +711,7 @@ AnalysisKey DXILResourceAnalysis::Key;
 DXILResourceMap DXILResourceAnalysis::run(Module &M,
                                           ModuleAnalysisManager &AM) {
   DXILResourceMap Data;
+  ResourceMapper(M, Data).mapResources();
   return Data;
 }
 
@@ -351,7 +724,7 @@ PreservedAnalyses DXILResourcePrinterPass::run(Module &M,
     OS << "Binding for ";
     Handle->print(OS);
     OS << "\n";
-    // TODO: Info.print(OS);
+    Info.print(OS);
     OS << "\n";
   }
 
@@ -373,6 +746,7 @@ void DXILResourceWrapperPass::getAnalysisUsage(AnalysisUsage &AU) const {
 
 bool DXILResourceWrapperPass::runOnModule(Module &M) {
   ResourceMap.reset(new DXILResourceMap());
+  ResourceMapper(M, *ResourceMap).mapResources();
   return false;
 }
 
@@ -387,7 +761,7 @@ void DXILResourceWrapperPass::print(raw_ostream &OS, const Module *) const {
     OS << "Binding for ";
     Handle->print(OS);
     OS << "\n";
-    // TODO: Info.print(OS);
+    Info.print(OS);
     OS << "\n";
   }
 }
diff --git a/llvm/test/Analysis/DXILResource/buffer-frombinding.ll b/llvm/test/Analysis/DXILResource/buffer-frombinding.ll
new file mode 100644
index 000000000000000..4349adb8ef8ebb3
--- /dev/null
+++ b/llvm/test/Analysis/DXILResource/buffer-frombinding.ll
@@ -0,0 +1,126 @@
+; RUN: opt -S -disable-output -passes="print<dxil-resource>" < %s 2>&1 | FileCheck %s
+
+@G = external constant <4 x float>, align 4
+
+define void @test_typedbuffer() {
+  ; RWBuffer<float4> Buf : register(u5, space3)
+  %typed0 = call target("dx.TypedBuffer", <4 x float>, 1, 0, 0)
+              @llvm.dx.handle.fromBinding.tdx.TypedBuffer_f32_1_0(
+                  i32 3, i32 5, i32 1, i32 0, i1 false)
+  ; CHECK: Binding for %typed0
+  ; CHECK:   Symbol: ptr undef
+  ; CHECK:   Name: ""
+  ; CHECK:   Binding:
+  ; CHECK:     Record ID: 0
+  ; CHECK:     Space: 3
+  ; CHECK:     Lower Bound: 5
+  ; CHECK:     Size: 1
+  ; CHECK:   Class: UAV
+  ; CHECK:   Kind: TypedBuffer
+  ; CHECK:   Globally Coherent: 0
+  ; CHECK:   HasCounter: 0
+  ; CHECK:   IsROV: 0
+  ; CHECK:   Element Type: f32
+  ; CHECK:   Element Count: 4
+
+  ; RWBuffer<int> Buf : register(u7, space2)
+  %typed1 = call target("dx.TypedBuffer", i32, 1, 0, 1)
+      @llvm.dx.handle.fromBinding.tdx.TypedBuffer_i32_1_0t(
+          i32 2, i32 7, i32 1, i32 0, i1 false)
+  ; CHECK: Binding for %typed1
+  ; CHECK:   Symbol: ptr undef
+  ; CHECK:   Name: ""
+  ; CHECK:   Binding:
+  ; CHECK:     Record ID: 1
+  ; CHECK:     Space: 2
+  ; CHECK:     Lower Bound: 7
+  ; CHECK:     Size: 1
+  ; CHECK:   Class: UAV
+  ; CHECK:   Kind: TypedBuffer
+  ; CHECK:   Globally Coherent: 0
+  ; CHECK:   HasCounter: 0
+  ; CHECK:   IsROV: 0
+  ; CHECK:   Element Type: i32
+  ; CHECK:   Element Count: 1
+
+  ; Buffer<uint4> Buf[24] : register(t3, space5)
+  %typed2 = call target("dx.TypedBuffer", <4 x i32>, 0, 0, 0)
+      @llvm.dx.handle.fromBinding.tdx.TypedBuffer_i32_0_0t(
+          i32 2, i32 7, i32 24, i32 0, i1 false)
+  ; CHECK: Binding for %typed2
+  ; CHECK:   Symbol: ptr undef
+  ; CHECK:   Name: ""
+  ; CHECK:   Binding:
+  ; CHECK:     Record ID: 0
+  ; CHECK:     Space: 2
+  ; CHECK:     Lower Bound: 7
+  ; CHECK:     Size: 24
+  ; CHECK:   Class: SRV
+  ; CHECK:   Kind: TypedBuffer
+  ; CHECK:   Element Type: u32
+  ; CHECK:   Element Count: 4
+
+  ret void
+}
+
+define void @test_structbuffer() {
+  ; struct S { float4 a; uint4 b; };
+  ; StructuredBuffer<S> Buf : register(t2, space4)
+  %struct0 = call target("dx.RawBuffer", {<4 x float>, <4 x i32>}, 0, 0)
+      @llvm.dx.handle.fromBinding.tdx.RawBuffer_sl_v4f32v4i32s_0_0t(
+          i32 4, i32 2, i32 1, i32 0, i1 false)
+  ; CHECK: Binding for %struct0
+  ; CHECK:   Symbol: ptr undef
+  ; CHECK:   Name: ""
+  ; CHECK:   Binding:
+  ; CHECK:     Record ID: 1
+  ; CHECK:     Space: 4
+  ; CHECK:     Lower Bound: 2
+  ; CHECK:     Size: 1
+  ; CHECK:   Class: SRV
+  ; CHECK:   Kind: StructuredBuffer
+  ; CHECK:   Buffer Stride: 32
+  ; CHECK:   Alignment: 4
+
+  ret void
+}
+
+define void @test_bytebuffer() {
+  ; ByteAddressBuffer Buf : register(t8, space1)
+  %byteaddr0 = call target("dx.RawBuffer", i8, 0, 0)
+      @llvm.dx.handle.fromBinding.tdx.RawBuffer_i8_0_0t(
+          i32 1, i32 8, i32 1, i32 0, i1 false)
+  ; CHECK: Binding for %byteaddr0
+  ; CHECK:   Symbol: ptr undef
+  ; CHECK:   Name: ""
+  ; CHECK:   Binding:
+  ; CHECK:     Record ID: 2
+  ; CHECK:     Space: 1
+  ; CHECK:     Lower Bound: 8
+  ; CHECK:     Size: 1
+  ; CHECK:   Class: SRV
+  ; CHECK:   Kind: RawBuffer
+
+  ret void
+}
+
+; Note: We need declarations for each handle.fromBinding in the same
+; order as they appear in source to ensure that we can put our CHECK
+; lines along side the thing they're checking.
+declare target("dx.TypedBuffer", <4 x float>, 1, 0, 0)
+        @llvm.dx.handle.fromBinding.tdx.TypedBuffer_v4f32_1_0_0t(
+        i32, i32, i32, i32, i1) #0
+declare target("dx.TypedBuffer", i32, 1, 0, 1)
+        @llvm.dx.handle.fromBinding.tdx.TypedBuffer_i32_1_0_1t(
+            i32, i32, i32, i32, i1) #0
+declare target("dx.TypedBuffer", <4 x i32>, 0, 0, 0)
+        @llvm.dx.handle.fromBinding.tdx.TypedBuffer_v4i32_0_0_0t(
+            i32, i32, i32, i32, i1) #0
+declare target("dx.RawBuffer", { <4 x float>, <4 x i32> }, 0, 0)
+        @llvm.dx.handle.fromBinding.tdx.RawBuffer_sl_v4f32v4i32s_0_0t(
+            i32, i32, i32, i32, i1) #0
+declare target("dx.RawBuffer", i8, 0, 0)
+        @llvm.dx.handle.fromBinding.tdx.RawBuffer_i8_0_0t(
+            i32, i32, i32, i32, i1) #0
+
+attributes #0 = { nocallback nofree nosync nounwind willreturn memory(none) }

From 3e7ca5f1efabb488663caec371e408d74c634d84 Mon Sep 17 00:00:00 2001
From: Kevin McAfee <kmcafee@nvidia.com>
Date: Fri, 16 Aug 2024 11:48:57 -0700
Subject: [PATCH 216/441] [SDAG] Read-only intrinsics must have WillReturn and
 !Throws attributes to be treated as loads (#99999)

This change avoids deleting `!willReturn` intrinsics for which the
return value is unused when building the SDAG. Currently, calls to
read-only intrinsics not marked with `IntrWillReturn` cannot be deleted
at the LLVM IR level but may be deleted when building the SDAG.
These calls are unsafe to remove from the IR because the functions are
`!willReturn` and should also be unsafe to remove fromthe SDAG for
the same reason. This change aligns the behavior of the SDAG to that
of LLVM IR. This change also requires that intrinsics not have the
`Throws` attribute to be treated as loads for the same reason.
---
 llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 8e24f5bd75ec86e..87f6b4388a728c7 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -5229,7 +5229,8 @@ void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
   // definition.
   const Function *F = I.getCalledFunction();
   bool HasChain = !F->doesNotAccessMemory();
-  bool OnlyLoad = HasChain && F->onlyReadsMemory();
+  bool OnlyLoad =
+      HasChain && F->onlyReadsMemory() && F->willReturn() && F->doesNotThrow();
 
   // Build the operand list.
   SmallVector<SDValue, 8> Ops;

From 3c603f857c26fd979ebff3a469c41ecc24bf548f Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Fri, 16 Aug 2024 13:49:19 -0500
Subject: [PATCH 217/441] [libcxx][fix] Rename incorrect filename variable

Summary:
This used an old name I forgot to fix, linter didn't catch it because it
was behind `ifdef` and the branch which I tested it on I forgot to
update the one I landed.
---
 libcxx/test/support/platform_support.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcxx/test/support/platform_support.h b/libcxx/test/support/platform_support.h
index 0d4fa63b03f107c..99e60f60c5998f6 100644
--- a/libcxx/test/support/platform_support.h
+++ b/libcxx/test/support/platform_support.h
@@ -78,7 +78,7 @@ inline std::string get_temp_file_name() {
   for (std::size_t i = 0; i < sizeof(Name); ++i)
     if (Name[i] == 'X')
       Name[i] = chars[rand() % strlen(chars)];
-  FILE* file = fopen(filename, "w");
+  FILE* file = fopen(Name, "w");
   if (!file)
     abort();
   if (fclose(file) == EOF)

From 845461093c6275022e90fb6af15fc330b246b86a Mon Sep 17 00:00:00 2001
From: Michael Jones <michaelrj@google.com>
Date: Fri, 16 Aug 2024 11:52:37 -0700
Subject: [PATCH 218/441] [libc] Fix generated header definitions in cmake
 (#104628)

Some new headers were not being properly built with
new headergen, since they were using the old "add_gen_header" instead of
the new "add_header_macro". This patch fixes the issue.
---
 libc/include/CMakeLists.txt | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/libc/include/CMakeLists.txt b/libc/include/CMakeLists.txt
index 2b6eb61782a6326..4e3ae7f801f4a08 100644
--- a/libc/include/CMakeLists.txt
+++ b/libc/include/CMakeLists.txt
@@ -420,19 +420,21 @@ add_header_macro(
     .llvm-libc-types.posix_spawn_file_actions_t
 )
 
-add_gen_header(
+add_header_macro(
   link
-  DEF_FILE link.h.def
-  GEN_HDR link.h
+  ../libc/newhdrgen/yaml/link.yaml
+  link.h.def
+  link.h
   DEPENDS
     .llvm_libc_common_h
     .llvm-libc-macros.link_macros
 )
 
-add_gen_header(
+add_header_macro(
   elf
-  DEF_FILE elf.h.def
-  GEN_HDR elf.h
+  ../libc/newhdrgen/yaml/elf.yaml
+  elf.h.def
+  elf.h
   DEPENDS
     .llvm-libc-macros.elf_macros
 )

From ef56061dcfd162fa3ba9dafd05762707f4c30095 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Fri, 16 Aug 2024 22:48:23 +0400
Subject: [PATCH 219/441] AMDGPU: Rename type helper functions in atomic
 handling

Requested on #95394
---
 llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 25fee559faa29ca..d9387b2dda30ed6 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -16093,7 +16093,7 @@ static OptimizationRemark emitAtomicRMWLegalRemark(const AtomicRMWInst *RMW) {
          << " operation at memory scope " << MemScope;
 }
 
-static bool isHalf2OrBFloat2(Type *Ty) {
+static bool isV2F16OrV2BF16(Type *Ty) {
   if (auto *VT = dyn_cast<FixedVectorType>(Ty)) {
     Type *EltTy = VT->getElementType();
     return VT->getNumElements() == 2 &&
@@ -16103,12 +16103,12 @@ static bool isHalf2OrBFloat2(Type *Ty) {
   return false;
 }
 
-static bool isHalf2(Type *Ty) {
+static bool isV2F16(Type *Ty) {
   FixedVectorType *VT = dyn_cast<FixedVectorType>(Ty);
   return VT && VT->getNumElements() == 2 && VT->getElementType()->isHalfTy();
 }
 
-static bool isBFloat2(Type *Ty) {
+static bool isV2BF16(Type *Ty) {
   FixedVectorType *VT = dyn_cast<FixedVectorType>(Ty);
   return VT && VT->getNumElements() == 2 && VT->getElementType()->isBFloatTy();
 }
@@ -16248,7 +16248,7 @@ SITargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const {
                                                  : AtomicExpansionKind::CmpXChg;
       }
 
-      if (Subtarget->hasAtomicDsPkAdd16Insts() && isHalf2OrBFloat2(Ty))
+      if (Subtarget->hasAtomicDsPkAdd16Insts() && isV2F16OrV2BF16(Ty))
         return AtomicExpansionKind::None;
 
       return AtomicExpansionKind::CmpXChg;
@@ -16273,24 +16273,24 @@ SITargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const {
     if (globalMemoryFPAtomicIsLegal(*Subtarget, RMW, HasSystemScope)) {
       if (AS == AMDGPUAS::FLAT_ADDRESS) {
         // gfx940, gfx12
-        if (Subtarget->hasAtomicFlatPkAdd16Insts() && isHalf2OrBFloat2(Ty))
+        if (Subtarget->hasAtomicFlatPkAdd16Insts() && isV2F16OrV2BF16(Ty))
           return ReportUnsafeHWInst(AtomicExpansionKind::None);
       } else if (AMDGPU::isExtendedGlobalAddrSpace(AS)) {
         // gfx90a, gfx940, gfx12
-        if (Subtarget->hasAtomicBufferGlobalPkAddF16Insts() && isHalf2(Ty))
+        if (Subtarget->hasAtomicBufferGlobalPkAddF16Insts() && isV2F16(Ty))
           return ReportUnsafeHWInst(AtomicExpansionKind::None);
 
         // gfx940, gfx12
-        if (Subtarget->hasAtomicGlobalPkAddBF16Inst() && isBFloat2(Ty))
+        if (Subtarget->hasAtomicGlobalPkAddBF16Inst() && isV2BF16(Ty))
           return ReportUnsafeHWInst(AtomicExpansionKind::None);
       } else if (AS == AMDGPUAS::BUFFER_FAT_POINTER) {
         // gfx90a, gfx940, gfx12
-        if (Subtarget->hasAtomicBufferGlobalPkAddF16Insts() && isHalf2(Ty))
+        if (Subtarget->hasAtomicBufferGlobalPkAddF16Insts() && isV2F16(Ty))
           return ReportUnsafeHWInst(AtomicExpansionKind::None);
 
         // While gfx90a/gfx940 supports v2bf16 for global/flat, it does not for
         // buffer. gfx12 does have the buffer version.
-        if (Subtarget->hasAtomicBufferPkAddBF16Inst() && isBFloat2(Ty))
+        if (Subtarget->hasAtomicBufferPkAddBF16Inst() && isV2BF16(Ty))
           return ReportUnsafeHWInst(AtomicExpansionKind::None);
       }
 
@@ -16311,7 +16311,7 @@ SITargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const {
           // gfx908
           if (RMW->use_empty() &&
               Subtarget->hasAtomicBufferGlobalPkAddF16NoRtnInsts() &&
-              isHalf2(Ty))
+              isV2F16(Ty))
             return ReportUnsafeHWInst(AtomicExpansionKind::None);
         }
       }

From e6b9f12b0ac0f1e6f7f7145719092c10731e4fe4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Valentin=20Clement=20=28=E3=83=90=E3=83=AC=E3=83=B3?=
 =?UTF-8?q?=E3=82=BF=E3=82=A4=E3=83=B3=20=E3=82=AF=E3=83=AC=E3=83=A1?=
 =?UTF-8?q?=E3=83=B3=29?= <clementval@gmail.com>
Date: Fri, 16 Aug 2024 11:56:22 -0700
Subject: [PATCH 220/441] [flang][cuda][driver] Make sure flang does not switch
 to cc1 (#104613)

Flang is switch to cc1 when we use `-x cuda`. Make sure we can use fc1
with cuda fortran input.

The current pipeline will fail at MLIR level for the moment.

#104483
---
 clang/lib/Driver/Types.cpp        | 3 +++
 flang/test/Driver/cuda-option.f90 | 7 ++++++-
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/clang/lib/Driver/Types.cpp b/clang/lib/Driver/Types.cpp
index 2b9b391c19c9fd0..3de45b00b4d00fd 100644
--- a/clang/lib/Driver/Types.cpp
+++ b/clang/lib/Driver/Types.cpp
@@ -170,6 +170,9 @@ bool types::isAcceptedByFlang(ID Id) {
   case TY_LLVM_IR:
   case TY_LLVM_BC:
     return true;
+  case TY_PP_CUDA:
+  case TY_CUDA:
+    return true;
   }
 }
 
diff --git a/flang/test/Driver/cuda-option.f90 b/flang/test/Driver/cuda-option.f90
index 562f8683b0ff7d8..591d7d070d3a85c 100644
--- a/flang/test/Driver/cuda-option.f90
+++ b/flang/test/Driver/cuda-option.f90
@@ -1,5 +1,6 @@
 ! Test -fcuda option
 ! RUN: %flang_fc1 -cpp -x cuda -fdebug-unparse %s -o - | FileCheck %s
+! RUN: not %flang -cpp -x cuda %s -o - 2>&1 | FileCheck %s --check-prefix=MLIRERROR
 ! RUN: not %flang_fc1 -cpp %s -o - 2>&1 | FileCheck %s --check-prefix=ERROR
 program main
 #if _CUDA
@@ -12,4 +13,8 @@ program main
 ! CHECK: INTEGER :: var = 1
 ! CHECK: INTEGER, DEVICE :: dvar
 
-! ERROR: cuda-option.f90:8:19: error: expected end of statement
+! ERROR: cuda-option.f90:{{.*}}:{{.*}}: error: expected end of statement
+
+! The whole pipeline is not in place yet. It will currently fail at MLIR
+! translation level.
+! MLIRERROR: failed to legalize operation 'cuf.alloc'

From 464fa3b3b047518699689b57c473c87701986593 Mon Sep 17 00:00:00 2001
From: Ahmed Bougacha <ahmed@bougacha.org>
Date: Fri, 16 Aug 2024 11:58:59 -0700
Subject: [PATCH 221/441] [AArch64][MachO] Encode @AUTH to
 ARM64_RELOC_AUTHENTICATED_POINTER.

This adds MachO support for emission of authenticated pointer
relocations.

We already support AArch64AuthMCExpr, to represent assembly expressions
such as:
  .quad <symbol>@AUTH(<key>, <discriminator> [, addr])
For example:
  .quad _g3@AUTH(ib, 1234, addr)

These @AUTH expressions lower to a new kind of MachO relocation:
  ARM64_RELOC_AUTHENTICATED_POINTER (11)

The relocation points to the referenced symbol.
The other data, describing the signing scheme and original addend
(only 32 bits instead of 64), is encoded into the addend (in the
relocated location):

  |63|62|61-51|50-49|  48  |47     -     32|31  -  0|
  | 1| 0|  0  | key | addr | discriminator | addend |
---
 llvm/docs/PointerAuth.md                      |  13 ++
 llvm/lib/Object/MachOObjectFile.cpp           |   2 +-
 .../MCTargetDesc/AArch64MachObjectWriter.cpp  |  41 ++++++
 .../arm64e-authenticated-pointer-reloc.s      | 119 ++++++++++++++++++
 4 files changed, 174 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/MC/AArch64/arm64e-authenticated-pointer-reloc.s

diff --git a/llvm/docs/PointerAuth.md b/llvm/docs/PointerAuth.md
index e027c902e58e123..84e0af7577c7d6d 100644
--- a/llvm/docs/PointerAuth.md
+++ b/llvm/docs/PointerAuth.md
@@ -374,6 +374,19 @@ For example:
     .quad _sym@AUTH(ia,12,addr)
 ```
 
+#### MachO Object File Representation
+
+At the object file level, authenticated relocations are represented using the
+``ARM64_RELOC_AUTHENTICATED_POINTER`` relocation kind (with value ``11``).
+
+The pointer authentication information is encoded into the addend as follows:
+
+```
+| 63 | 62 | 61-51 | 50-49 |   48   | 47     -     32 | 31  -  0 |
+| -- | -- | ----- | ----- | ------ | --------------- | -------- |
+|  1 |  0 |   0   |  key  |  addr  |  discriminator  |  addend  |
+```
+
 #### ELF Object File Representation
 
 At the object file level, authenticated relocations are represented
diff --git a/llvm/lib/Object/MachOObjectFile.cpp b/llvm/lib/Object/MachOObjectFile.cpp
index ff55a847c34327d..e2d1723b1fa9813 100644
--- a/llvm/lib/Object/MachOObjectFile.cpp
+++ b/llvm/lib/Object/MachOObjectFile.cpp
@@ -2341,7 +2341,7 @@ void MachOObjectFile::getRelocationTypeName(
         "ARM64_RELOC_PAGEOFF12",          "ARM64_RELOC_GOT_LOAD_PAGE21",
         "ARM64_RELOC_GOT_LOAD_PAGEOFF12", "ARM64_RELOC_POINTER_TO_GOT",
         "ARM64_RELOC_TLVP_LOAD_PAGE21",   "ARM64_RELOC_TLVP_LOAD_PAGEOFF12",
-        "ARM64_RELOC_ADDEND"
+        "ARM64_RELOC_ADDEND",             "ARM64_RELOC_AUTHENTICATED_POINTER"
       };
 
       if (RType >= std::size(Table))
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MachObjectWriter.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MachObjectWriter.cpp
index 33f0ca237b26dd1..ed0a972cc50abdc 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MachObjectWriter.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MachObjectWriter.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "MCTargetDesc/AArch64FixupKinds.h"
+#include "MCTargetDesc/AArch64MCExpr.h"
 #include "MCTargetDesc/AArch64MCTargetDesc.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/BinaryFormat/MachO.h"
@@ -394,6 +395,46 @@ void AArch64MachObjectWriter::recordRelocation(
     Value = 0;
   }
 
+  if (Target.getRefKind() == AArch64MCExpr::VK_AUTH ||
+      Target.getRefKind() == AArch64MCExpr::VK_AUTHADDR) {
+    auto *Expr = cast<AArch64AuthMCExpr>(Fixup.getValue());
+
+    assert(Type == MachO::ARM64_RELOC_UNSIGNED);
+
+    if (IsPCRel) {
+      Asm.getContext().reportError(Fixup.getLoc(),
+                                   "invalid PC relative auth relocation");
+      return;
+    }
+
+    if (Log2Size != 3) {
+      Asm.getContext().reportError(
+          Fixup.getLoc(), "invalid auth relocation size, must be 8 bytes");
+      return;
+    }
+
+    if (Target.getSymB()) {
+      Asm.getContext().reportError(
+          Fixup.getLoc(),
+          "invalid auth relocation, can't reference two symbols");
+      return;
+    }
+
+    uint16_t Discriminator = Expr->getDiscriminator();
+    AArch64PACKey::ID Key = Expr->getKey();
+
+    if (!isInt<32>(Value)) {
+      Asm.getContext().reportError(Fixup.getLoc(),
+                                   "addend too big for relocation");
+      return;
+    }
+
+    Type = MachO::ARM64_RELOC_AUTHENTICATED_POINTER;
+    Value = (uint32_t(Value)) | (uint64_t(Discriminator) << 32) |
+            (uint64_t(Expr->hasAddressDiversity()) << 48) |
+            (uint64_t(Key) << 49) | (1ULL << 63);
+  }
+
   // If there's any addend left to handle, encode it in the instruction.
   FixedValue = Value;
 
diff --git a/llvm/test/MC/AArch64/arm64e-authenticated-pointer-reloc.s b/llvm/test/MC/AArch64/arm64e-authenticated-pointer-reloc.s
new file mode 100644
index 000000000000000..04fa5098c4a3e2b
--- /dev/null
+++ b/llvm/test/MC/AArch64/arm64e-authenticated-pointer-reloc.s
@@ -0,0 +1,119 @@
+// RUN: llvm-mc -triple=arm64-apple-darwin < %s | \
+// RUN:   FileCheck %s --check-prefix=ASM
+
+// RUN: llvm-mc -triple=arm64-apple-darwin -filetype=obj < %s | \
+// RUN:   llvm-readobj --expand-relocs --sections \
+// RUN:   --section-relocations --section-data - | \
+// RUN:   FileCheck %s --check-prefix=RELOC
+
+
+
+// RELOC:    Sections [
+// RELOC-LABEL: Section {
+// RELOC-LABEL: Section {
+// RELOC-NEXT:   Index: 1
+// RELOC-NEXT:   Name: __const (5F 5F 63 6F 6E 73 74 00 00 00 00 00 00 00 00 00)
+// RELOC-NEXT:   Segment: __DATA (5F 5F 44 41 54 41 00 00 00 00 00 00 00 00 00 00)
+
+.section	__DATA,__const
+.p2align	3
+
+// RELOC-LABEL: Relocations [
+// RELOC-NEXT:   Relocation {
+// RELOC-NEXT:     Offset: 0x70
+// RELOC-NEXT:     PCRel: 0
+// RELOC-NEXT:     Length: 3
+// RELOC-NEXT:     Type: ARM64_RELOC_AUTHENTICATED_POINTER (11)
+// RELOC-NEXT:     Symbol: _g 7
+// RELOC-NEXT:   }
+// RELOC-NEXT:   Relocation {
+// RELOC-NEXT:     Offset: 0x60
+// RELOC-NEXT:     PCRel: 0
+// RELOC-NEXT:     Length: 3
+// RELOC-NEXT:     Type: ARM64_RELOC_AUTHENTICATED_POINTER (11)
+// RELOC-NEXT:     Symbol: _g 6
+// RELOC-NEXT:   }
+// RELOC-NEXT:   Relocation {
+// RELOC-NEXT:     Offset: 0x50
+// RELOC-NEXT:     PCRel: 0
+// RELOC-NEXT:     Length: 3
+// RELOC-NEXT:     Type: ARM64_RELOC_AUTHENTICATED_POINTER (11)
+// RELOC-NEXT:     Symbol: _g5
+// RELOC-NEXT:   }
+// RELOC-NEXT:   Relocation {
+// RELOC-NEXT:     Offset: 0x40
+// RELOC-NEXT:     PCRel: 0
+// RELOC-NEXT:     Length: 3
+// RELOC-NEXT:     Type: ARM64_RELOC_AUTHENTICATED_POINTER (11)
+// RELOC-NEXT:     Symbol: _g4
+// RELOC-NEXT:   }
+// RELOC-NEXT:   Relocation {
+// RELOC-NEXT:     Offset: 0x30
+// RELOC-NEXT:     PCRel: 0
+// RELOC-NEXT:     Length: 3
+// RELOC-NEXT:     Type: ARM64_RELOC_AUTHENTICATED_POINTER (11)
+// RELOC-NEXT:     Symbol: _g3
+// RELOC-NEXT:   }
+// RELOC-NEXT:   Relocation {
+// RELOC-NEXT:     Offset: 0x20
+// RELOC-NEXT:     PCRel: 0
+// RELOC-NEXT:     Length: 3
+// RELOC-NEXT:     Type: ARM64_RELOC_AUTHENTICATED_POINTER (11)
+// RELOC-NEXT:     Symbol: _g2
+// RELOC-NEXT:   }
+// RELOC-NEXT:   Relocation {
+// RELOC-NEXT:     Offset: 0x10
+// RELOC-NEXT:     PCRel: 0
+// RELOC-NEXT:     Length: 3
+// RELOC-NEXT:     Type: ARM64_RELOC_AUTHENTICATED_POINTER (11)
+// RELOC-NEXT:     Symbol: _g1
+// RELOC-NEXT:   }
+// RELOC-NEXT:   Relocation {
+// RELOC-NEXT:     Offset: 0x0
+// RELOC-NEXT:     PCRel: 0
+// RELOC-NEXT:     Length: 3
+// RELOC-NEXT:     Type: ARM64_RELOC_AUTHENTICATED_POINTER (11)
+// RELOC-NEXT:     Symbol: _g0
+// RELOC-NEXT:   }
+// RELOC-NEXT: ]
+// RELOC-NEXT: SectionData (
+
+// RELOC-NEXT:   0000: 00000000 2A000080
+// ASM:          .quad _g0@AUTH(ia,42)
+.quad _g0@AUTH(ia,42)
+.quad 0
+
+// RELOC-NEXT:   0010: 00000000 00000280
+// ASM:          .quad _g1@AUTH(ib,0)
+.quad _g1@AUTH(ib,0)
+.quad 0
+
+// RELOC-NEXT:   0020: 00000000 05000580
+// ASM:          .quad _g2@AUTH(da,5,addr)
+.quad _g2@AUTH(da,5,addr)
+.quad 0
+
+// RELOC-NEXT:   0030: 00000000 FFFF0780
+// ASM:          .quad _g3@AUTH(db,65535,addr)
+.quad _g3@AUTH(db,0xffff,addr)
+.quad 0
+
+// RELOC-NEXT:   0040: 07000000 00000080
+// ASM:          .quad (_g4+7)@AUTH(ia,0)
+.quad (_g4 + 7)@AUTH(ia,0)
+.quad 0
+
+// RELOC-NEXT:   0050: FDFFFFFF 00DE0280
+// ASM:          .quad (_g5-3)@AUTH(ib,56832)
+.quad (_g5 - 3)@AUTH(ib,0xde00)
+.quad 0
+
+// RELOC-NEXT:   0060: 00000000 FF000780
+// ASM:          .quad "_g 6"@AUTH(db,255,addr)
+.quad "_g 6"@AUTH(db,0xff,addr)
+.quad 0
+
+// RELOC-NEXT:   0070: 07000000 10000080
+// ASM:          .quad ("_g 7"+7)@AUTH(ia,16)
+.quad ("_g 7" + 7)@AUTH(ia,16)
+.quad 0

From e315ba185b835c8ee520076b434377cfb7350067 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Valentin=20Clement=20=28=E3=83=90=E3=83=AC=E3=83=B3?=
 =?UTF-8?q?=E3=82=BF=E3=82=A4=E3=83=B3=20=E3=82=AF=E3=83=AC=E3=83=A1?=
 =?UTF-8?q?=E3=83=B3=29?= <clementval@gmail.com>
Date: Fri, 16 Aug 2024 12:02:37 -0700
Subject: [PATCH 222/441] Revert "[flang][cuda][driver] Make sure flang does
 not switch to cc1" (#104632)

Reverts llvm/llvm-project#104613
---
 clang/lib/Driver/Types.cpp        | 3 ---
 flang/test/Driver/cuda-option.f90 | 7 +------
 2 files changed, 1 insertion(+), 9 deletions(-)

diff --git a/clang/lib/Driver/Types.cpp b/clang/lib/Driver/Types.cpp
index 3de45b00b4d00fd..2b9b391c19c9fd0 100644
--- a/clang/lib/Driver/Types.cpp
+++ b/clang/lib/Driver/Types.cpp
@@ -170,9 +170,6 @@ bool types::isAcceptedByFlang(ID Id) {
   case TY_LLVM_IR:
   case TY_LLVM_BC:
     return true;
-  case TY_PP_CUDA:
-  case TY_CUDA:
-    return true;
   }
 }
 
diff --git a/flang/test/Driver/cuda-option.f90 b/flang/test/Driver/cuda-option.f90
index 591d7d070d3a85c..562f8683b0ff7d8 100644
--- a/flang/test/Driver/cuda-option.f90
+++ b/flang/test/Driver/cuda-option.f90
@@ -1,6 +1,5 @@
 ! Test -fcuda option
 ! RUN: %flang_fc1 -cpp -x cuda -fdebug-unparse %s -o - | FileCheck %s
-! RUN: not %flang -cpp -x cuda %s -o - 2>&1 | FileCheck %s --check-prefix=MLIRERROR
 ! RUN: not %flang_fc1 -cpp %s -o - 2>&1 | FileCheck %s --check-prefix=ERROR
 program main
 #if _CUDA
@@ -13,8 +12,4 @@ program main
 ! CHECK: INTEGER :: var = 1
 ! CHECK: INTEGER, DEVICE :: dvar
 
-! ERROR: cuda-option.f90:{{.*}}:{{.*}}: error: expected end of statement
-
-! The whole pipeline is not in place yet. It will currently fail at MLIR
-! translation level.
-! MLIRERROR: failed to legalize operation 'cuf.alloc'
+! ERROR: cuda-option.f90:8:19: error: expected end of statement

From cf721e29c6a3f220bd66475b7d29aff7fe9d56e3 Mon Sep 17 00:00:00 2001
From: Amy Kwan <amy.kwan1@ibm.com>
Date: Fri, 16 Aug 2024 15:06:50 -0400
Subject: [PATCH 223/441] [PowerPC] Do not merge TLS constants within
 PPCMergeStringPool.cpp (#94059)

This patch prevents thread-local constants to be merged within
PPCMergeStringPool.cpp.

The PPCMergeStringPool pass primarily merges non-thread-local constants
together, and thread-local constants should not be mixed together with
other (non-thread-local) constants. In the event that thread-local and
other non-thread-local constants are pooled together, the
llvm.threadlocal.address intrinsic can fail as it expects its argument
to be a thread-local global value, but the merged string structure
created by the PPCMergeStringPool pass is not thread-local as a whole.
---
 .../lib/Target/PowerPC/PPCMergeStringPool.cpp |   5 +-
 .../PowerPC/mergeable-string-pool-tls.ll      | 373 ++++++++++++++++++
 2 files changed, 376 insertions(+), 2 deletions(-)
 create mode 100644 llvm/test/CodeGen/PowerPC/mergeable-string-pool-tls.ll

diff --git a/llvm/lib/Target/PowerPC/PPCMergeStringPool.cpp b/llvm/lib/Target/PowerPC/PPCMergeStringPool.cpp
index 309938accdf4c3b..daf6a0e65d54b24 100644
--- a/llvm/lib/Target/PowerPC/PPCMergeStringPool.cpp
+++ b/llvm/lib/Target/PowerPC/PPCMergeStringPool.cpp
@@ -170,8 +170,9 @@ void PPCMergeStringPool::collectCandidateConstants(Module &M) {
     LLVM_DEBUG(dbgs() << "hasInitializer() " << Global.hasInitializer()
                       << "\n");
 
-    // We can only pool constants.
-    if (!Global.isConstant() || !Global.hasInitializer())
+    // We can only pool non-thread-local constants.
+    if (!Global.isConstant() || !Global.hasInitializer() ||
+        Global.isThreadLocal())
       continue;
 
     // If a global constant has a section we do not try to pool it because
diff --git a/llvm/test/CodeGen/PowerPC/mergeable-string-pool-tls.ll b/llvm/test/CodeGen/PowerPC/mergeable-string-pool-tls.ll
new file mode 100644
index 000000000000000..fde135246c2a98c
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/mergeable-string-pool-tls.ll
@@ -0,0 +1,373 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -verify-machineinstrs -mtriple powerpc64-ibm-aix-xcoff \
+; RUN:     -ppc-asm-full-reg-names -enable-global-merge=false < %s | FileCheck %s \
+; RUN:     --check-prefix=CHECK64
+; RUN: llc -verify-machineinstrs -mtriple powerpc-ibm-aix-xcoff \
+; RUN:     -ppc-asm-full-reg-names -enable-global-merge=false < %s | FileCheck %s \
+; RUN:     --check-prefix=CHECK32
+; RUN: llc -verify-machineinstrs -mtriple powerpc64le-unknown-linux \
+; RUN:     -ppc-asm-full-reg-names < %s | FileCheck %s \
+; RUN:     --check-prefix=LINUX64LE
+; RUN: llc -verify-machineinstrs -mtriple powerpc64-unknown-linux \
+; RUN:     -ppc-asm-full-reg-names < %s | FileCheck %s \
+; RUN:     --check-prefix=LINUX64BE
+
+@.str = private unnamed_addr constant [47 x i8] c"TLS variable 1, 2 and non-TLS var: %s, %s, %s\0A\00", align 1
+@a = internal thread_local constant [5 x i8] c"tls1\00", align 1
+@b = internal thread_local constant [5 x i8] c"tls2\00", align 1
+@c = internal constant [15 x i8] c"Regular global\00", align 1
+@d = internal constant [10 x i32] [i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10], align 4
+@e = internal constant [4 x float] [float 0x4055F33340000000, float 0x4056333340000000, float 0x40567999A0000000, float 0x4056B33340000000], align 4
+
+declare noundef signext i32 @printf(ptr nocapture noundef readonly, ...) local_unnamed_addr #0
+declare nonnull ptr @llvm.threadlocal.address.p0(ptr nonnull) #1
+declare void @callee(ptr noundef) local_unnamed_addr #3
+declare void @callee2(ptr noundef) local_unnamed_addr #3
+
+define void @print_tls_func() {
+; CHECK64-LABEL: print_tls_func:
+; CHECK64:       # %bb.0: # %entry
+; CHECK64-NEXT:    mflr r0
+; CHECK64-NEXT:    stdu r1, -112(r1)
+; CHECK64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; CHECK64-NEXT:    std r0, 128(r1)
+; CHECK64-NEXT:    ld r6, L..C1(r2) # @__ModuleStringPool
+; CHECK64-NEXT:    bla .__tls_get_mod[PR]
+; CHECK64-NEXT:    ld r4, L..C2(r2) # target-flags(ppc-tlsld) @a
+; CHECK64-NEXT:    ld r5, L..C3(r2) # target-flags(ppc-tlsld) @b
+; CHECK64-NEXT:    add r4, r3, r4
+; CHECK64-NEXT:    add r5, r3, r5
+; CHECK64-NEXT:    addi r3, r6, 71
+; CHECK64-NEXT:    addi r6, r6, 56
+; CHECK64-NEXT:    bl .printf[PR]
+; CHECK64-NEXT:    nop
+; CHECK64-NEXT:    addi r1, r1, 112
+; CHECK64-NEXT:    ld r0, 16(r1)
+; CHECK64-NEXT:    mtlr r0
+; CHECK64-NEXT:    blr
+;
+; CHECK32-LABEL: print_tls_func:
+; CHECK32:       # %bb.0: # %entry
+; CHECK32-NEXT:    mflr r0
+; CHECK32-NEXT:    stwu r1, -64(r1)
+; CHECK32-NEXT:    lwz r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; CHECK32-NEXT:    stw r0, 72(r1)
+; CHECK32-NEXT:    lwz r6, L..C1(r2) # @__ModuleStringPool
+; CHECK32-NEXT:    bla .__tls_get_mod[PR]
+; CHECK32-NEXT:    lwz r4, L..C2(r2) # target-flags(ppc-tlsld) @a
+; CHECK32-NEXT:    lwz r5, L..C3(r2) # target-flags(ppc-tlsld) @b
+; CHECK32-NEXT:    add r4, r3, r4
+; CHECK32-NEXT:    add r5, r3, r5
+; CHECK32-NEXT:    addi r3, r6, 71
+; CHECK32-NEXT:    addi r6, r6, 56
+; CHECK32-NEXT:    bl .printf[PR]
+; CHECK32-NEXT:    nop
+; CHECK32-NEXT:    addi r1, r1, 64
+; CHECK32-NEXT:    lwz r0, 8(r1)
+; CHECK32-NEXT:    mtlr r0
+; CHECK32-NEXT:    blr
+;
+; LINUX64LE-LABEL: print_tls_func:
+; LINUX64LE:       # %bb.0: # %entry
+; LINUX64LE-NEXT:    mflr r0
+; LINUX64LE-NEXT:    stdu r1, -96(r1)
+; LINUX64LE-NEXT:    std r0, 112(r1)
+; LINUX64LE-NEXT:    .cfi_def_cfa_offset 96
+; LINUX64LE-NEXT:    .cfi_offset lr, 16
+; LINUX64LE-NEXT:    addis r3, r13, a@tprel@ha
+; LINUX64LE-NEXT:    addi r4, r3, a@tprel@l
+; LINUX64LE-NEXT:    addis r3, r13, b@tprel@ha
+; LINUX64LE-NEXT:    addi r5, r3, b@tprel@l
+; LINUX64LE-NEXT:    addis r3, r2, .L__ModuleStringPool@toc@ha
+; LINUX64LE-NEXT:    addi r6, r3, .L__ModuleStringPool@toc@l
+; LINUX64LE-NEXT:    addi r3, r6, 71
+; LINUX64LE-NEXT:    addi r6, r6, 56
+; LINUX64LE-NEXT:    bl printf
+; LINUX64LE-NEXT:    nop
+; LINUX64LE-NEXT:    addi r1, r1, 96
+; LINUX64LE-NEXT:    ld r0, 16(r1)
+; LINUX64LE-NEXT:    mtlr r0
+; LINUX64LE-NEXT:    blr
+;
+; LINUX64BE-LABEL: print_tls_func:
+; LINUX64BE:       # %bb.0: # %entry
+; LINUX64BE-NEXT:    mflr r0
+; LINUX64BE-NEXT:    stdu r1, -128(r1)
+; LINUX64BE-NEXT:    std r0, 144(r1)
+; LINUX64BE-NEXT:    .cfi_def_cfa_offset 128
+; LINUX64BE-NEXT:    .cfi_offset lr, 16
+; LINUX64BE-NEXT:    .cfi_offset r30, -16
+; LINUX64BE-NEXT:    addis r3, r2, a@got@tlsld@ha
+; LINUX64BE-NEXT:    std r30, 112(r1) # 8-byte Folded Spill
+; LINUX64BE-NEXT:    addi r3, r3, a@got@tlsld@l
+; LINUX64BE-NEXT:    bl __tls_get_addr(a@tlsld)
+; LINUX64BE-NEXT:    nop
+; LINUX64BE-NEXT:    addis r4, r2, b@got@tlsld@ha
+; LINUX64BE-NEXT:    addis r3, r3, a@dtprel@ha
+; LINUX64BE-NEXT:    addi r30, r3, a@dtprel@l
+; LINUX64BE-NEXT:    addi r3, r4, b@got@tlsld@l
+; LINUX64BE-NEXT:    bl __tls_get_addr(b@tlsld)
+; LINUX64BE-NEXT:    nop
+; LINUX64BE-NEXT:    addis r3, r3, b@dtprel@ha
+; LINUX64BE-NEXT:    addi r5, r3, b@dtprel@l
+; LINUX64BE-NEXT:    addis r3, r2, .L__ModuleStringPool@toc@ha
+; LINUX64BE-NEXT:    addi r4, r3, .L__ModuleStringPool@toc@l
+; LINUX64BE-NEXT:    addi r3, r4, 71
+; LINUX64BE-NEXT:    addi r6, r4, 56
+; LINUX64BE-NEXT:    mr r4, r30
+; LINUX64BE-NEXT:    bl printf
+; LINUX64BE-NEXT:    nop
+; LINUX64BE-NEXT:    ld r30, 112(r1) # 8-byte Folded Reload
+; LINUX64BE-NEXT:    addi r1, r1, 128
+; LINUX64BE-NEXT:    ld r0, 16(r1)
+; LINUX64BE-NEXT:    mtlr r0
+; LINUX64BE-NEXT:    blr
+entry:
+  %0 = tail call align 1 ptr @llvm.threadlocal.address.p0(ptr align 1 @a)
+  %1 = tail call align 1 ptr @llvm.threadlocal.address.p0(ptr align 1 @b)
+  %call = tail call signext i32 (ptr, ...) @printf(ptr noundef nonnull dereferenceable(1) @.str, ptr noundef nonnull %0, ptr noundef nonnull %1, ptr noundef nonnull @c)
+  ret void
+}
+
+define void @test_func() {
+; CHECK64-LABEL: test_func:
+; CHECK64:       # %bb.0: # %entry
+; CHECK64-NEXT:    mflr r0
+; CHECK64-NEXT:    stdu r1, -112(r1)
+; CHECK64-NEXT:    ld r3, L..C1(r2) # @__ModuleStringPool
+; CHECK64-NEXT:    std r0, 128(r1)
+; CHECK64-NEXT:    addi r3, r3, 16
+; CHECK64-NEXT:    bl .callee[PR]
+; CHECK64-NEXT:    nop
+; CHECK64-NEXT:    addi r1, r1, 112
+; CHECK64-NEXT:    ld r0, 16(r1)
+; CHECK64-NEXT:    mtlr r0
+; CHECK64-NEXT:    blr
+;
+; CHECK32-LABEL: test_func:
+; CHECK32:       # %bb.0: # %entry
+; CHECK32-NEXT:    mflr r0
+; CHECK32-NEXT:    stwu r1, -64(r1)
+; CHECK32-NEXT:    lwz r3, L..C1(r2) # @__ModuleStringPool
+; CHECK32-NEXT:    stw r0, 72(r1)
+; CHECK32-NEXT:    addi r3, r3, 16
+; CHECK32-NEXT:    bl .callee[PR]
+; CHECK32-NEXT:    nop
+; CHECK32-NEXT:    addi r1, r1, 64
+; CHECK32-NEXT:    lwz r0, 8(r1)
+; CHECK32-NEXT:    mtlr r0
+; CHECK32-NEXT:    blr
+;
+; LINUX64LE-LABEL: test_func:
+; LINUX64LE:       # %bb.0: # %entry
+; LINUX64LE-NEXT:    mflr r0
+; LINUX64LE-NEXT:    stdu r1, -32(r1)
+; LINUX64LE-NEXT:    std r0, 48(r1)
+; LINUX64LE-NEXT:    .cfi_def_cfa_offset 32
+; LINUX64LE-NEXT:    .cfi_offset lr, 16
+; LINUX64LE-NEXT:    addis r3, r2, .L__ModuleStringPool@toc@ha
+; LINUX64LE-NEXT:    addi r3, r3, .L__ModuleStringPool@toc@l
+; LINUX64LE-NEXT:    addi r3, r3, 16
+; LINUX64LE-NEXT:    bl callee
+; LINUX64LE-NEXT:    nop
+; LINUX64LE-NEXT:    addi r1, r1, 32
+; LINUX64LE-NEXT:    ld r0, 16(r1)
+; LINUX64LE-NEXT:    mtlr r0
+; LINUX64LE-NEXT:    blr
+;
+; LINUX64BE-LABEL: test_func:
+; LINUX64BE:       # %bb.0: # %entry
+; LINUX64BE-NEXT:    mflr r0
+; LINUX64BE-NEXT:    stdu r1, -112(r1)
+; LINUX64BE-NEXT:    std r0, 128(r1)
+; LINUX64BE-NEXT:    .cfi_def_cfa_offset 112
+; LINUX64BE-NEXT:    .cfi_offset lr, 16
+; LINUX64BE-NEXT:    addis r3, r2, .L__ModuleStringPool@toc@ha
+; LINUX64BE-NEXT:    addi r3, r3, .L__ModuleStringPool@toc@l
+; LINUX64BE-NEXT:    addi r3, r3, 16
+; LINUX64BE-NEXT:    bl callee
+; LINUX64BE-NEXT:    nop
+; LINUX64BE-NEXT:    addi r1, r1, 112
+; LINUX64BE-NEXT:    ld r0, 16(r1)
+; LINUX64BE-NEXT:    mtlr r0
+; LINUX64BE-NEXT:    blr
+entry:
+  tail call void @callee(ptr noundef nonnull @d) #4
+  ret void
+}
+
+define void @test_func2() {
+; CHECK64-LABEL: test_func2:
+; CHECK64:       # %bb.0: # %entry
+; CHECK64-NEXT:    mflr r0
+; CHECK64-NEXT:    stdu r1, -112(r1)
+; CHECK64-NEXT:    ld r3, L..C1(r2) # @__ModuleStringPool
+; CHECK64-NEXT:    std r0, 128(r1)
+; CHECK64-NEXT:    bl .callee2[PR]
+; CHECK64-NEXT:    nop
+; CHECK64-NEXT:    addi r1, r1, 112
+; CHECK64-NEXT:    ld r0, 16(r1)
+; CHECK64-NEXT:    mtlr r0
+; CHECK64-NEXT:    blr
+;
+; CHECK32-LABEL: test_func2:
+; CHECK32:       # %bb.0: # %entry
+; CHECK32-NEXT:    mflr r0
+; CHECK32-NEXT:    stwu r1, -64(r1)
+; CHECK32-NEXT:    lwz r3, L..C1(r2) # @__ModuleStringPool
+; CHECK32-NEXT:    stw r0, 72(r1)
+; CHECK32-NEXT:    bl .callee2[PR]
+; CHECK32-NEXT:    nop
+; CHECK32-NEXT:    addi r1, r1, 64
+; CHECK32-NEXT:    lwz r0, 8(r1)
+; CHECK32-NEXT:    mtlr r0
+; CHECK32-NEXT:    blr
+;
+; LINUX64LE-LABEL: test_func2:
+; LINUX64LE:       # %bb.0: # %entry
+; LINUX64LE-NEXT:    mflr r0
+; LINUX64LE-NEXT:    stdu r1, -32(r1)
+; LINUX64LE-NEXT:    std r0, 48(r1)
+; LINUX64LE-NEXT:    .cfi_def_cfa_offset 32
+; LINUX64LE-NEXT:    .cfi_offset lr, 16
+; LINUX64LE-NEXT:    addis r3, r2, .L__ModuleStringPool@toc@ha
+; LINUX64LE-NEXT:    addi r3, r3, .L__ModuleStringPool@toc@l
+; LINUX64LE-NEXT:    bl callee2
+; LINUX64LE-NEXT:    nop
+; LINUX64LE-NEXT:    addi r1, r1, 32
+; LINUX64LE-NEXT:    ld r0, 16(r1)
+; LINUX64LE-NEXT:    mtlr r0
+; LINUX64LE-NEXT:    blr
+;
+; LINUX64BE-LABEL: test_func2:
+; LINUX64BE:       # %bb.0: # %entry
+; LINUX64BE-NEXT:    mflr r0
+; LINUX64BE-NEXT:    stdu r1, -112(r1)
+; LINUX64BE-NEXT:    std r0, 128(r1)
+; LINUX64BE-NEXT:    .cfi_def_cfa_offset 112
+; LINUX64BE-NEXT:    .cfi_offset lr, 16
+; LINUX64BE-NEXT:    addis r3, r2, .L__ModuleStringPool@toc@ha
+; LINUX64BE-NEXT:    addi r3, r3, .L__ModuleStringPool@toc@l
+; LINUX64BE-NEXT:    bl callee2
+; LINUX64BE-NEXT:    nop
+; LINUX64BE-NEXT:    addi r1, r1, 112
+; LINUX64BE-NEXT:    ld r0, 16(r1)
+; LINUX64BE-NEXT:    mtlr r0
+; LINUX64BE-NEXT:    blr
+entry:
+  tail call void @callee2(ptr noundef nonnull @e) #4
+  ret void
+}
+
+; Check the contents of the TLS data and the __ModuleStringPool structure to
+; check that TLS data has been skipped during string pool merging.
+
+; CHECK64: 	.csect a[TL],2
+; CHECK64-NEXT:	.lglobl	a[TL]
+; CHECK64-NEXT:	.string	"tls1"
+; CHECK64:	.csect b[TL],2
+; CHECK64-NEXT:	.lglobl	b[TL]
+; CHECK64-NEXT:	.string	"tls2"
+; CHECK64:	.csect L..__ModuleStringPool[RO],2
+; CHECK64-NEXT:	.align	2
+; CHECK64-NEXT:	.vbyte	4, 0x42af999a
+; CHECK64-NEXT:	.vbyte	4, 0x42b1999a
+; CHECK64-NEXT:	.vbyte	4, 0x42b3cccd
+; CHECK64-NEXT:	.vbyte	4, 0x42b5999a
+; CHECK64-NEXT:	.vbyte	4, 1
+; CHECK64-NEXT:	.vbyte	4, 2
+; CHECK64-NEXT:	.vbyte	4, 3
+; CHECK64-NEXT:	.vbyte	4, 4
+; CHECK64-NEXT:	.vbyte	4, 5
+; CHECK64-NEXT:	.vbyte	4, 6
+; CHECK64-NEXT:	.vbyte	4, 7
+; CHECK64-NEXT:	.vbyte	4, 8
+; CHECK64-NEXT:	.vbyte	4, 9
+; CHECK64-NEXT:	.vbyte	4, 10
+; CHECK64-NEXT:	.string	"Regular global"
+; CHECK64-NEXT:	.byte	'T,'L,'S,' ,'v,'a,'r,'i,'a,'b,'l,'e,' ,'1,',,' ,'2,' ,'a,'n,'d,' ,'n,'o,'n,'-,'T,'L,'S,' ,'v,'a,'r,':,' ,'%,'s,',,' ,'%,'s,',,' ,'%,'s,0012,0000
+; CHECK64: L..C1:
+; CHECK64-NEXT:	.tc L..__ModuleStringPool[TC],L..__ModuleStringPool[RO]
+; CHECK64: L..C2:
+; CHECK64-NEXT:	.tc a[TC],a[TL]@ld
+; CHECK64: L..C3:
+; CHECK64-NEXT:	.tc b[TC],b[TL]@ld
+
+; CHECK32: 	.csect a[TL],2
+; CHECK32-NEXT:	.lglobl	a[TL]
+; CHECK32-NEXT:	.string	"tls1"
+; CHECK32:	.csect b[TL],2
+; CHECK32-NEXT:	.lglobl	b[TL]
+; CHECK32-NEXT:	.string	"tls2"
+; CHECK32:	.csect L..__ModuleStringPool[RO],2
+; CHECK32-NEXT:	.align	2
+; CHECK32-NEXT:	.vbyte	4, 0x42af999a
+; CHECK32-NEXT:	.vbyte	4, 0x42b1999a
+; CHECK32-NEXT:	.vbyte	4, 0x42b3cccd
+; CHECK32-NEXT:	.vbyte	4, 0x42b5999a
+; CHECK32-NEXT:	.vbyte	4, 1
+; CHECK32-NEXT:	.vbyte	4, 2
+; CHECK32-NEXT:	.vbyte	4, 3
+; CHECK32-NEXT:	.vbyte	4, 4
+; CHECK32-NEXT:	.vbyte	4, 5
+; CHECK32-NEXT:	.vbyte	4, 6
+; CHECK32-NEXT:	.vbyte	4, 7
+; CHECK32-NEXT:	.vbyte	4, 8
+; CHECK32-NEXT:	.vbyte	4, 9
+; CHECK32-NEXT:	.vbyte	4, 10
+; CHECK32-NEXT:	.string	"Regular global"
+; CHECK32-NEXT:	.byte	'T,'L,'S,' ,'v,'a,'r,'i,'a,'b,'l,'e,' ,'1,',,' ,'2,' ,'a,'n,'d,' ,'n,'o,'n,'-,'T,'L,'S,' ,'v,'a,'r,':,' ,'%,'s,',,' ,'%,'s,',,' ,'%,'s,0012,0000
+; CHECK32: L..C1:
+; CHECK32-NEXT:	.tc L..__ModuleStringPool[TC],L..__ModuleStringPool[RO]
+; CHECK32: L..C2:
+; CHECK32-NEXT:	.tc a[TC],a[TL]@ld
+; CHECK32: L..C3:
+; CHECK32-NEXT:	.tc b[TC],b[TL]@ld
+
+; LINUX64LE: a:
+; LINUX64LE-NEXT:	.asciz	"tls1"
+; LINUX64LE-NEXT:	.size	a, 5
+; LINUX64LE: b:
+; LINUX64LE-NEXT:	.asciz	"tls2"
+; LINUX64LE-NEXT:	.size	b, 5
+; LINUX64LE: .L__ModuleStringPool:
+; LINUX64LE-NEXT:	.long	0x42af999a
+; LINUX64LE-NEXT:	.long	0x42b1999a
+; LINUX64LE-NEXT:	.long	0x42b3cccd
+; LINUX64LE-NEXT:	.long	0x42b5999a
+; LINUX64LE-NEXT:	.long	1
+; LINUX64LE-NEXT:	.long	2
+; LINUX64LE-NEXT:	.long	3
+; LINUX64LE-NEXT:	.long	4
+; LINUX64LE-NEXT:	.long	5
+; LINUX64LE-NEXT:	.long	6
+; LINUX64LE-NEXT:	.long	7
+; LINUX64LE-NEXT:	.long	8
+; LINUX64LE-NEXT:	.long	9
+; LINUX64LE-NEXT:	.long	10
+; LINUX64LE-NEXT:	.asciz	"Regular global"
+; LINUX64LE-NEXT:	.asciz	"TLS variable 1, 2 and non-TLS var: %s, %s, %s\n"
+
+; LINUX64BE: a:
+; LINUX64BE-NEXT:	.asciz	"tls1"
+; LINUX64BE-NEXT:	.size	a, 5
+; LINUX64BE: b:
+; LINUX64BE-NEXT:	.asciz	"tls2"
+; LINUX64BE-NEXT:	.size	b, 5
+; LINUX64BE: .L__ModuleStringPool:
+; LINUX64BE-NEXT:	.long	0x42af999a
+; LINUX64BE-NEXT:	.long	0x42b1999a
+; LINUX64BE-NEXT:	.long	0x42b3cccd
+; LINUX64BE-NEXT:	.long	0x42b5999a
+; LINUX64BE-NEXT:	.long	1
+; LINUX64BE-NEXT:	.long	2
+; LINUX64BE-NEXT:	.long	3
+; LINUX64BE-NEXT:	.long	4
+; LINUX64BE-NEXT:	.long	5
+; LINUX64BE-NEXT:	.long	6
+; LINUX64BE-NEXT:	.long	7
+; LINUX64BE-NEXT:	.long	8
+; LINUX64BE-NEXT:	.long	9
+; LINUX64BE-NEXT:	.long	10
+; LINUX64BE-NEXT:	.asciz	"Regular global"
+; LINUX64BE-NEXT:	.asciz	"TLS variable 1, 2 and non-TLS var: %s, %s, %s\n"

From 1aa8a6f6917edbdcad53afad3b00c7e86aa57ffe Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Fri, 16 Aug 2024 21:20:23 +0200
Subject: [PATCH 224/441] [VPlan] Compute cost for most opcodes in
 VPWidenRecipe (NFCI). (#98764)

Implement VPWidenRecipe::computeCost for most cases (except
UDiv,SDiv,URem,SRem which require additional logic).

Note that this specializes `::computeCost` instead of `::cost`, as
`VPRecipeBase::cost` is responsible for skipping cost-computations
for pre-computed recipes for now.

The most recent version of the VPlan-based cost model introduction
has been committed on Jul 10 (b841e2eca3b5c8b) and we should
probably give it at least a week in case additional mismatches surface.

PR: https://github.com/llvm/llvm-project/pull/98764
---
 .../Transforms/Vectorize/LoopVectorize.cpp    |  7 +-
 llvm/lib/Transforms/Vectorize/VPlan.h         | 22 ++++--
 .../lib/Transforms/Vectorize/VPlanRecipes.cpp | 74 +++++++++++++++++++
 3 files changed, 93 insertions(+), 10 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 58ef665e86d65ea..27e99694aafd305 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -4354,8 +4354,8 @@ void LoopVectorizationPlanner::emitInvalidCostRemarks(
   SmallVector<RecipeVFPair> InvalidCosts;
   for (const auto &Plan : VPlans) {
     for (ElementCount VF : Plan->vectorFactors()) {
-      VPCostContext CostCtx(CM.TTI, Legal->getWidestInductionType(), LLVMCtx,
-                            CM);
+      VPCostContext CostCtx(CM.TTI, *CM.TLI, Legal->getWidestInductionType(),
+                            LLVMCtx, CM);
       auto Iter = vp_depth_first_deep(Plan->getVectorLoopRegion()->getEntry());
       for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Iter)) {
         for (auto &R : *VPBB) {
@@ -7062,7 +7062,8 @@ InstructionCost LoopVectorizationPlanner::cost(VPlan &Plan,
                                                ElementCount VF) const {
   InstructionCost Cost = 0;
   LLVMContext &LLVMCtx = OrigLoop->getHeader()->getContext();
-  VPCostContext CostCtx(CM.TTI, Legal->getWidestInductionType(), LLVMCtx, CM);
+  VPCostContext CostCtx(CM.TTI, *CM.TLI, Legal->getWidestInductionType(),
+                        LLVMCtx, CM);
 
   // Cost modeling for inductions is inaccurate in the legacy cost model
   // compared to the recipes that are generated. To match here initially during
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 016ad75c21ceb07..a99f3882092c2c9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -736,14 +736,16 @@ class VPLiveOut : public VPUser {
 /// Struct to hold various analysis needed for cost computations.
 struct VPCostContext {
   const TargetTransformInfo &TTI;
+  const TargetLibraryInfo &TLI;
   VPTypeAnalysis Types;
   LLVMContext &LLVMCtx;
   LoopVectorizationCostModel &CM;
   SmallPtrSet<Instruction *, 8> SkipCostComputation;
 
-  VPCostContext(const TargetTransformInfo &TTI, Type *CanIVTy,
-                LLVMContext &LLVMCtx, LoopVectorizationCostModel &CM)
-      : TTI(TTI), Types(CanIVTy, LLVMCtx), LLVMCtx(LLVMCtx), CM(CM) {}
+  VPCostContext(const TargetTransformInfo &TTI, const TargetLibraryInfo &TLI,
+                Type *CanIVTy, LLVMContext &LLVMCtx,
+                LoopVectorizationCostModel &CM)
+      : TTI(TTI), TLI(TLI), Types(CanIVTy, LLVMCtx), LLVMCtx(LLVMCtx), CM(CM) {}
 
   /// Return the cost for \p UI with \p VF using the legacy cost model as
   /// fallback until computing the cost of all recipes migrates to VPlan.
@@ -796,7 +798,7 @@ class VPRecipeBase : public ilist_node_with_parent<VPRecipeBase, VPBasicBlock>,
   /// Return the cost of this recipe, taking into account if the cost
   /// computation should be skipped and the ForceTargetInstructionCost flag.
   /// Also takes care of printing the cost for debugging.
-  virtual InstructionCost cost(ElementCount VF, VPCostContext &Ctx);
+  InstructionCost cost(ElementCount VF, VPCostContext &Ctx);
 
   /// Insert an unlinked recipe into a basic block immediately before
   /// the specified recipe.
@@ -860,9 +862,11 @@ class VPRecipeBase : public ilist_node_with_parent<VPRecipeBase, VPBasicBlock>,
   DebugLoc getDebugLoc() const { return DL; }
 
 protected:
-  /// Compute the cost of this recipe using the legacy cost model and the
-  /// underlying instructions.
-  InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const;
+  /// Compute the cost of this recipe either using a recipe's specialized
+  /// implementation or using the legacy cost model and the underlying
+  /// instructions.
+  virtual InstructionCost computeCost(ElementCount VF,
+                                      VPCostContext &Ctx) const;
 };
 
 // Helper macro to define common classof implementations for recipes.
@@ -1426,6 +1430,10 @@ class VPWidenRecipe : public VPRecipeWithIRFlags {
   /// processing State.VF elements.
   void execute(VPTransformState &State) override;
 
+  /// Return the cost of this VPWidenRecipe.
+  InstructionCost computeCost(ElementCount VF,
+                              VPCostContext &Ctx) const override;
+
   unsigned getOpcode() const { return Opcode; }
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index bc57ea4d52471e4..c9d603612aeceaa 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1140,6 +1140,80 @@ void VPWidenRecipe::execute(VPTransformState &State) {
 #endif
 }
 
+InstructionCost VPWidenRecipe::computeCost(ElementCount VF,
+                                           VPCostContext &Ctx) const {
+  TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
+  switch (Opcode) {
+  case Instruction::FNeg: {
+    Type *VectorTy =
+        ToVectorTy(Ctx.Types.inferScalarType(this->getVPSingleValue()), VF);
+    return Ctx.TTI.getArithmeticInstrCost(
+        Opcode, VectorTy, CostKind,
+        {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
+        {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None});
+  }
+
+  case Instruction::UDiv:
+  case Instruction::SDiv:
+  case Instruction::SRem:
+  case Instruction::URem:
+    // More complex computation, let the legacy cost-model handle this for now.
+    return Ctx.getLegacyCost(cast<Instruction>(getUnderlyingValue()), VF);
+  case Instruction::Add:
+  case Instruction::FAdd:
+  case Instruction::Sub:
+  case Instruction::FSub:
+  case Instruction::Mul:
+  case Instruction::FMul:
+  case Instruction::FDiv:
+  case Instruction::FRem:
+  case Instruction::Shl:
+  case Instruction::LShr:
+  case Instruction::AShr:
+  case Instruction::And:
+  case Instruction::Or:
+  case Instruction::Xor: {
+    VPValue *RHS = getOperand(1);
+    // Certain instructions can be cheaper to vectorize if they have a constant
+    // second vector operand. One example of this are shifts on x86.
+    TargetTransformInfo::OperandValueInfo RHSInfo = {
+        TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None};
+    if (RHS->isLiveIn())
+      RHSInfo = Ctx.TTI.getOperandInfo(RHS->getLiveInIRValue());
+
+    if (RHSInfo.Kind == TargetTransformInfo::OK_AnyValue &&
+        getOperand(1)->isDefinedOutsideVectorRegions())
+      RHSInfo.Kind = TargetTransformInfo::OK_UniformValue;
+    Type *VectorTy =
+        ToVectorTy(Ctx.Types.inferScalarType(this->getVPSingleValue()), VF);
+    Instruction *CtxI = dyn_cast_or_null<Instruction>(getUnderlyingValue());
+
+    SmallVector<const Value *, 4> Operands;
+    if (CtxI)
+      Operands.append(CtxI->value_op_begin(), CtxI->value_op_end());
+    return Ctx.TTI.getArithmeticInstrCost(
+        Opcode, VectorTy, CostKind,
+        {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
+        RHSInfo, Operands, CtxI, &Ctx.TLI);
+  }
+  case Instruction::Freeze: {
+    // This opcode is unknown. Assume that it is the same as 'mul'.
+    Type *VectorTy =
+        ToVectorTy(Ctx.Types.inferScalarType(this->getVPSingleValue()), VF);
+    return Ctx.TTI.getArithmeticInstrCost(Instruction::Mul, VectorTy, CostKind);
+  }
+  case Instruction::ICmp:
+  case Instruction::FCmp: {
+    Instruction *CtxI = dyn_cast_or_null<Instruction>(getUnderlyingValue());
+    Type *VectorTy = ToVectorTy(Ctx.Types.inferScalarType(getOperand(0)), VF);
+    return Ctx.TTI.getCmpSelInstrCost(Opcode, VectorTy, nullptr, getPredicate(),
+                                      CostKind, CtxI);
+  }
+  default:
+    llvm_unreachable("Unsupported opcode for instruction");
+  }
+}
+
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void VPWidenRecipe::print(raw_ostream &O, const Twine &Indent,
                           VPSlotTracker &SlotTracker) const {

From 71d54faa65f490f2a2825e591c3eb25ff7996221 Mon Sep 17 00:00:00 2001
From: Justin Bogner <mail@justinbogner.com>
Date: Fri, 16 Aug 2024 21:21:24 +0200
Subject: [PATCH 225/441] [DirectX] Revert specialized createOp methods part of
 #101250

In 8cf85653b6f5 "[DirectX] Make DXILOpBuilder's API more useable" we introduced
specialized createOp methods for each DirectX op for convenience, but the API
is buggy and untested. It also isn't actually as useful as I imagined it would
be since we don't have argument names or const-ness represented in DXIL.td
currently. Remove these methods for now and we can reintroduce them if we
actually need them later.

Pull Request: https://github.com/llvm/llvm-project/pull/104245
---
 llvm/lib/Target/DirectX/DXILOpBuilder.cpp |  2 +-
 llvm/lib/Target/DirectX/DXILOpBuilder.h   | 14 +-------------
 2 files changed, 2 insertions(+), 14 deletions(-)

diff --git a/llvm/lib/Target/DirectX/DXILOpBuilder.cpp b/llvm/lib/Target/DirectX/DXILOpBuilder.cpp
index 42df7c90cb337ea..91e6931b3f788eb 100644
--- a/llvm/lib/Target/DirectX/DXILOpBuilder.cpp
+++ b/llvm/lib/Target/DirectX/DXILOpBuilder.cpp
@@ -424,7 +424,7 @@ Expected<CallInst *> DXILOpBuilder::tryCreateOp(dxil::OpCode OpCode,
   return B.CreateCall(DXILFn, OpArgs);
 }
 
-CallInst *DXILOpBuilder::createOp(dxil::OpCode OpCode, ArrayRef<Value *> &Args,
+CallInst *DXILOpBuilder::createOp(dxil::OpCode OpCode, ArrayRef<Value *> Args,
                                   Type *RetTy) {
   Expected<CallInst *> Result = tryCreateOp(OpCode, Args, RetTy);
   if (Error E = Result.takeError())
diff --git a/llvm/lib/Target/DirectX/DXILOpBuilder.h b/llvm/lib/Target/DirectX/DXILOpBuilder.h
index ff66f39a3ceb3d6..5d83357f7a2e944 100644
--- a/llvm/lib/Target/DirectX/DXILOpBuilder.h
+++ b/llvm/lib/Target/DirectX/DXILOpBuilder.h
@@ -33,25 +33,13 @@ class DXILOpBuilder {
 
   /// Create a call instruction for the given DXIL op. The arguments
   /// must be valid for an overload of the operation.
-  CallInst *createOp(dxil::OpCode Op, ArrayRef<Value *> &Args,
+  CallInst *createOp(dxil::OpCode Op, ArrayRef<Value *> Args,
                      Type *RetTy = nullptr);
 
-#define DXIL_OPCODE(Op, Name)                                                  \
-  CallInst *create##Name##Op(ArrayRef<Value *> &Args, Type *RetTy = nullptr) { \
-    return createOp(dxil::OpCode(Op), Args, RetTy);                            \
-  }
-#include "DXILOperation.inc"
-
   /// Try to create a call instruction for the given DXIL op. Fails if the
   /// overload is invalid.
   Expected<CallInst *> tryCreateOp(dxil::OpCode Op, ArrayRef<Value *> Args,
                                    Type *RetTy = nullptr);
-#define DXIL_OPCODE(Op, Name)                                                  \
-  Expected<CallInst *> tryCreate##Name##Op(ArrayRef<Value *> &Args,            \
-                                           Type *RetTy = nullptr) {            \
-    return tryCreateOp(dxil::OpCode(Op), Args, RetTy);                         \
-  }
-#include "DXILOperation.inc"
 
   /// Return the name of the given opcode.
   static const char *getOpCodeName(dxil::OpCode DXILOp);

From 7ff377ba60bf9e615f3efc60268fbd74b75e54c4 Mon Sep 17 00:00:00 2001
From: Tyler Nowicki <tyler.nowicki@amd.com>
Date: Fri, 16 Aug 2024 15:23:51 -0400
Subject: [PATCH 226/441] [Analysis] Fix null ptr dereference when using
 WriteGraph without branch probability info (#104102)

The call to 'CFGInfo->getBPI()->getEdgeProbability(Node, SuccBB);' fails
when BPI is not provided. In this case we can give up and not print any
edge attributes.

---------

Co-authored-by: tnowicki <tnowicki.nowicki@amd.com>
---
 llvm/include/llvm/Analysis/CFGPrinter.h     |  7 +-
 llvm/unittests/Analysis/CMakeLists.txt      |  1 +
 llvm/unittests/Analysis/GraphWriterTest.cpp | 77 +++++++++++++++++++++
 3 files changed, 82 insertions(+), 3 deletions(-)
 create mode 100644 llvm/unittests/Analysis/GraphWriterTest.cpp

diff --git a/llvm/include/llvm/Analysis/CFGPrinter.h b/llvm/include/llvm/Analysis/CFGPrinter.h
index 7d4d3a574b9e81e..cd785331d1f1468 100644
--- a/llvm/include/llvm/Analysis/CFGPrinter.h
+++ b/llvm/include/llvm/Analysis/CFGPrinter.h
@@ -265,18 +265,19 @@ struct DOTGraphTraits<DOTFuncInfo *> : public DefaultDOTGraphTraits {
   /// Display the raw branch weights from PGO.
   std::string getEdgeAttributes(const BasicBlock *Node, const_succ_iterator I,
                                 DOTFuncInfo *CFGInfo) {
+    // If BPI is not provided do not display any edge attributes
+    if (!CFGInfo->showEdgeWeights())
+      return "";
+
     unsigned OpNo = I.getSuccessorIndex();
     const Instruction *TI = Node->getTerminator();
     BasicBlock *SuccBB = TI->getSuccessor(OpNo);
     auto BranchProb = CFGInfo->getBPI()->getEdgeProbability(Node, SuccBB);
     double WeightPercent = ((double)BranchProb.getNumerator()) /
                            ((double)BranchProb.getDenominator());
-
     std::string TTAttr =
         formatv("tooltip=\"{0} -> {1}\\nProbability {2:P}\" ", getBBName(Node),
                 getBBName(SuccBB), WeightPercent);
-    if (!CFGInfo->showEdgeWeights())
-      return TTAttr;
 
     if (TI->getNumSuccessors() == 1)
       return TTAttr + "penwidth=2";
diff --git a/llvm/unittests/Analysis/CMakeLists.txt b/llvm/unittests/Analysis/CMakeLists.txt
index 3cba630867a83b3..d9eb81faac42add 100644
--- a/llvm/unittests/Analysis/CMakeLists.txt
+++ b/llvm/unittests/Analysis/CMakeLists.txt
@@ -25,6 +25,7 @@ set(ANALYSIS_TEST_SOURCES
   DDGTest.cpp
   DomTreeUpdaterTest.cpp
   DXILResourceTest.cpp
+  GraphWriterTest.cpp
   GlobalsModRefTest.cpp
   FunctionPropertiesAnalysisTest.cpp
   InlineCostTest.cpp
diff --git a/llvm/unittests/Analysis/GraphWriterTest.cpp b/llvm/unittests/Analysis/GraphWriterTest.cpp
new file mode 100644
index 000000000000000..a723c92d1576185
--- /dev/null
+++ b/llvm/unittests/Analysis/GraphWriterTest.cpp
@@ -0,0 +1,77 @@
+//===- GraphWriterTest.cpp - GraphWriter unit tests -----------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Support/GraphWriter.h"
+#include "llvm/Analysis/CFGPrinter.h"
+#include "llvm/AsmParser/Parser.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/Module.h"
+#include "llvm/Support/SourceMgr.h"
+#include "llvm/Support/raw_ostream.h"
+#include "gtest/gtest.h"
+#include <string>
+
+#define ASSERT_NO_ERROR(x)                                                     \
+  if (std::error_code ASSERT_NO_ERROR_ec = x) {                                \
+    SmallString<128> MessageStorage;                                           \
+    raw_svector_ostream Message(MessageStorage);                               \
+    Message << #x ": did not return errc::success.\n"                          \
+            << "error number: " << ASSERT_NO_ERROR_ec.value() << "\n"          \
+            << "error message: " << ASSERT_NO_ERROR_ec.message() << "\n";      \
+    GTEST_FATAL_FAILURE_(MessageStorage.c_str());                              \
+  } else {                                                                     \
+  }
+
+namespace llvm {
+namespace {
+
+class GraphWriterTest : public testing::Test {
+protected:
+  LLVMContext C;
+
+  std::unique_ptr<Module> makeLLVMModule() {
+    const char *ModuleStrig = "define i32 @f(i32 %x) {\n"
+                              "bb0:\n"
+                              "  %y1 = icmp eq i32 %x, 0 \n"
+                              "  br i1 %y1, label %bb1, label %bb2 \n"
+                              "bb1:\n"
+                              "  br label %bb3\n"
+                              "bb2:\n"
+                              "  br label %bb3\n"
+                              "bb3:\n"
+                              "  %y2 = phi i32 [0, %bb1], [1, %bb2] \n"
+                              "  ret i32 %y2\n"
+                              "}\n";
+    SMDiagnostic Err;
+    return parseAssemblyString(ModuleStrig, Err, C);
+  }
+};
+
+static void writeCFGToDotFile(Function &F, std::string Name,
+                              bool CFGOnly = false) {
+  std::error_code EC;
+  raw_fd_ostream File(Name + ".dot", EC, sys::fs::OpenFlags::OF_Text);
+
+  DOTFuncInfo CFGInfo(&F);
+
+  ASSERT_NO_ERROR(EC);
+  // Test intentionally does not pass BPI, WriteGraph should work without it.
+  WriteGraph(File, &CFGInfo, CFGOnly);
+}
+
+TEST_F(GraphWriterTest, WriteCFGDotFileTest) {
+  auto M = makeLLVMModule();
+  Function *F = M->getFunction("f");
+
+  writeCFGToDotFile(*F, "test-full");
+  writeCFGToDotFile(*F, "test-cfg-only", true);
+}
+
+} // end anonymous namespace
+} // end namespace llvm

From f5b81aa6ec371c13794aac82eb246a1a92966b43 Mon Sep 17 00:00:00 2001
From: gulfemsavrun <gulfem@google.com>
Date: Fri, 16 Aug 2024 12:33:50 -0700
Subject: [PATCH 227/441] [InstrProf] Support conditional counter updates
 (#102542)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This patch adds support for conditional counter updates in single byte
counters mode to reduce the write contention by first checking whether
the counter is set before overwriting it.

---------

Co-authored-by: Juan Manuel Martinez Caamaño <jmartinezcaamao@gmail.com>
---
 .../Instrumentation/InstrProfiling.cpp        | 16 +++++
 .../conditional-counter-updates.ll            | 69 +++++++++++++++++++
 2 files changed, 85 insertions(+)
 create mode 100644 llvm/test/Instrumentation/InstrProfiling/conditional-counter-updates.ll

diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
index 1805ea89272ec7f..1b3954a36699a05 100644
--- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
+++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
@@ -132,6 +132,11 @@ cl::opt<bool> AtomicFirstCounter(
              "the entry counter)"),
     cl::init(false));
 
+cl::opt<bool> ConditionalCounterUpdate(
+    "conditional-counter-update",
+    cl::desc("Do conditional counter updates in single byte counters mode)"),
+    cl::init(false));
+
 // If the option is not specified, the default behavior about whether
 // counter promotion is done depends on how instrumentaiton lowering
 // pipeline is setup, i.e., the default value of true of this option
@@ -1213,6 +1218,17 @@ Value *InstrLowerer::getBitmapAddress(InstrProfMCDCTVBitmapUpdate *I) {
 void InstrLowerer::lowerCover(InstrProfCoverInst *CoverInstruction) {
   auto *Addr = getCounterAddress(CoverInstruction);
   IRBuilder<> Builder(CoverInstruction);
+  if (ConditionalCounterUpdate) {
+    Instruction *SplitBefore = CoverInstruction->getNextNode();
+    auto &Ctx = CoverInstruction->getParent()->getContext();
+    auto *Int8Ty = llvm::Type::getInt8Ty(Ctx);
+    Value *Load = Builder.CreateLoad(Int8Ty, Addr, "pgocount");
+    Value *Cmp = Builder.CreateIsNotNull(Load, "pgocount.ifnonzero");
+    Instruction *ThenBranch =
+        SplitBlockAndInsertIfThen(Cmp, SplitBefore, false);
+    Builder.SetInsertPoint(ThenBranch);
+  }
+
   // We store zero to represent that this block is covered.
   Builder.CreateStore(Builder.getInt8(0), Addr);
   CoverInstruction->eraseFromParent();
diff --git a/llvm/test/Instrumentation/InstrProfiling/conditional-counter-updates.ll b/llvm/test/Instrumentation/InstrProfiling/conditional-counter-updates.ll
new file mode 100644
index 000000000000000..998443ec0cd568e
--- /dev/null
+++ b/llvm/test/Instrumentation/InstrProfiling/conditional-counter-updates.ll
@@ -0,0 +1,69 @@
+; RUN: opt < %s -S -passes=instrprof -conditional-counter-update | FileCheck %s
+
+target triple = "x86_64-unknown-linux-gnu"
+
+@__profn_foo = private constant [3 x i8] c"foo"
+@__profn_bar = private constant [3 x i8] c"bar"
+
+; CHECK-LABEL: define void @foo
+; CHECK-NEXT: %pgocount = load i8, ptr @__profc_foo, align 1
+; CHECK-NEXT: %pgocount.ifnonzero = icmp ne i8 %pgocount, 0
+; CHECK-NEXT: br i1 %pgocount.ifnonzero, label %1, label %2
+
+; CHECK-LABEL: 1:
+; CHECK-NEXT: store i8 0, ptr @__profc_foo, align 1
+; CHECK-NEXT: br label %2
+
+; CHECK-LABEL: 2:
+; CHECK-NEXT: ret void
+define void @foo() {
+  call void @llvm.instrprof.cover(ptr @__profn_foo, i64 0, i32 1, i32 0)
+  ret void
+}
+
+; CHECK-LABEL: define i32 @bar
+; CHECK-LABEL: entry:
+; CHECK-NEXT: %retval = alloca i32, align 4
+; CHECK-NEXT: %cond.addr = alloca i32, align 4
+; CHECK-NEXT: store i32 %cond, ptr %cond.addr, align 4
+; CHECK-NEXT: %pgocount = load i8, ptr @__profc_bar, align 1
+; CHECK-NEXT: %pgocount.ifnonzero = icmp ne i8 %pgocount, 0
+; CHECK-NEXT: br i1 %pgocount.ifnonzero, label %0, label %1
+
+; CHECK-LABEL: 0:                                  ; preds = %entry
+; CHECK-NEXT: store i8 0, ptr @__profc_bar, align 1
+; CHECK-NEXT: br label %1
+
+; CHECK-LABEL: 1:                                  ; preds = %entry, %0
+; CHECK-NEXT: %2 = load i32, ptr %cond.addr, align 4
+; CHECK-NEXT: %cmp = icmp slt i32 %2, 0
+; CHECK-NEXT: br i1 %cmp, label %if.then, label %if.end
+
+; CHECK-LABEL: if.then:                            ; preds = %1
+; CHECK-NEXT: %pgocount1 = load i8, ptr getelementptr inbounds ([3 x i8], ptr @__profc_bar, i32 0, i32 1), align 1
+; CHECK-NEXT: %pgocount.ifnonzero2 = icmp ne i8 %pgocount1, 0
+; CHECK-NEXT: br i1 %pgocount.ifnonzero2, label %3, label %4
+define i32 @bar(i32 %cond) #0 {
+entry:
+  %retval = alloca i32, align 4
+  %cond.addr = alloca i32, align 4
+  store i32 %cond, ptr %cond.addr, align 4
+  call void @llvm.instrprof.cover(ptr @__profn_bar, i64 0, i32 3, i32 0)
+  %0 = load i32, ptr %cond.addr, align 4
+  %cmp = icmp slt i32 %0, 0
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  call void @llvm.instrprof.cover(ptr @__profn_bar, i64 0, i32 3, i32 1)
+  store i32 -1, ptr %retval, align 4
+  br label %return
+
+if.end:                                           ; preds = %entry
+  call void @llvm.instrprof.cover(ptr @__profn_bar, i64 0, i32 3, i32 2)
+  store i32 0, ptr %retval, align 4
+  br label %return
+
+return:                                           ; preds = %if.end, %if.then
+  %1 = load i32, ptr %retval, align 4
+  ret i32 %1
+}

From 0e4f7c7207d26e141e271dc88102905d14815596 Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Fri, 16 Aug 2024 19:34:46 +0000
Subject: [PATCH 228/441] [gn build] Port 7ff377ba60bf

---
 llvm/utils/gn/secondary/llvm/unittests/Analysis/BUILD.gn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/utils/gn/secondary/llvm/unittests/Analysis/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/Analysis/BUILD.gn
index 4749ef0a733da34..ee635fd145c035c 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/Analysis/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/Analysis/BUILD.gn
@@ -28,6 +28,7 @@ unittest("AnalysisTests") {
     "DomTreeUpdaterTest.cpp",
     "FunctionPropertiesAnalysisTest.cpp",
     "GlobalsModRefTest.cpp",
+    "GraphWriterTest.cpp",
     "IRSimilarityIdentifierTest.cpp",
     "IVDescriptorsTest.cpp",
     "InlineCostTest.cpp",

From 808933f60b574c45087d8255ff113083c4536d85 Mon Sep 17 00:00:00 2001
From: Ivan Kosarev <ivan.kosarev@amd.com>
Date: Fri, 16 Aug 2024 22:48:15 +0300
Subject: [PATCH 229/441] [MC] Drop whitespace padding in AMDGPU combined
 asm/disasm tests. (#104433)

A follow-up from

<https://github.com/llvm/llvm-project/pull/92895#discussion_r1684390909>.
---
 llvm/include/llvm/MC/MCAsmInfo.h     |    8 +-
 llvm/test/MC/AMDGPU/gfx12_asm_vop1.s | 2388 +++++++++++++-------------
 llvm/tools/llvm-mc/llvm-mc.cpp       |    5 +
 3 files changed, 1204 insertions(+), 1197 deletions(-)

diff --git a/llvm/include/llvm/MC/MCAsmInfo.h b/llvm/include/llvm/MC/MCAsmInfo.h
index 278f16d685eed99..ccb7c524d45f8d0 100644
--- a/llvm/include/llvm/MC/MCAsmInfo.h
+++ b/llvm/include/llvm/MC/MCAsmInfo.h
@@ -493,6 +493,9 @@ class MCAsmInfo {
   /// Preserve Comments in assembly
   bool PreserveAsmComments;
 
+  /// The column (zero-based) at which asm comments should be printed.
+  unsigned CommentColumn = 40;
+
   /// True if the integrated assembler should interpret 'a >> b' constant
   /// expressions as logical rather than arithmetic.
   bool UseLogicalShr = true;
@@ -606,9 +609,8 @@ class MCAsmInfo {
   bool getStarIsPC() const { return StarIsPC; }
   const char *getSeparatorString() const { return SeparatorString; }
 
-  /// This indicates the column (zero-based) at which asm comments should be
-  /// printed.
-  unsigned getCommentColumn() const { return 40; }
+  unsigned getCommentColumn() const { return CommentColumn; }
+  void setCommentColumn(unsigned Col) { CommentColumn = Col; }
 
   StringRef getCommentString() const { return CommentString; }
   bool getRestrictCommentStringToStartOfStatement() const {
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop1.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop1.s
index d2fe7b3d8a95a35..bd640e786e4ac12 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop1.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop1.s
@@ -1,3608 +1,3608 @@
-// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | FileCheck --strict-whitespace --check-prefixes=GFX12,GFX12-ASM %s
-// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | %extract-encodings | llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32 -disassemble -show-encoding | FileCheck --strict-whitespace --check-prefixes=GFX12,GFX12-DIS %s
-// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s | FileCheck --strict-whitespace --check-prefixes=GFX12,GFX12-ASM %s
-// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s | %extract-encodings | llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 -disassemble -show-encoding | FileCheck --strict-whitespace --check-prefixes=GFX12,GFX12-DIS %s
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,+real-true16 -show-encoding -comment-column=0 %s | FileCheck --strict-whitespace --check-prefixes=GFX12,GFX12-ASM %s
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | %extract-encodings | llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32 -disassemble -show-encoding -comment-column=0 | FileCheck --strict-whitespace --check-prefixes=GFX12,GFX12-DIS %s
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -show-encoding -comment-column=0 %s | FileCheck --strict-whitespace --check-prefixes=GFX12,GFX12-ASM %s
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s | %extract-encodings | llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 -disassemble -show-encoding -comment-column=0 | FileCheck --strict-whitespace --check-prefixes=GFX12,GFX12-DIS %s
 
 // this file will be converted to true16 format when more true16 instructions are supported
 
 v_bfrev_b32_e32 v5, v1
-// GFX12: v_bfrev_b32_e32 v5, v1                  ; encoding: [0x01,0x71,0x0a,0x7e]
+// GFX12: v_bfrev_b32_e32 v5, v1 ; encoding: [0x01,0x71,0x0a,0x7e]
 
 v_bfrev_b32 v5, v255
-// GFX12: v_bfrev_b32_e32 v5, v255                ; encoding: [0xff,0x71,0x0a,0x7e]
+// GFX12: v_bfrev_b32_e32 v5, v255 ; encoding: [0xff,0x71,0x0a,0x7e]
 
 v_bfrev_b32 v5, s1
-// GFX12: v_bfrev_b32_e32 v5, s1                  ; encoding: [0x01,0x70,0x0a,0x7e]
+// GFX12: v_bfrev_b32_e32 v5, s1 ; encoding: [0x01,0x70,0x0a,0x7e]
 
 v_bfrev_b32 v5, s105
-// GFX12: v_bfrev_b32_e32 v5, s105                ; encoding: [0x69,0x70,0x0a,0x7e]
+// GFX12: v_bfrev_b32_e32 v5, s105 ; encoding: [0x69,0x70,0x0a,0x7e]
 
 v_bfrev_b32 v5, vcc_lo
-// GFX12: v_bfrev_b32_e32 v5, vcc_lo              ; encoding: [0x6a,0x70,0x0a,0x7e]
+// GFX12: v_bfrev_b32_e32 v5, vcc_lo ; encoding: [0x6a,0x70,0x0a,0x7e]
 
 v_bfrev_b32 v5, vcc_hi
-// GFX12: v_bfrev_b32_e32 v5, vcc_hi              ; encoding: [0x6b,0x70,0x0a,0x7e]
+// GFX12: v_bfrev_b32_e32 v5, vcc_hi ; encoding: [0x6b,0x70,0x0a,0x7e]
 
 v_bfrev_b32 v5, ttmp15
-// GFX12: v_bfrev_b32_e32 v5, ttmp15              ; encoding: [0x7b,0x70,0x0a,0x7e]
+// GFX12: v_bfrev_b32_e32 v5, ttmp15 ; encoding: [0x7b,0x70,0x0a,0x7e]
 
 v_bfrev_b32 v5, m0
-// GFX12: v_bfrev_b32_e32 v5, m0                  ; encoding: [0x7d,0x70,0x0a,0x7e]
+// GFX12: v_bfrev_b32_e32 v5, m0 ; encoding: [0x7d,0x70,0x0a,0x7e]
 
 v_bfrev_b32 v5, exec_lo
-// GFX12: v_bfrev_b32_e32 v5, exec_lo             ; encoding: [0x7e,0x70,0x0a,0x7e]
+// GFX12: v_bfrev_b32_e32 v5, exec_lo ; encoding: [0x7e,0x70,0x0a,0x7e]
 
 v_bfrev_b32 v5, exec_hi
-// GFX12: v_bfrev_b32_e32 v5, exec_hi             ; encoding: [0x7f,0x70,0x0a,0x7e]
+// GFX12: v_bfrev_b32_e32 v5, exec_hi ; encoding: [0x7f,0x70,0x0a,0x7e]
 
 v_bfrev_b32 v5, null
-// GFX12: v_bfrev_b32_e32 v5, null                ; encoding: [0x7c,0x70,0x0a,0x7e]
+// GFX12: v_bfrev_b32_e32 v5, null ; encoding: [0x7c,0x70,0x0a,0x7e]
 
 v_bfrev_b32 v5, -1
-// GFX12: v_bfrev_b32_e32 v5, -1                  ; encoding: [0xc1,0x70,0x0a,0x7e]
+// GFX12: v_bfrev_b32_e32 v5, -1 ; encoding: [0xc1,0x70,0x0a,0x7e]
 
 v_bfrev_b32 v5, 0.5
-// GFX12: v_bfrev_b32_e32 v5, 0.5                 ; encoding: [0xf0,0x70,0x0a,0x7e]
+// GFX12: v_bfrev_b32_e32 v5, 0.5 ; encoding: [0xf0,0x70,0x0a,0x7e]
 
 v_bfrev_b32 v5, src_scc
-// GFX12: v_bfrev_b32_e32 v5, src_scc             ; encoding: [0xfd,0x70,0x0a,0x7e]
+// GFX12: v_bfrev_b32_e32 v5, src_scc ; encoding: [0xfd,0x70,0x0a,0x7e]
 
 v_bfrev_b32 v255, 0xaf123456
-// GFX12: v_bfrev_b32_e32 v255, 0xaf123456        ; encoding: [0xff,0x70,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_bfrev_b32_e32 v255, 0xaf123456 ; encoding: [0xff,0x70,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_ceil_f16 v5, v1
-// GFX12: v_ceil_f16_e32 v5, v1                   ; encoding: [0x01,0xb9,0x0a,0x7e]
+// GFX12: v_ceil_f16_e32 v5, v1 ; encoding: [0x01,0xb9,0x0a,0x7e]
 
 v_ceil_f16 v5, v127
-// GFX12: v_ceil_f16_e32 v5, v127                 ; encoding: [0x7f,0xb9,0x0a,0x7e]
+// GFX12: v_ceil_f16_e32 v5, v127 ; encoding: [0x7f,0xb9,0x0a,0x7e]
 
 v_ceil_f16 v5, s1
-// GFX12: v_ceil_f16_e32 v5, s1                   ; encoding: [0x01,0xb8,0x0a,0x7e]
+// GFX12: v_ceil_f16_e32 v5, s1 ; encoding: [0x01,0xb8,0x0a,0x7e]
 
 v_ceil_f16 v5, s105
-// GFX12: v_ceil_f16_e32 v5, s105                 ; encoding: [0x69,0xb8,0x0a,0x7e]
+// GFX12: v_ceil_f16_e32 v5, s105 ; encoding: [0x69,0xb8,0x0a,0x7e]
 
 v_ceil_f16 v5, vcc_lo
-// GFX12: v_ceil_f16_e32 v5, vcc_lo               ; encoding: [0x6a,0xb8,0x0a,0x7e]
+// GFX12: v_ceil_f16_e32 v5, vcc_lo ; encoding: [0x6a,0xb8,0x0a,0x7e]
 
 v_ceil_f16 v5, vcc_hi
-// GFX12: v_ceil_f16_e32 v5, vcc_hi               ; encoding: [0x6b,0xb8,0x0a,0x7e]
+// GFX12: v_ceil_f16_e32 v5, vcc_hi ; encoding: [0x6b,0xb8,0x0a,0x7e]
 
 v_ceil_f16 v5, ttmp15
-// GFX12: v_ceil_f16_e32 v5, ttmp15               ; encoding: [0x7b,0xb8,0x0a,0x7e]
+// GFX12: v_ceil_f16_e32 v5, ttmp15 ; encoding: [0x7b,0xb8,0x0a,0x7e]
 
 v_ceil_f16 v5, m0
-// GFX12: v_ceil_f16_e32 v5, m0                   ; encoding: [0x7d,0xb8,0x0a,0x7e]
+// GFX12: v_ceil_f16_e32 v5, m0 ; encoding: [0x7d,0xb8,0x0a,0x7e]
 
 v_ceil_f16 v5, exec_lo
-// GFX12: v_ceil_f16_e32 v5, exec_lo              ; encoding: [0x7e,0xb8,0x0a,0x7e]
+// GFX12: v_ceil_f16_e32 v5, exec_lo ; encoding: [0x7e,0xb8,0x0a,0x7e]
 
 v_ceil_f16 v5, exec_hi
-// GFX12: v_ceil_f16_e32 v5, exec_hi              ; encoding: [0x7f,0xb8,0x0a,0x7e]
+// GFX12: v_ceil_f16_e32 v5, exec_hi ; encoding: [0x7f,0xb8,0x0a,0x7e]
 
 v_ceil_f16 v5, null
-// GFX12: v_ceil_f16_e32 v5, null                 ; encoding: [0x7c,0xb8,0x0a,0x7e]
+// GFX12: v_ceil_f16_e32 v5, null ; encoding: [0x7c,0xb8,0x0a,0x7e]
 
 v_ceil_f16 v5, -1
-// GFX12: v_ceil_f16_e32 v5, -1                   ; encoding: [0xc1,0xb8,0x0a,0x7e]
+// GFX12: v_ceil_f16_e32 v5, -1 ; encoding: [0xc1,0xb8,0x0a,0x7e]
 
 v_ceil_f16 v5, 0.5
-// GFX12: v_ceil_f16_e32 v5, 0.5                  ; encoding: [0xf0,0xb8,0x0a,0x7e]
+// GFX12: v_ceil_f16_e32 v5, 0.5 ; encoding: [0xf0,0xb8,0x0a,0x7e]
 
 v_ceil_f16 v5, src_scc
-// GFX12: v_ceil_f16_e32 v5, src_scc              ; encoding: [0xfd,0xb8,0x0a,0x7e]
+// GFX12: v_ceil_f16_e32 v5, src_scc ; encoding: [0xfd,0xb8,0x0a,0x7e]
 
 v_ceil_f16 v127, 0xfe0b
-// GFX12: v_ceil_f16_e32 v127, 0xfe0b             ; encoding: [0xff,0xb8,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_ceil_f16_e32 v127, 0xfe0b ; encoding: [0xff,0xb8,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_ceil_f32 v5, v1
-// GFX12: v_ceil_f32_e32 v5, v1                   ; encoding: [0x01,0x45,0x0a,0x7e]
+// GFX12: v_ceil_f32_e32 v5, v1 ; encoding: [0x01,0x45,0x0a,0x7e]
 
 v_ceil_f32 v5, v255
-// GFX12: v_ceil_f32_e32 v5, v255                 ; encoding: [0xff,0x45,0x0a,0x7e]
+// GFX12: v_ceil_f32_e32 v5, v255 ; encoding: [0xff,0x45,0x0a,0x7e]
 
 v_ceil_f32 v5, s1
-// GFX12: v_ceil_f32_e32 v5, s1                   ; encoding: [0x01,0x44,0x0a,0x7e]
+// GFX12: v_ceil_f32_e32 v5, s1 ; encoding: [0x01,0x44,0x0a,0x7e]
 
 v_ceil_f32 v5, s105
-// GFX12: v_ceil_f32_e32 v5, s105                 ; encoding: [0x69,0x44,0x0a,0x7e]
+// GFX12: v_ceil_f32_e32 v5, s105 ; encoding: [0x69,0x44,0x0a,0x7e]
 
 v_ceil_f32 v5, vcc_lo
-// GFX12: v_ceil_f32_e32 v5, vcc_lo               ; encoding: [0x6a,0x44,0x0a,0x7e]
+// GFX12: v_ceil_f32_e32 v5, vcc_lo ; encoding: [0x6a,0x44,0x0a,0x7e]
 
 v_ceil_f32 v5, vcc_hi
-// GFX12: v_ceil_f32_e32 v5, vcc_hi               ; encoding: [0x6b,0x44,0x0a,0x7e]
+// GFX12: v_ceil_f32_e32 v5, vcc_hi ; encoding: [0x6b,0x44,0x0a,0x7e]
 
 v_ceil_f32 v5, ttmp15
-// GFX12: v_ceil_f32_e32 v5, ttmp15               ; encoding: [0x7b,0x44,0x0a,0x7e]
+// GFX12: v_ceil_f32_e32 v5, ttmp15 ; encoding: [0x7b,0x44,0x0a,0x7e]
 
 v_ceil_f32 v5, m0
-// GFX12: v_ceil_f32_e32 v5, m0                   ; encoding: [0x7d,0x44,0x0a,0x7e]
+// GFX12: v_ceil_f32_e32 v5, m0 ; encoding: [0x7d,0x44,0x0a,0x7e]
 
 v_ceil_f32 v5, exec_lo
-// GFX12: v_ceil_f32_e32 v5, exec_lo              ; encoding: [0x7e,0x44,0x0a,0x7e]
+// GFX12: v_ceil_f32_e32 v5, exec_lo ; encoding: [0x7e,0x44,0x0a,0x7e]
 
 v_ceil_f32 v5, exec_hi
-// GFX12: v_ceil_f32_e32 v5, exec_hi              ; encoding: [0x7f,0x44,0x0a,0x7e]
+// GFX12: v_ceil_f32_e32 v5, exec_hi ; encoding: [0x7f,0x44,0x0a,0x7e]
 
 v_ceil_f32 v5, null
-// GFX12: v_ceil_f32_e32 v5, null                 ; encoding: [0x7c,0x44,0x0a,0x7e]
+// GFX12: v_ceil_f32_e32 v5, null ; encoding: [0x7c,0x44,0x0a,0x7e]
 
 v_ceil_f32 v5, -1
-// GFX12: v_ceil_f32_e32 v5, -1                   ; encoding: [0xc1,0x44,0x0a,0x7e]
+// GFX12: v_ceil_f32_e32 v5, -1 ; encoding: [0xc1,0x44,0x0a,0x7e]
 
 v_ceil_f32 v5, 0.5
-// GFX12: v_ceil_f32_e32 v5, 0.5                  ; encoding: [0xf0,0x44,0x0a,0x7e]
+// GFX12: v_ceil_f32_e32 v5, 0.5 ; encoding: [0xf0,0x44,0x0a,0x7e]
 
 v_ceil_f32 v5, src_scc
-// GFX12: v_ceil_f32_e32 v5, src_scc              ; encoding: [0xfd,0x44,0x0a,0x7e]
+// GFX12: v_ceil_f32_e32 v5, src_scc ; encoding: [0xfd,0x44,0x0a,0x7e]
 
 v_ceil_f32 v255, 0xaf123456
-// GFX12: v_ceil_f32_e32 v255, 0xaf123456         ; encoding: [0xff,0x44,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_ceil_f32_e32 v255, 0xaf123456 ; encoding: [0xff,0x44,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_ceil_f64 v[5:6], v[1:2]
-// GFX12: v_ceil_f64_e32 v[5:6], v[1:2]           ; encoding: [0x01,0x31,0x0a,0x7e]
+// GFX12: v_ceil_f64_e32 v[5:6], v[1:2] ; encoding: [0x01,0x31,0x0a,0x7e]
 
 v_ceil_f64 v[5:6], v[254:255]
-// GFX12: v_ceil_f64_e32 v[5:6], v[254:255]       ; encoding: [0xfe,0x31,0x0a,0x7e]
+// GFX12: v_ceil_f64_e32 v[5:6], v[254:255] ; encoding: [0xfe,0x31,0x0a,0x7e]
 
 v_ceil_f64 v[5:6], s[2:3]
-// GFX12: v_ceil_f64_e32 v[5:6], s[2:3]           ; encoding: [0x02,0x30,0x0a,0x7e]
+// GFX12: v_ceil_f64_e32 v[5:6], s[2:3] ; encoding: [0x02,0x30,0x0a,0x7e]
 
 v_ceil_f64 v[5:6], s[104:105]
-// GFX12: v_ceil_f64_e32 v[5:6], s[104:105]       ; encoding: [0x68,0x30,0x0a,0x7e]
+// GFX12: v_ceil_f64_e32 v[5:6], s[104:105] ; encoding: [0x68,0x30,0x0a,0x7e]
 
 v_ceil_f64 v[5:6], vcc
-// GFX12: v_ceil_f64_e32 v[5:6], vcc              ; encoding: [0x6a,0x30,0x0a,0x7e]
+// GFX12: v_ceil_f64_e32 v[5:6], vcc ; encoding: [0x6a,0x30,0x0a,0x7e]
 
 v_ceil_f64 v[5:6], ttmp[14:15]
-// GFX12: v_ceil_f64_e32 v[5:6], ttmp[14:15]      ; encoding: [0x7a,0x30,0x0a,0x7e]
+// GFX12: v_ceil_f64_e32 v[5:6], ttmp[14:15] ; encoding: [0x7a,0x30,0x0a,0x7e]
 
 v_ceil_f64 v[5:6], exec
-// GFX12: v_ceil_f64_e32 v[5:6], exec             ; encoding: [0x7e,0x30,0x0a,0x7e]
+// GFX12: v_ceil_f64_e32 v[5:6], exec ; encoding: [0x7e,0x30,0x0a,0x7e]
 
 v_ceil_f64 v[5:6], null
-// GFX12: v_ceil_f64_e32 v[5:6], null             ; encoding: [0x7c,0x30,0x0a,0x7e]
+// GFX12: v_ceil_f64_e32 v[5:6], null ; encoding: [0x7c,0x30,0x0a,0x7e]
 
 v_ceil_f64 v[5:6], -1
-// GFX12: v_ceil_f64_e32 v[5:6], -1               ; encoding: [0xc1,0x30,0x0a,0x7e]
+// GFX12: v_ceil_f64_e32 v[5:6], -1 ; encoding: [0xc1,0x30,0x0a,0x7e]
 
 v_ceil_f64 v[5:6], 0.5
-// GFX12: v_ceil_f64_e32 v[5:6], 0.5              ; encoding: [0xf0,0x30,0x0a,0x7e]
+// GFX12: v_ceil_f64_e32 v[5:6], 0.5 ; encoding: [0xf0,0x30,0x0a,0x7e]
 
 v_ceil_f64 v[5:6], src_scc
-// GFX12: v_ceil_f64_e32 v[5:6], src_scc          ; encoding: [0xfd,0x30,0x0a,0x7e]
+// GFX12: v_ceil_f64_e32 v[5:6], src_scc ; encoding: [0xfd,0x30,0x0a,0x7e]
 
 v_ceil_f64 v[254:255], 0xaf123456
-// GFX12: v_ceil_f64_e32 v[254:255], 0xaf123456   ; encoding: [0xff,0x30,0xfc,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_ceil_f64_e32 v[254:255], 0xaf123456 ; encoding: [0xff,0x30,0xfc,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cls_i32 v5, v1
-// GFX12: v_cls_i32_e32 v5, v1                    ; encoding: [0x01,0x77,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, v1 ; encoding: [0x01,0x77,0x0a,0x7e]
 
 v_cls_i32 v5, v255
-// GFX12: v_cls_i32_e32 v5, v255                  ; encoding: [0xff,0x77,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, v255 ; encoding: [0xff,0x77,0x0a,0x7e]
 
 v_cls_i32 v5, s1
-// GFX12: v_cls_i32_e32 v5, s1                    ; encoding: [0x01,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, s1 ; encoding: [0x01,0x76,0x0a,0x7e]
 
 v_cls_i32 v5, s105
-// GFX12: v_cls_i32_e32 v5, s105                  ; encoding: [0x69,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, s105 ; encoding: [0x69,0x76,0x0a,0x7e]
 
 v_cls_i32 v5, vcc_lo
-// GFX12: v_cls_i32_e32 v5, vcc_lo                ; encoding: [0x6a,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, vcc_lo ; encoding: [0x6a,0x76,0x0a,0x7e]
 
 v_cls_i32 v5, vcc_hi
-// GFX12: v_cls_i32_e32 v5, vcc_hi                ; encoding: [0x6b,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, vcc_hi ; encoding: [0x6b,0x76,0x0a,0x7e]
 
 v_cls_i32 v5, ttmp15
-// GFX12: v_cls_i32_e32 v5, ttmp15                ; encoding: [0x7b,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, ttmp15 ; encoding: [0x7b,0x76,0x0a,0x7e]
 
 v_cls_i32 v5, m0
-// GFX12: v_cls_i32_e32 v5, m0                    ; encoding: [0x7d,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, m0 ; encoding: [0x7d,0x76,0x0a,0x7e]
 
 v_cls_i32 v5, exec_lo
-// GFX12: v_cls_i32_e32 v5, exec_lo               ; encoding: [0x7e,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, exec_lo ; encoding: [0x7e,0x76,0x0a,0x7e]
 
 v_cls_i32 v5, exec_hi
-// GFX12: v_cls_i32_e32 v5, exec_hi               ; encoding: [0x7f,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, exec_hi ; encoding: [0x7f,0x76,0x0a,0x7e]
 
 v_cls_i32 v5, null
-// GFX12: v_cls_i32_e32 v5, null                  ; encoding: [0x7c,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, null ; encoding: [0x7c,0x76,0x0a,0x7e]
 
 v_cls_i32 v5, -1
-// GFX12: v_cls_i32_e32 v5, -1                    ; encoding: [0xc1,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, -1 ; encoding: [0xc1,0x76,0x0a,0x7e]
 
 v_cls_i32 v5, 0.5
-// GFX12: v_cls_i32_e32 v5, 0.5                   ; encoding: [0xf0,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, 0.5 ; encoding: [0xf0,0x76,0x0a,0x7e]
 
 v_cls_i32 v5, src_scc
-// GFX12: v_cls_i32_e32 v5, src_scc               ; encoding: [0xfd,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, src_scc ; encoding: [0xfd,0x76,0x0a,0x7e]
 
 v_cls_i32 v255, 0xaf123456
-// GFX12: v_cls_i32_e32 v255, 0xaf123456          ; encoding: [0xff,0x76,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_cls_i32_e32 v255, 0xaf123456 ; encoding: [0xff,0x76,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_clz_i32_u32 v5, v1
-// GFX12: v_clz_i32_u32_e32 v5, v1                ; encoding: [0x01,0x73,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, v1 ; encoding: [0x01,0x73,0x0a,0x7e]
 
 v_clz_i32_u32 v5, v255
-// GFX12: v_clz_i32_u32_e32 v5, v255              ; encoding: [0xff,0x73,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, v255 ; encoding: [0xff,0x73,0x0a,0x7e]
 
 v_clz_i32_u32 v5, s1
-// GFX12: v_clz_i32_u32_e32 v5, s1                ; encoding: [0x01,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, s1 ; encoding: [0x01,0x72,0x0a,0x7e]
 
 v_clz_i32_u32 v5, s105
-// GFX12: v_clz_i32_u32_e32 v5, s105              ; encoding: [0x69,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, s105 ; encoding: [0x69,0x72,0x0a,0x7e]
 
 v_clz_i32_u32 v5, vcc_lo
-// GFX12: v_clz_i32_u32_e32 v5, vcc_lo            ; encoding: [0x6a,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, vcc_lo ; encoding: [0x6a,0x72,0x0a,0x7e]
 
 v_clz_i32_u32 v5, vcc_hi
-// GFX12: v_clz_i32_u32_e32 v5, vcc_hi            ; encoding: [0x6b,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, vcc_hi ; encoding: [0x6b,0x72,0x0a,0x7e]
 
 v_clz_i32_u32 v5, ttmp15
-// GFX12: v_clz_i32_u32_e32 v5, ttmp15            ; encoding: [0x7b,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, ttmp15 ; encoding: [0x7b,0x72,0x0a,0x7e]
 
 v_clz_i32_u32 v5, m0
-// GFX12: v_clz_i32_u32_e32 v5, m0                ; encoding: [0x7d,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, m0 ; encoding: [0x7d,0x72,0x0a,0x7e]
 
 v_clz_i32_u32 v5, exec_lo
-// GFX12: v_clz_i32_u32_e32 v5, exec_lo           ; encoding: [0x7e,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, exec_lo ; encoding: [0x7e,0x72,0x0a,0x7e]
 
 v_clz_i32_u32 v5, exec_hi
-// GFX12: v_clz_i32_u32_e32 v5, exec_hi           ; encoding: [0x7f,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, exec_hi ; encoding: [0x7f,0x72,0x0a,0x7e]
 
 v_clz_i32_u32 v5, null
-// GFX12: v_clz_i32_u32_e32 v5, null              ; encoding: [0x7c,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, null ; encoding: [0x7c,0x72,0x0a,0x7e]
 
 v_clz_i32_u32 v5, -1
-// GFX12: v_clz_i32_u32_e32 v5, -1                ; encoding: [0xc1,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, -1 ; encoding: [0xc1,0x72,0x0a,0x7e]
 
 v_clz_i32_u32 v5, 0.5
-// GFX12: v_clz_i32_u32_e32 v5, 0.5               ; encoding: [0xf0,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, 0.5 ; encoding: [0xf0,0x72,0x0a,0x7e]
 
 v_clz_i32_u32 v5, src_scc
-// GFX12: v_clz_i32_u32_e32 v5, src_scc           ; encoding: [0xfd,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, src_scc ; encoding: [0xfd,0x72,0x0a,0x7e]
 
 v_clz_i32_u32 v255, 0xaf123456
-// GFX12: v_clz_i32_u32_e32 v255, 0xaf123456      ; encoding: [0xff,0x72,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_clz_i32_u32_e32 v255, 0xaf123456 ; encoding: [0xff,0x72,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cos_f16 v5, v1
-// GFX12: v_cos_f16_e32 v5, v1                    ; encoding: [0x01,0xc3,0x0a,0x7e]
+// GFX12: v_cos_f16_e32 v5, v1 ; encoding: [0x01,0xc3,0x0a,0x7e]
 
 v_cos_f16 v5, v127
-// GFX12: v_cos_f16_e32 v5, v127                  ; encoding: [0x7f,0xc3,0x0a,0x7e]
+// GFX12: v_cos_f16_e32 v5, v127 ; encoding: [0x7f,0xc3,0x0a,0x7e]
 
 v_cos_f16 v5, s1
-// GFX12: v_cos_f16_e32 v5, s1                    ; encoding: [0x01,0xc2,0x0a,0x7e]
+// GFX12: v_cos_f16_e32 v5, s1 ; encoding: [0x01,0xc2,0x0a,0x7e]
 
 v_cos_f16 v5, s105
-// GFX12: v_cos_f16_e32 v5, s105                  ; encoding: [0x69,0xc2,0x0a,0x7e]
+// GFX12: v_cos_f16_e32 v5, s105 ; encoding: [0x69,0xc2,0x0a,0x7e]
 
 v_cos_f16 v5, vcc_lo
-// GFX12: v_cos_f16_e32 v5, vcc_lo                ; encoding: [0x6a,0xc2,0x0a,0x7e]
+// GFX12: v_cos_f16_e32 v5, vcc_lo ; encoding: [0x6a,0xc2,0x0a,0x7e]
 
 v_cos_f16 v5, vcc_hi
-// GFX12: v_cos_f16_e32 v5, vcc_hi                ; encoding: [0x6b,0xc2,0x0a,0x7e]
+// GFX12: v_cos_f16_e32 v5, vcc_hi ; encoding: [0x6b,0xc2,0x0a,0x7e]
 
 v_cos_f16 v5, ttmp15
-// GFX12: v_cos_f16_e32 v5, ttmp15                ; encoding: [0x7b,0xc2,0x0a,0x7e]
+// GFX12: v_cos_f16_e32 v5, ttmp15 ; encoding: [0x7b,0xc2,0x0a,0x7e]
 
 v_cos_f16 v5, m0
-// GFX12: v_cos_f16_e32 v5, m0                    ; encoding: [0x7d,0xc2,0x0a,0x7e]
+// GFX12: v_cos_f16_e32 v5, m0 ; encoding: [0x7d,0xc2,0x0a,0x7e]
 
 v_cos_f16 v5, exec_lo
-// GFX12: v_cos_f16_e32 v5, exec_lo               ; encoding: [0x7e,0xc2,0x0a,0x7e]
+// GFX12: v_cos_f16_e32 v5, exec_lo ; encoding: [0x7e,0xc2,0x0a,0x7e]
 
 v_cos_f16 v5, exec_hi
-// GFX12: v_cos_f16_e32 v5, exec_hi               ; encoding: [0x7f,0xc2,0x0a,0x7e]
+// GFX12: v_cos_f16_e32 v5, exec_hi ; encoding: [0x7f,0xc2,0x0a,0x7e]
 
 v_cos_f16 v5, null
-// GFX12: v_cos_f16_e32 v5, null                  ; encoding: [0x7c,0xc2,0x0a,0x7e]
+// GFX12: v_cos_f16_e32 v5, null ; encoding: [0x7c,0xc2,0x0a,0x7e]
 
 v_cos_f16 v5, -1
-// GFX12: v_cos_f16_e32 v5, -1                    ; encoding: [0xc1,0xc2,0x0a,0x7e]
+// GFX12: v_cos_f16_e32 v5, -1 ; encoding: [0xc1,0xc2,0x0a,0x7e]
 
 v_cos_f16 v5, 0.5
-// GFX12: v_cos_f16_e32 v5, 0.5                   ; encoding: [0xf0,0xc2,0x0a,0x7e]
+// GFX12: v_cos_f16_e32 v5, 0.5 ; encoding: [0xf0,0xc2,0x0a,0x7e]
 
 v_cos_f16 v5, src_scc
-// GFX12: v_cos_f16_e32 v5, src_scc               ; encoding: [0xfd,0xc2,0x0a,0x7e]
+// GFX12: v_cos_f16_e32 v5, src_scc ; encoding: [0xfd,0xc2,0x0a,0x7e]
 
 v_cos_f16 v127, 0xfe0b
-// GFX12: v_cos_f16_e32 v127, 0xfe0b              ; encoding: [0xff,0xc2,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_cos_f16_e32 v127, 0xfe0b ; encoding: [0xff,0xc2,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_cos_f32 v5, v1
-// GFX12: v_cos_f32_e32 v5, v1                    ; encoding: [0x01,0x6d,0x0a,0x7e]
+// GFX12: v_cos_f32_e32 v5, v1 ; encoding: [0x01,0x6d,0x0a,0x7e]
 
 v_cos_f32 v5, v255
-// GFX12: v_cos_f32_e32 v5, v255                  ; encoding: [0xff,0x6d,0x0a,0x7e]
+// GFX12: v_cos_f32_e32 v5, v255 ; encoding: [0xff,0x6d,0x0a,0x7e]
 
 v_cos_f32 v5, s1
-// GFX12: v_cos_f32_e32 v5, s1                    ; encoding: [0x01,0x6c,0x0a,0x7e]
+// GFX12: v_cos_f32_e32 v5, s1 ; encoding: [0x01,0x6c,0x0a,0x7e]
 
 v_cos_f32 v5, s105
-// GFX12: v_cos_f32_e32 v5, s105                  ; encoding: [0x69,0x6c,0x0a,0x7e]
+// GFX12: v_cos_f32_e32 v5, s105 ; encoding: [0x69,0x6c,0x0a,0x7e]
 
 v_cos_f32 v5, vcc_lo
-// GFX12: v_cos_f32_e32 v5, vcc_lo                ; encoding: [0x6a,0x6c,0x0a,0x7e]
+// GFX12: v_cos_f32_e32 v5, vcc_lo ; encoding: [0x6a,0x6c,0x0a,0x7e]
 
 v_cos_f32 v5, vcc_hi
-// GFX12: v_cos_f32_e32 v5, vcc_hi                ; encoding: [0x6b,0x6c,0x0a,0x7e]
+// GFX12: v_cos_f32_e32 v5, vcc_hi ; encoding: [0x6b,0x6c,0x0a,0x7e]
 
 v_cos_f32 v5, ttmp15
-// GFX12: v_cos_f32_e32 v5, ttmp15                ; encoding: [0x7b,0x6c,0x0a,0x7e]
+// GFX12: v_cos_f32_e32 v5, ttmp15 ; encoding: [0x7b,0x6c,0x0a,0x7e]
 
 v_cos_f32 v5, m0
-// GFX12: v_cos_f32_e32 v5, m0                    ; encoding: [0x7d,0x6c,0x0a,0x7e]
+// GFX12: v_cos_f32_e32 v5, m0 ; encoding: [0x7d,0x6c,0x0a,0x7e]
 
 v_cos_f32 v5, exec_lo
-// GFX12: v_cos_f32_e32 v5, exec_lo               ; encoding: [0x7e,0x6c,0x0a,0x7e]
+// GFX12: v_cos_f32_e32 v5, exec_lo ; encoding: [0x7e,0x6c,0x0a,0x7e]
 
 v_cos_f32 v5, exec_hi
-// GFX12: v_cos_f32_e32 v5, exec_hi               ; encoding: [0x7f,0x6c,0x0a,0x7e]
+// GFX12: v_cos_f32_e32 v5, exec_hi ; encoding: [0x7f,0x6c,0x0a,0x7e]
 
 v_cos_f32 v5, null
-// GFX12: v_cos_f32_e32 v5, null                  ; encoding: [0x7c,0x6c,0x0a,0x7e]
+// GFX12: v_cos_f32_e32 v5, null ; encoding: [0x7c,0x6c,0x0a,0x7e]
 
 v_cos_f32 v5, -1
-// GFX12: v_cos_f32_e32 v5, -1                    ; encoding: [0xc1,0x6c,0x0a,0x7e]
+// GFX12: v_cos_f32_e32 v5, -1 ; encoding: [0xc1,0x6c,0x0a,0x7e]
 
 v_cos_f32 v5, 0.5
-// GFX12: v_cos_f32_e32 v5, 0.5                   ; encoding: [0xf0,0x6c,0x0a,0x7e]
+// GFX12: v_cos_f32_e32 v5, 0.5 ; encoding: [0xf0,0x6c,0x0a,0x7e]
 
 v_cos_f32 v5, src_scc
-// GFX12: v_cos_f32_e32 v5, src_scc               ; encoding: [0xfd,0x6c,0x0a,0x7e]
+// GFX12: v_cos_f32_e32 v5, src_scc ; encoding: [0xfd,0x6c,0x0a,0x7e]
 
 v_cos_f32 v255, 0xaf123456
-// GFX12: v_cos_f32_e32 v255, 0xaf123456          ; encoding: [0xff,0x6c,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_cos_f32_e32 v255, 0xaf123456 ; encoding: [0xff,0x6c,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_ctz_i32_b32 v5, v1
-// GFX12: v_ctz_i32_b32_e32 v5, v1                ; encoding: [0x01,0x75,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, v1 ; encoding: [0x01,0x75,0x0a,0x7e]
 
 v_ctz_i32_b32 v5, v255
-// GFX12: v_ctz_i32_b32_e32 v5, v255              ; encoding: [0xff,0x75,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, v255 ; encoding: [0xff,0x75,0x0a,0x7e]
 
 v_ctz_i32_b32 v5, s1
-// GFX12: v_ctz_i32_b32_e32 v5, s1                ; encoding: [0x01,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, s1 ; encoding: [0x01,0x74,0x0a,0x7e]
 
 v_ctz_i32_b32 v5, s105
-// GFX12: v_ctz_i32_b32_e32 v5, s105              ; encoding: [0x69,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, s105 ; encoding: [0x69,0x74,0x0a,0x7e]
 
 v_ctz_i32_b32 v5, vcc_lo
-// GFX12: v_ctz_i32_b32_e32 v5, vcc_lo            ; encoding: [0x6a,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, vcc_lo ; encoding: [0x6a,0x74,0x0a,0x7e]
 
 v_ctz_i32_b32 v5, vcc_hi
-// GFX12: v_ctz_i32_b32_e32 v5, vcc_hi            ; encoding: [0x6b,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, vcc_hi ; encoding: [0x6b,0x74,0x0a,0x7e]
 
 v_ctz_i32_b32 v5, ttmp15
-// GFX12: v_ctz_i32_b32_e32 v5, ttmp15            ; encoding: [0x7b,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, ttmp15 ; encoding: [0x7b,0x74,0x0a,0x7e]
 
 v_ctz_i32_b32 v5, m0
-// GFX12: v_ctz_i32_b32_e32 v5, m0                ; encoding: [0x7d,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, m0 ; encoding: [0x7d,0x74,0x0a,0x7e]
 
 v_ctz_i32_b32 v5, exec_lo
-// GFX12: v_ctz_i32_b32_e32 v5, exec_lo           ; encoding: [0x7e,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, exec_lo ; encoding: [0x7e,0x74,0x0a,0x7e]
 
 v_ctz_i32_b32 v5, exec_hi
-// GFX12: v_ctz_i32_b32_e32 v5, exec_hi           ; encoding: [0x7f,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, exec_hi ; encoding: [0x7f,0x74,0x0a,0x7e]
 
 v_ctz_i32_b32 v5, null
-// GFX12: v_ctz_i32_b32_e32 v5, null              ; encoding: [0x7c,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, null ; encoding: [0x7c,0x74,0x0a,0x7e]
 
 v_ctz_i32_b32 v5, -1
-// GFX12: v_ctz_i32_b32_e32 v5, -1                ; encoding: [0xc1,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, -1 ; encoding: [0xc1,0x74,0x0a,0x7e]
 
 v_ctz_i32_b32 v5, 0.5
-// GFX12: v_ctz_i32_b32_e32 v5, 0.5               ; encoding: [0xf0,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, 0.5 ; encoding: [0xf0,0x74,0x0a,0x7e]
 
 v_ctz_i32_b32 v5, src_scc
-// GFX12: v_ctz_i32_b32_e32 v5, src_scc           ; encoding: [0xfd,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, src_scc ; encoding: [0xfd,0x74,0x0a,0x7e]
 
 v_ctz_i32_b32 v255, 0xaf123456
-// GFX12: v_ctz_i32_b32_e32 v255, 0xaf123456      ; encoding: [0xff,0x74,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_ctz_i32_b32_e32 v255, 0xaf123456 ; encoding: [0xff,0x74,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cvt_f32_bf8_e32 v1, s3
-// GFX12: v_cvt_f32_bf8_e32 v1, s3                ; encoding: [0x03,0xda,0x02,0x7e]
+// GFX12: v_cvt_f32_bf8_e32 v1, s3 ; encoding: [0x03,0xda,0x02,0x7e]
 
 v_cvt_f32_bf8_e32 v1, 3
-// GFX12: v_cvt_f32_bf8_e32 v1, 3                 ; encoding: [0x83,0xda,0x02,0x7e]
+// GFX12: v_cvt_f32_bf8_e32 v1, 3 ; encoding: [0x83,0xda,0x02,0x7e]
 
 v_cvt_f32_bf8_e32 v1, v3
-// GFX12: v_cvt_f32_bf8_e32 v1, v3                ; encoding: [0x03,0xdb,0x02,0x7e]
+// GFX12: v_cvt_f32_bf8_e32 v1, v3 ; encoding: [0x03,0xdb,0x02,0x7e]
 
 v_cvt_f32_fp8_e32 v1, s3
-// GFX12: v_cvt_f32_fp8_e32 v1, s3                ; encoding: [0x03,0xd8,0x02,0x7e]
+// GFX12: v_cvt_f32_fp8_e32 v1, s3 ; encoding: [0x03,0xd8,0x02,0x7e]
 
 v_cvt_f32_fp8_e32 v1, 3
-// GFX12: v_cvt_f32_fp8_e32 v1, 3                 ; encoding: [0x83,0xd8,0x02,0x7e]
+// GFX12: v_cvt_f32_fp8_e32 v1, 3 ; encoding: [0x83,0xd8,0x02,0x7e]
 
 v_cvt_f32_fp8_e32 v1, v3
-// GFX12: v_cvt_f32_fp8_e32 v1, v3                ; encoding: [0x03,0xd9,0x02,0x7e]
+// GFX12: v_cvt_f32_fp8_e32 v1, v3 ; encoding: [0x03,0xd9,0x02,0x7e]
 
 v_cvt_pk_f32_bf8_e32 v[2:3], s3
-// GFX12: v_cvt_pk_f32_bf8_e32 v[2:3], s3         ; encoding: [0x03,0xde,0x04,0x7e]
+// GFX12: v_cvt_pk_f32_bf8_e32 v[2:3], s3 ; encoding: [0x03,0xde,0x04,0x7e]
 
 v_cvt_pk_f32_bf8_e32 v[3:4], s5
-// GFX12: v_cvt_pk_f32_bf8_e32 v[3:4], s5         ; encoding: [0x05,0xde,0x06,0x7e]
+// GFX12: v_cvt_pk_f32_bf8_e32 v[3:4], s5 ; encoding: [0x05,0xde,0x06,0x7e]
 
 v_cvt_pk_f32_bf8_e32 v[2:3], 3
-// GFX12: v_cvt_pk_f32_bf8_e32 v[2:3], 3          ; encoding: [0x83,0xde,0x04,0x7e]
+// GFX12: v_cvt_pk_f32_bf8_e32 v[2:3], 3 ; encoding: [0x83,0xde,0x04,0x7e]
 
 v_cvt_pk_f32_bf8_e32 v[3:4], 3
-// GFX12: v_cvt_pk_f32_bf8_e32 v[3:4], 3          ; encoding: [0x83,0xde,0x06,0x7e]
+// GFX12: v_cvt_pk_f32_bf8_e32 v[3:4], 3 ; encoding: [0x83,0xde,0x06,0x7e]
 
 v_cvt_pk_f32_bf8_e32 v[2:3], v3
-// GFX12: v_cvt_pk_f32_bf8_e32 v[2:3], v3         ; encoding: [0x03,0xdf,0x04,0x7e]
+// GFX12: v_cvt_pk_f32_bf8_e32 v[2:3], v3 ; encoding: [0x03,0xdf,0x04,0x7e]
 
 v_cvt_pk_f32_bf8_e32 v[3:4], v3
-// GFX12: v_cvt_pk_f32_bf8_e32 v[3:4], v3         ; encoding: [0x03,0xdf,0x06,0x7e]
+// GFX12: v_cvt_pk_f32_bf8_e32 v[3:4], v3 ; encoding: [0x03,0xdf,0x06,0x7e]
 
 v_cvt_pk_f32_fp8_e32 v[2:3], s3
-// GFX12: v_cvt_pk_f32_fp8_e32 v[2:3], s3         ; encoding: [0x03,0xdc,0x04,0x7e]
+// GFX12: v_cvt_pk_f32_fp8_e32 v[2:3], s3 ; encoding: [0x03,0xdc,0x04,0x7e]
 
 v_cvt_pk_f32_fp8_e32 v[2:3], 3
-// GFX12: v_cvt_pk_f32_fp8_e32 v[2:3], 3          ; encoding: [0x83,0xdc,0x04,0x7e]
+// GFX12: v_cvt_pk_f32_fp8_e32 v[2:3], 3 ; encoding: [0x83,0xdc,0x04,0x7e]
 
 v_cvt_pk_f32_fp8_e32 v[2:3], v3
-// GFX12: v_cvt_pk_f32_fp8_e32 v[2:3], v3         ; encoding: [0x03,0xdd,0x04,0x7e]
+// GFX12: v_cvt_pk_f32_fp8_e32 v[2:3], v3 ; encoding: [0x03,0xdd,0x04,0x7e]
 
 v_cvt_f16_f32 v5, v1
-// GFX12: v_cvt_f16_f32_e32 v5, v1                ; encoding: [0x01,0x15,0x0a,0x7e]
+// GFX12: v_cvt_f16_f32_e32 v5, v1 ; encoding: [0x01,0x15,0x0a,0x7e]
 
 v_cvt_f16_f32 v5, v255
-// GFX12: v_cvt_f16_f32_e32 v5, v255              ; encoding: [0xff,0x15,0x0a,0x7e]
+// GFX12: v_cvt_f16_f32_e32 v5, v255 ; encoding: [0xff,0x15,0x0a,0x7e]
 
 v_cvt_f16_f32 v5, s1
-// GFX12: v_cvt_f16_f32_e32 v5, s1                ; encoding: [0x01,0x14,0x0a,0x7e]
+// GFX12: v_cvt_f16_f32_e32 v5, s1 ; encoding: [0x01,0x14,0x0a,0x7e]
 
 v_cvt_f16_f32 v5, s105
-// GFX12: v_cvt_f16_f32_e32 v5, s105              ; encoding: [0x69,0x14,0x0a,0x7e]
+// GFX12: v_cvt_f16_f32_e32 v5, s105 ; encoding: [0x69,0x14,0x0a,0x7e]
 
 v_cvt_f16_f32 v5, vcc_lo
-// GFX12: v_cvt_f16_f32_e32 v5, vcc_lo            ; encoding: [0x6a,0x14,0x0a,0x7e]
+// GFX12: v_cvt_f16_f32_e32 v5, vcc_lo ; encoding: [0x6a,0x14,0x0a,0x7e]
 
 v_cvt_f16_f32 v5, vcc_hi
-// GFX12: v_cvt_f16_f32_e32 v5, vcc_hi            ; encoding: [0x6b,0x14,0x0a,0x7e]
+// GFX12: v_cvt_f16_f32_e32 v5, vcc_hi ; encoding: [0x6b,0x14,0x0a,0x7e]
 
 v_cvt_f16_f32 v5, ttmp15
-// GFX12: v_cvt_f16_f32_e32 v5, ttmp15            ; encoding: [0x7b,0x14,0x0a,0x7e]
+// GFX12: v_cvt_f16_f32_e32 v5, ttmp15 ; encoding: [0x7b,0x14,0x0a,0x7e]
 
 v_cvt_f16_f32 v5, m0
-// GFX12: v_cvt_f16_f32_e32 v5, m0                ; encoding: [0x7d,0x14,0x0a,0x7e]
+// GFX12: v_cvt_f16_f32_e32 v5, m0 ; encoding: [0x7d,0x14,0x0a,0x7e]
 
 v_cvt_f16_f32 v5, exec_lo
-// GFX12: v_cvt_f16_f32_e32 v5, exec_lo           ; encoding: [0x7e,0x14,0x0a,0x7e]
+// GFX12: v_cvt_f16_f32_e32 v5, exec_lo ; encoding: [0x7e,0x14,0x0a,0x7e]
 
 v_cvt_f16_f32 v5, exec_hi
-// GFX12: v_cvt_f16_f32_e32 v5, exec_hi           ; encoding: [0x7f,0x14,0x0a,0x7e]
+// GFX12: v_cvt_f16_f32_e32 v5, exec_hi ; encoding: [0x7f,0x14,0x0a,0x7e]
 
 v_cvt_f16_f32 v5, null
-// GFX12: v_cvt_f16_f32_e32 v5, null              ; encoding: [0x7c,0x14,0x0a,0x7e]
+// GFX12: v_cvt_f16_f32_e32 v5, null ; encoding: [0x7c,0x14,0x0a,0x7e]
 
 v_cvt_f16_f32 v5, -1
-// GFX12: v_cvt_f16_f32_e32 v5, -1                ; encoding: [0xc1,0x14,0x0a,0x7e]
+// GFX12: v_cvt_f16_f32_e32 v5, -1 ; encoding: [0xc1,0x14,0x0a,0x7e]
 
 v_cvt_f16_f32 v5, 0.5
-// GFX12: v_cvt_f16_f32_e32 v5, 0.5               ; encoding: [0xf0,0x14,0x0a,0x7e]
+// GFX12: v_cvt_f16_f32_e32 v5, 0.5 ; encoding: [0xf0,0x14,0x0a,0x7e]
 
 v_cvt_f16_f32 v5, src_scc
-// GFX12: v_cvt_f16_f32_e32 v5, src_scc           ; encoding: [0xfd,0x14,0x0a,0x7e]
+// GFX12: v_cvt_f16_f32_e32 v5, src_scc ; encoding: [0xfd,0x14,0x0a,0x7e]
 
 v_cvt_f16_f32 v127, 0xaf123456
-// GFX12: v_cvt_f16_f32_e32 v127, 0xaf123456      ; encoding: [0xff,0x14,0xfe,0x7e,0x56,0x34,0x12,0xaf]
+// GFX12: v_cvt_f16_f32_e32 v127, 0xaf123456 ; encoding: [0xff,0x14,0xfe,0x7e,0x56,0x34,0x12,0xaf]
 
 v_cvt_f16_i16 v5, v1
-// GFX12: v_cvt_f16_i16_e32 v5, v1                ; encoding: [0x01,0xa3,0x0a,0x7e]
+// GFX12: v_cvt_f16_i16_e32 v5, v1 ; encoding: [0x01,0xa3,0x0a,0x7e]
 
 v_cvt_f16_i16 v5, v127
-// GFX12: v_cvt_f16_i16_e32 v5, v127              ; encoding: [0x7f,0xa3,0x0a,0x7e]
+// GFX12: v_cvt_f16_i16_e32 v5, v127 ; encoding: [0x7f,0xa3,0x0a,0x7e]
 
 v_cvt_f16_i16 v5, s1
-// GFX12: v_cvt_f16_i16_e32 v5, s1                ; encoding: [0x01,0xa2,0x0a,0x7e]
+// GFX12: v_cvt_f16_i16_e32 v5, s1 ; encoding: [0x01,0xa2,0x0a,0x7e]
 
 v_cvt_f16_i16 v5, s105
-// GFX12: v_cvt_f16_i16_e32 v5, s105              ; encoding: [0x69,0xa2,0x0a,0x7e]
+// GFX12: v_cvt_f16_i16_e32 v5, s105 ; encoding: [0x69,0xa2,0x0a,0x7e]
 
 v_cvt_f16_i16 v5, vcc_lo
-// GFX12: v_cvt_f16_i16_e32 v5, vcc_lo            ; encoding: [0x6a,0xa2,0x0a,0x7e]
+// GFX12: v_cvt_f16_i16_e32 v5, vcc_lo ; encoding: [0x6a,0xa2,0x0a,0x7e]
 
 v_cvt_f16_i16 v5, vcc_hi
-// GFX12: v_cvt_f16_i16_e32 v5, vcc_hi            ; encoding: [0x6b,0xa2,0x0a,0x7e]
+// GFX12: v_cvt_f16_i16_e32 v5, vcc_hi ; encoding: [0x6b,0xa2,0x0a,0x7e]
 
 v_cvt_f16_i16 v5, ttmp15
-// GFX12: v_cvt_f16_i16_e32 v5, ttmp15            ; encoding: [0x7b,0xa2,0x0a,0x7e]
+// GFX12: v_cvt_f16_i16_e32 v5, ttmp15 ; encoding: [0x7b,0xa2,0x0a,0x7e]
 
 v_cvt_f16_i16 v5, m0
-// GFX12: v_cvt_f16_i16_e32 v5, m0                ; encoding: [0x7d,0xa2,0x0a,0x7e]
+// GFX12: v_cvt_f16_i16_e32 v5, m0 ; encoding: [0x7d,0xa2,0x0a,0x7e]
 
 v_cvt_f16_i16 v5, exec_lo
-// GFX12: v_cvt_f16_i16_e32 v5, exec_lo           ; encoding: [0x7e,0xa2,0x0a,0x7e]
+// GFX12: v_cvt_f16_i16_e32 v5, exec_lo ; encoding: [0x7e,0xa2,0x0a,0x7e]
 
 v_cvt_f16_i16 v5, exec_hi
-// GFX12: v_cvt_f16_i16_e32 v5, exec_hi           ; encoding: [0x7f,0xa2,0x0a,0x7e]
+// GFX12: v_cvt_f16_i16_e32 v5, exec_hi ; encoding: [0x7f,0xa2,0x0a,0x7e]
 
 v_cvt_f16_i16 v5, null
-// GFX12: v_cvt_f16_i16_e32 v5, null              ; encoding: [0x7c,0xa2,0x0a,0x7e]
+// GFX12: v_cvt_f16_i16_e32 v5, null ; encoding: [0x7c,0xa2,0x0a,0x7e]
 
 v_cvt_f16_i16 v5, -1
-// GFX12: v_cvt_f16_i16_e32 v5, -1                ; encoding: [0xc1,0xa2,0x0a,0x7e]
+// GFX12: v_cvt_f16_i16_e32 v5, -1 ; encoding: [0xc1,0xa2,0x0a,0x7e]
 
 v_cvt_f16_i16 v5, 0.5
-// GFX12-ASM: v_cvt_f16_i16_e32 v5, 0.5               ; encoding: [0xf0,0xa2,0x0a,0x7e]
-// GFX12-DIS: v_cvt_f16_i16_e32 v5, 0x3800            ; encoding: [0xff,0xa2,0x0a,0x7e,0x00,0x38,0x00,0x00]
+// GFX12-ASM: v_cvt_f16_i16_e32 v5, 0.5 ; encoding: [0xf0,0xa2,0x0a,0x7e]
+// GFX12-DIS: v_cvt_f16_i16_e32 v5, 0x3800 ; encoding: [0xff,0xa2,0x0a,0x7e,0x00,0x38,0x00,0x00]
 
 v_cvt_f16_i16 v5, src_scc
-// GFX12: v_cvt_f16_i16_e32 v5, src_scc           ; encoding: [0xfd,0xa2,0x0a,0x7e]
+// GFX12: v_cvt_f16_i16_e32 v5, src_scc ; encoding: [0xfd,0xa2,0x0a,0x7e]
 
 v_cvt_f16_i16 v127, 0xfe0b
-// GFX12: v_cvt_f16_i16_e32 v127, 0xfe0b          ; encoding: [0xff,0xa2,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_cvt_f16_i16_e32 v127, 0xfe0b ; encoding: [0xff,0xa2,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_cvt_f16_u16 v5, v1
-// GFX12: v_cvt_f16_u16_e32 v5, v1                ; encoding: [0x01,0xa1,0x0a,0x7e]
+// GFX12: v_cvt_f16_u16_e32 v5, v1 ; encoding: [0x01,0xa1,0x0a,0x7e]
 
 v_cvt_f16_u16 v5, v127
-// GFX12: v_cvt_f16_u16_e32 v5, v127              ; encoding: [0x7f,0xa1,0x0a,0x7e]
+// GFX12: v_cvt_f16_u16_e32 v5, v127 ; encoding: [0x7f,0xa1,0x0a,0x7e]
 
 v_cvt_f16_u16 v5, s1
-// GFX12: v_cvt_f16_u16_e32 v5, s1                ; encoding: [0x01,0xa0,0x0a,0x7e]
+// GFX12: v_cvt_f16_u16_e32 v5, s1 ; encoding: [0x01,0xa0,0x0a,0x7e]
 
 v_cvt_f16_u16 v5, s105
-// GFX12: v_cvt_f16_u16_e32 v5, s105              ; encoding: [0x69,0xa0,0x0a,0x7e]
+// GFX12: v_cvt_f16_u16_e32 v5, s105 ; encoding: [0x69,0xa0,0x0a,0x7e]
 
 v_cvt_f16_u16 v5, vcc_lo
-// GFX12: v_cvt_f16_u16_e32 v5, vcc_lo            ; encoding: [0x6a,0xa0,0x0a,0x7e]
+// GFX12: v_cvt_f16_u16_e32 v5, vcc_lo ; encoding: [0x6a,0xa0,0x0a,0x7e]
 
 v_cvt_f16_u16 v5, vcc_hi
-// GFX12: v_cvt_f16_u16_e32 v5, vcc_hi            ; encoding: [0x6b,0xa0,0x0a,0x7e]
+// GFX12: v_cvt_f16_u16_e32 v5, vcc_hi ; encoding: [0x6b,0xa0,0x0a,0x7e]
 
 v_cvt_f16_u16 v5, ttmp15
-// GFX12: v_cvt_f16_u16_e32 v5, ttmp15            ; encoding: [0x7b,0xa0,0x0a,0x7e]
+// GFX12: v_cvt_f16_u16_e32 v5, ttmp15 ; encoding: [0x7b,0xa0,0x0a,0x7e]
 
 v_cvt_f16_u16 v5, m0
-// GFX12: v_cvt_f16_u16_e32 v5, m0                ; encoding: [0x7d,0xa0,0x0a,0x7e]
+// GFX12: v_cvt_f16_u16_e32 v5, m0 ; encoding: [0x7d,0xa0,0x0a,0x7e]
 
 v_cvt_f16_u16 v5, exec_lo
-// GFX12: v_cvt_f16_u16_e32 v5, exec_lo           ; encoding: [0x7e,0xa0,0x0a,0x7e]
+// GFX12: v_cvt_f16_u16_e32 v5, exec_lo ; encoding: [0x7e,0xa0,0x0a,0x7e]
 
 v_cvt_f16_u16 v5, exec_hi
-// GFX12: v_cvt_f16_u16_e32 v5, exec_hi           ; encoding: [0x7f,0xa0,0x0a,0x7e]
+// GFX12: v_cvt_f16_u16_e32 v5, exec_hi ; encoding: [0x7f,0xa0,0x0a,0x7e]
 
 v_cvt_f16_u16 v5, null
-// GFX12: v_cvt_f16_u16_e32 v5, null              ; encoding: [0x7c,0xa0,0x0a,0x7e]
+// GFX12: v_cvt_f16_u16_e32 v5, null ; encoding: [0x7c,0xa0,0x0a,0x7e]
 
 v_cvt_f16_u16 v5, -1
-// GFX12: v_cvt_f16_u16_e32 v5, -1                ; encoding: [0xc1,0xa0,0x0a,0x7e]
+// GFX12: v_cvt_f16_u16_e32 v5, -1 ; encoding: [0xc1,0xa0,0x0a,0x7e]
 
 v_cvt_f16_u16 v5, 0.5
-// GFX12-ASM: v_cvt_f16_u16_e32 v5, 0.5               ; encoding: [0xf0,0xa0,0x0a,0x7e]
-// GFX12-DIS: v_cvt_f16_u16_e32 v5, 0x3800            ; encoding: [0xff,0xa0,0x0a,0x7e,0x00,0x38,0x00,0x00]
+// GFX12-ASM: v_cvt_f16_u16_e32 v5, 0.5 ; encoding: [0xf0,0xa0,0x0a,0x7e]
+// GFX12-DIS: v_cvt_f16_u16_e32 v5, 0x3800 ; encoding: [0xff,0xa0,0x0a,0x7e,0x00,0x38,0x00,0x00]
 
 v_cvt_f16_u16 v5, src_scc
-// GFX12: v_cvt_f16_u16_e32 v5, src_scc           ; encoding: [0xfd,0xa0,0x0a,0x7e]
+// GFX12: v_cvt_f16_u16_e32 v5, src_scc ; encoding: [0xfd,0xa0,0x0a,0x7e]
 
 v_cvt_f16_u16 v127, 0xfe0b
-// GFX12: v_cvt_f16_u16_e32 v127, 0xfe0b          ; encoding: [0xff,0xa0,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_cvt_f16_u16_e32 v127, 0xfe0b ; encoding: [0xff,0xa0,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_cvt_f32_f16 v5, v1
-// GFX12: v_cvt_f32_f16_e32 v5, v1                ; encoding: [0x01,0x17,0x0a,0x7e]
+// GFX12: v_cvt_f32_f16_e32 v5, v1 ; encoding: [0x01,0x17,0x0a,0x7e]
 
 v_cvt_f32_f16 v5, v127
-// GFX12: v_cvt_f32_f16_e32 v5, v127              ; encoding: [0x7f,0x17,0x0a,0x7e]
+// GFX12: v_cvt_f32_f16_e32 v5, v127 ; encoding: [0x7f,0x17,0x0a,0x7e]
 
 v_cvt_f32_f16 v5, s1
-// GFX12: v_cvt_f32_f16_e32 v5, s1                ; encoding: [0x01,0x16,0x0a,0x7e]
+// GFX12: v_cvt_f32_f16_e32 v5, s1 ; encoding: [0x01,0x16,0x0a,0x7e]
 
 v_cvt_f32_f16 v5, s105
-// GFX12: v_cvt_f32_f16_e32 v5, s105              ; encoding: [0x69,0x16,0x0a,0x7e]
+// GFX12: v_cvt_f32_f16_e32 v5, s105 ; encoding: [0x69,0x16,0x0a,0x7e]
 
 v_cvt_f32_f16 v5, vcc_lo
-// GFX12: v_cvt_f32_f16_e32 v5, vcc_lo            ; encoding: [0x6a,0x16,0x0a,0x7e]
+// GFX12: v_cvt_f32_f16_e32 v5, vcc_lo ; encoding: [0x6a,0x16,0x0a,0x7e]
 
 v_cvt_f32_f16 v5, vcc_hi
-// GFX12: v_cvt_f32_f16_e32 v5, vcc_hi            ; encoding: [0x6b,0x16,0x0a,0x7e]
+// GFX12: v_cvt_f32_f16_e32 v5, vcc_hi ; encoding: [0x6b,0x16,0x0a,0x7e]
 
 v_cvt_f32_f16 v5, ttmp15
-// GFX12: v_cvt_f32_f16_e32 v5, ttmp15            ; encoding: [0x7b,0x16,0x0a,0x7e]
+// GFX12: v_cvt_f32_f16_e32 v5, ttmp15 ; encoding: [0x7b,0x16,0x0a,0x7e]
 
 v_cvt_f32_f16 v5, m0
-// GFX12: v_cvt_f32_f16_e32 v5, m0                ; encoding: [0x7d,0x16,0x0a,0x7e]
+// GFX12: v_cvt_f32_f16_e32 v5, m0 ; encoding: [0x7d,0x16,0x0a,0x7e]
 
 v_cvt_f32_f16 v5, exec_lo
-// GFX12: v_cvt_f32_f16_e32 v5, exec_lo           ; encoding: [0x7e,0x16,0x0a,0x7e]
+// GFX12: v_cvt_f32_f16_e32 v5, exec_lo ; encoding: [0x7e,0x16,0x0a,0x7e]
 
 v_cvt_f32_f16 v5, exec_hi
-// GFX12: v_cvt_f32_f16_e32 v5, exec_hi           ; encoding: [0x7f,0x16,0x0a,0x7e]
+// GFX12: v_cvt_f32_f16_e32 v5, exec_hi ; encoding: [0x7f,0x16,0x0a,0x7e]
 
 v_cvt_f32_f16 v5, null
-// GFX12: v_cvt_f32_f16_e32 v5, null              ; encoding: [0x7c,0x16,0x0a,0x7e]
+// GFX12: v_cvt_f32_f16_e32 v5, null ; encoding: [0x7c,0x16,0x0a,0x7e]
 
 v_cvt_f32_f16 v5, -1
-// GFX12: v_cvt_f32_f16_e32 v5, -1                ; encoding: [0xc1,0x16,0x0a,0x7e]
+// GFX12: v_cvt_f32_f16_e32 v5, -1 ; encoding: [0xc1,0x16,0x0a,0x7e]
 
 v_cvt_f32_f16 v5, 0.5
-// GFX12: v_cvt_f32_f16_e32 v5, 0.5               ; encoding: [0xf0,0x16,0x0a,0x7e]
+// GFX12: v_cvt_f32_f16_e32 v5, 0.5 ; encoding: [0xf0,0x16,0x0a,0x7e]
 
 v_cvt_f32_f16 v5, src_scc
-// GFX12: v_cvt_f32_f16_e32 v5, src_scc           ; encoding: [0xfd,0x16,0x0a,0x7e]
+// GFX12: v_cvt_f32_f16_e32 v5, src_scc ; encoding: [0xfd,0x16,0x0a,0x7e]
 
 v_cvt_f32_f16 v255, 0xfe0b
-// GFX12: v_cvt_f32_f16_e32 v255, 0xfe0b          ; encoding: [0xff,0x16,0xfe,0x7f,0x0b,0xfe,0x00,0x00]
+// GFX12: v_cvt_f32_f16_e32 v255, 0xfe0b ; encoding: [0xff,0x16,0xfe,0x7f,0x0b,0xfe,0x00,0x00]
 
 v_cvt_f32_f64 v5, v[1:2]
-// GFX12: v_cvt_f32_f64_e32 v5, v[1:2]            ; encoding: [0x01,0x1f,0x0a,0x7e]
+// GFX12: v_cvt_f32_f64_e32 v5, v[1:2] ; encoding: [0x01,0x1f,0x0a,0x7e]
 
 v_cvt_f32_f64 v5, v[254:255]
-// GFX12: v_cvt_f32_f64_e32 v5, v[254:255]        ; encoding: [0xfe,0x1f,0x0a,0x7e]
+// GFX12: v_cvt_f32_f64_e32 v5, v[254:255] ; encoding: [0xfe,0x1f,0x0a,0x7e]
 
 v_cvt_f32_f64 v5, s[2:3]
-// GFX12: v_cvt_f32_f64_e32 v5, s[2:3]            ; encoding: [0x02,0x1e,0x0a,0x7e]
+// GFX12: v_cvt_f32_f64_e32 v5, s[2:3] ; encoding: [0x02,0x1e,0x0a,0x7e]
 
 v_cvt_f32_f64 v5, s[104:105]
-// GFX12: v_cvt_f32_f64_e32 v5, s[104:105]        ; encoding: [0x68,0x1e,0x0a,0x7e]
+// GFX12: v_cvt_f32_f64_e32 v5, s[104:105] ; encoding: [0x68,0x1e,0x0a,0x7e]
 
 v_cvt_f32_f64 v5, vcc
-// GFX12: v_cvt_f32_f64_e32 v5, vcc               ; encoding: [0x6a,0x1e,0x0a,0x7e]
+// GFX12: v_cvt_f32_f64_e32 v5, vcc ; encoding: [0x6a,0x1e,0x0a,0x7e]
 
 v_cvt_f32_f64 v5, ttmp[14:15]
-// GFX12: v_cvt_f32_f64_e32 v5, ttmp[14:15]       ; encoding: [0x7a,0x1e,0x0a,0x7e]
+// GFX12: v_cvt_f32_f64_e32 v5, ttmp[14:15] ; encoding: [0x7a,0x1e,0x0a,0x7e]
 
 v_cvt_f32_f64 v5, exec
-// GFX12: v_cvt_f32_f64_e32 v5, exec              ; encoding: [0x7e,0x1e,0x0a,0x7e]
+// GFX12: v_cvt_f32_f64_e32 v5, exec ; encoding: [0x7e,0x1e,0x0a,0x7e]
 
 v_cvt_f32_f64 v5, null
-// GFX12: v_cvt_f32_f64_e32 v5, null              ; encoding: [0x7c,0x1e,0x0a,0x7e]
+// GFX12: v_cvt_f32_f64_e32 v5, null ; encoding: [0x7c,0x1e,0x0a,0x7e]
 
 v_cvt_f32_f64 v5, -1
-// GFX12: v_cvt_f32_f64_e32 v5, -1                ; encoding: [0xc1,0x1e,0x0a,0x7e]
+// GFX12: v_cvt_f32_f64_e32 v5, -1 ; encoding: [0xc1,0x1e,0x0a,0x7e]
 
 v_cvt_f32_f64 v5, 0.5
-// GFX12: v_cvt_f32_f64_e32 v5, 0.5               ; encoding: [0xf0,0x1e,0x0a,0x7e]
+// GFX12: v_cvt_f32_f64_e32 v5, 0.5 ; encoding: [0xf0,0x1e,0x0a,0x7e]
 
 v_cvt_f32_f64 v5, src_scc
-// GFX12: v_cvt_f32_f64_e32 v5, src_scc           ; encoding: [0xfd,0x1e,0x0a,0x7e]
+// GFX12: v_cvt_f32_f64_e32 v5, src_scc ; encoding: [0xfd,0x1e,0x0a,0x7e]
 
 v_cvt_f32_f64 v255, 0xaf123456
-// GFX12: v_cvt_f32_f64_e32 v255, 0xaf123456      ; encoding: [0xff,0x1e,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_cvt_f32_f64_e32 v255, 0xaf123456 ; encoding: [0xff,0x1e,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cvt_f32_i32 v5, v1
-// GFX12: v_cvt_f32_i32_e32 v5, v1                ; encoding: [0x01,0x0b,0x0a,0x7e]
+// GFX12: v_cvt_f32_i32_e32 v5, v1 ; encoding: [0x01,0x0b,0x0a,0x7e]
 
 v_cvt_f32_i32 v5, v255
-// GFX12: v_cvt_f32_i32_e32 v5, v255              ; encoding: [0xff,0x0b,0x0a,0x7e]
+// GFX12: v_cvt_f32_i32_e32 v5, v255 ; encoding: [0xff,0x0b,0x0a,0x7e]
 
 v_cvt_f32_i32 v5, s1
-// GFX12: v_cvt_f32_i32_e32 v5, s1                ; encoding: [0x01,0x0a,0x0a,0x7e]
+// GFX12: v_cvt_f32_i32_e32 v5, s1 ; encoding: [0x01,0x0a,0x0a,0x7e]
 
 v_cvt_f32_i32 v5, s105
-// GFX12: v_cvt_f32_i32_e32 v5, s105              ; encoding: [0x69,0x0a,0x0a,0x7e]
+// GFX12: v_cvt_f32_i32_e32 v5, s105 ; encoding: [0x69,0x0a,0x0a,0x7e]
 
 v_cvt_f32_i32 v5, vcc_lo
-// GFX12: v_cvt_f32_i32_e32 v5, vcc_lo            ; encoding: [0x6a,0x0a,0x0a,0x7e]
+// GFX12: v_cvt_f32_i32_e32 v5, vcc_lo ; encoding: [0x6a,0x0a,0x0a,0x7e]
 
 v_cvt_f32_i32 v5, vcc_hi
-// GFX12: v_cvt_f32_i32_e32 v5, vcc_hi            ; encoding: [0x6b,0x0a,0x0a,0x7e]
+// GFX12: v_cvt_f32_i32_e32 v5, vcc_hi ; encoding: [0x6b,0x0a,0x0a,0x7e]
 
 v_cvt_f32_i32 v5, ttmp15
-// GFX12: v_cvt_f32_i32_e32 v5, ttmp15            ; encoding: [0x7b,0x0a,0x0a,0x7e]
+// GFX12: v_cvt_f32_i32_e32 v5, ttmp15 ; encoding: [0x7b,0x0a,0x0a,0x7e]
 
 v_cvt_f32_i32 v5, m0
-// GFX12: v_cvt_f32_i32_e32 v5, m0                ; encoding: [0x7d,0x0a,0x0a,0x7e]
+// GFX12: v_cvt_f32_i32_e32 v5, m0 ; encoding: [0x7d,0x0a,0x0a,0x7e]
 
 v_cvt_f32_i32 v5, exec_lo
-// GFX12: v_cvt_f32_i32_e32 v5, exec_lo           ; encoding: [0x7e,0x0a,0x0a,0x7e]
+// GFX12: v_cvt_f32_i32_e32 v5, exec_lo ; encoding: [0x7e,0x0a,0x0a,0x7e]
 
 v_cvt_f32_i32 v5, exec_hi
-// GFX12: v_cvt_f32_i32_e32 v5, exec_hi           ; encoding: [0x7f,0x0a,0x0a,0x7e]
+// GFX12: v_cvt_f32_i32_e32 v5, exec_hi ; encoding: [0x7f,0x0a,0x0a,0x7e]
 
 v_cvt_f32_i32 v5, null
-// GFX12: v_cvt_f32_i32_e32 v5, null              ; encoding: [0x7c,0x0a,0x0a,0x7e]
+// GFX12: v_cvt_f32_i32_e32 v5, null ; encoding: [0x7c,0x0a,0x0a,0x7e]
 
 v_cvt_f32_i32 v5, -1
-// GFX12: v_cvt_f32_i32_e32 v5, -1                ; encoding: [0xc1,0x0a,0x0a,0x7e]
+// GFX12: v_cvt_f32_i32_e32 v5, -1 ; encoding: [0xc1,0x0a,0x0a,0x7e]
 
 v_cvt_f32_i32 v5, 0.5
-// GFX12: v_cvt_f32_i32_e32 v5, 0.5               ; encoding: [0xf0,0x0a,0x0a,0x7e]
+// GFX12: v_cvt_f32_i32_e32 v5, 0.5 ; encoding: [0xf0,0x0a,0x0a,0x7e]
 
 v_cvt_f32_i32 v5, src_scc
-// GFX12: v_cvt_f32_i32_e32 v5, src_scc           ; encoding: [0xfd,0x0a,0x0a,0x7e]
+// GFX12: v_cvt_f32_i32_e32 v5, src_scc ; encoding: [0xfd,0x0a,0x0a,0x7e]
 
 v_cvt_f32_i32 v255, 0xaf123456
-// GFX12: v_cvt_f32_i32_e32 v255, 0xaf123456      ; encoding: [0xff,0x0a,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_cvt_f32_i32_e32 v255, 0xaf123456 ; encoding: [0xff,0x0a,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cvt_f32_u32 v5, v1
-// GFX12: v_cvt_f32_u32_e32 v5, v1                ; encoding: [0x01,0x0d,0x0a,0x7e]
+// GFX12: v_cvt_f32_u32_e32 v5, v1 ; encoding: [0x01,0x0d,0x0a,0x7e]
 
 v_cvt_f32_u32 v5, v255
-// GFX12: v_cvt_f32_u32_e32 v5, v255              ; encoding: [0xff,0x0d,0x0a,0x7e]
+// GFX12: v_cvt_f32_u32_e32 v5, v255 ; encoding: [0xff,0x0d,0x0a,0x7e]
 
 v_cvt_f32_u32 v5, s1
-// GFX12: v_cvt_f32_u32_e32 v5, s1                ; encoding: [0x01,0x0c,0x0a,0x7e]
+// GFX12: v_cvt_f32_u32_e32 v5, s1 ; encoding: [0x01,0x0c,0x0a,0x7e]
 
 v_cvt_f32_u32 v5, s105
-// GFX12: v_cvt_f32_u32_e32 v5, s105              ; encoding: [0x69,0x0c,0x0a,0x7e]
+// GFX12: v_cvt_f32_u32_e32 v5, s105 ; encoding: [0x69,0x0c,0x0a,0x7e]
 
 v_cvt_f32_u32 v5, vcc_lo
-// GFX12: v_cvt_f32_u32_e32 v5, vcc_lo            ; encoding: [0x6a,0x0c,0x0a,0x7e]
+// GFX12: v_cvt_f32_u32_e32 v5, vcc_lo ; encoding: [0x6a,0x0c,0x0a,0x7e]
 
 v_cvt_f32_u32 v5, vcc_hi
-// GFX12: v_cvt_f32_u32_e32 v5, vcc_hi            ; encoding: [0x6b,0x0c,0x0a,0x7e]
+// GFX12: v_cvt_f32_u32_e32 v5, vcc_hi ; encoding: [0x6b,0x0c,0x0a,0x7e]
 
 v_cvt_f32_u32 v5, ttmp15
-// GFX12: v_cvt_f32_u32_e32 v5, ttmp15            ; encoding: [0x7b,0x0c,0x0a,0x7e]
+// GFX12: v_cvt_f32_u32_e32 v5, ttmp15 ; encoding: [0x7b,0x0c,0x0a,0x7e]
 
 v_cvt_f32_u32 v5, m0
-// GFX12: v_cvt_f32_u32_e32 v5, m0                ; encoding: [0x7d,0x0c,0x0a,0x7e]
+// GFX12: v_cvt_f32_u32_e32 v5, m0 ; encoding: [0x7d,0x0c,0x0a,0x7e]
 
 v_cvt_f32_u32 v5, exec_lo
-// GFX12: v_cvt_f32_u32_e32 v5, exec_lo           ; encoding: [0x7e,0x0c,0x0a,0x7e]
+// GFX12: v_cvt_f32_u32_e32 v5, exec_lo ; encoding: [0x7e,0x0c,0x0a,0x7e]
 
 v_cvt_f32_u32 v5, exec_hi
-// GFX12: v_cvt_f32_u32_e32 v5, exec_hi           ; encoding: [0x7f,0x0c,0x0a,0x7e]
+// GFX12: v_cvt_f32_u32_e32 v5, exec_hi ; encoding: [0x7f,0x0c,0x0a,0x7e]
 
 v_cvt_f32_u32 v5, null
-// GFX12: v_cvt_f32_u32_e32 v5, null              ; encoding: [0x7c,0x0c,0x0a,0x7e]
+// GFX12: v_cvt_f32_u32_e32 v5, null ; encoding: [0x7c,0x0c,0x0a,0x7e]
 
 v_cvt_f32_u32 v5, -1
-// GFX12: v_cvt_f32_u32_e32 v5, -1                ; encoding: [0xc1,0x0c,0x0a,0x7e]
+// GFX12: v_cvt_f32_u32_e32 v5, -1 ; encoding: [0xc1,0x0c,0x0a,0x7e]
 
 v_cvt_f32_u32 v5, 0.5
-// GFX12: v_cvt_f32_u32_e32 v5, 0.5               ; encoding: [0xf0,0x0c,0x0a,0x7e]
+// GFX12: v_cvt_f32_u32_e32 v5, 0.5 ; encoding: [0xf0,0x0c,0x0a,0x7e]
 
 v_cvt_f32_u32 v5, src_scc
-// GFX12: v_cvt_f32_u32_e32 v5, src_scc           ; encoding: [0xfd,0x0c,0x0a,0x7e]
+// GFX12: v_cvt_f32_u32_e32 v5, src_scc ; encoding: [0xfd,0x0c,0x0a,0x7e]
 
 v_cvt_f32_u32 v255, 0xaf123456
-// GFX12: v_cvt_f32_u32_e32 v255, 0xaf123456      ; encoding: [0xff,0x0c,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_cvt_f32_u32_e32 v255, 0xaf123456 ; encoding: [0xff,0x0c,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cvt_f32_ubyte0 v5, v1
-// GFX12: v_cvt_f32_ubyte0_e32 v5, v1             ; encoding: [0x01,0x23,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte0_e32 v5, v1 ; encoding: [0x01,0x23,0x0a,0x7e]
 
 v_cvt_f32_ubyte0 v5, v255
-// GFX12: v_cvt_f32_ubyte0_e32 v5, v255           ; encoding: [0xff,0x23,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte0_e32 v5, v255 ; encoding: [0xff,0x23,0x0a,0x7e]
 
 v_cvt_f32_ubyte0 v5, s1
-// GFX12: v_cvt_f32_ubyte0_e32 v5, s1             ; encoding: [0x01,0x22,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte0_e32 v5, s1 ; encoding: [0x01,0x22,0x0a,0x7e]
 
 v_cvt_f32_ubyte0 v5, s105
-// GFX12: v_cvt_f32_ubyte0_e32 v5, s105           ; encoding: [0x69,0x22,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte0_e32 v5, s105 ; encoding: [0x69,0x22,0x0a,0x7e]
 
 v_cvt_f32_ubyte0 v5, vcc_lo
-// GFX12: v_cvt_f32_ubyte0_e32 v5, vcc_lo         ; encoding: [0x6a,0x22,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte0_e32 v5, vcc_lo ; encoding: [0x6a,0x22,0x0a,0x7e]
 
 v_cvt_f32_ubyte0 v5, vcc_hi
-// GFX12: v_cvt_f32_ubyte0_e32 v5, vcc_hi         ; encoding: [0x6b,0x22,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte0_e32 v5, vcc_hi ; encoding: [0x6b,0x22,0x0a,0x7e]
 
 v_cvt_f32_ubyte0 v5, ttmp15
-// GFX12: v_cvt_f32_ubyte0_e32 v5, ttmp15         ; encoding: [0x7b,0x22,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte0_e32 v5, ttmp15 ; encoding: [0x7b,0x22,0x0a,0x7e]
 
 v_cvt_f32_ubyte0 v5, m0
-// GFX12: v_cvt_f32_ubyte0_e32 v5, m0             ; encoding: [0x7d,0x22,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte0_e32 v5, m0 ; encoding: [0x7d,0x22,0x0a,0x7e]
 
 v_cvt_f32_ubyte0 v5, exec_lo
-// GFX12: v_cvt_f32_ubyte0_e32 v5, exec_lo        ; encoding: [0x7e,0x22,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte0_e32 v5, exec_lo ; encoding: [0x7e,0x22,0x0a,0x7e]
 
 v_cvt_f32_ubyte0 v5, exec_hi
-// GFX12: v_cvt_f32_ubyte0_e32 v5, exec_hi        ; encoding: [0x7f,0x22,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte0_e32 v5, exec_hi ; encoding: [0x7f,0x22,0x0a,0x7e]
 
 v_cvt_f32_ubyte0 v5, null
-// GFX12: v_cvt_f32_ubyte0_e32 v5, null           ; encoding: [0x7c,0x22,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte0_e32 v5, null ; encoding: [0x7c,0x22,0x0a,0x7e]
 
 v_cvt_f32_ubyte0 v5, -1
-// GFX12: v_cvt_f32_ubyte0_e32 v5, -1             ; encoding: [0xc1,0x22,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte0_e32 v5, -1 ; encoding: [0xc1,0x22,0x0a,0x7e]
 
 v_cvt_f32_ubyte0 v5, 0.5
-// GFX12: v_cvt_f32_ubyte0_e32 v5, 0.5            ; encoding: [0xf0,0x22,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte0_e32 v5, 0.5 ; encoding: [0xf0,0x22,0x0a,0x7e]
 
 v_cvt_f32_ubyte0 v5, src_scc
-// GFX12: v_cvt_f32_ubyte0_e32 v5, src_scc        ; encoding: [0xfd,0x22,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte0_e32 v5, src_scc ; encoding: [0xfd,0x22,0x0a,0x7e]
 
 v_cvt_f32_ubyte0 v255, 0xaf123456
-// GFX12: v_cvt_f32_ubyte0_e32 v255, 0xaf123456   ; encoding: [0xff,0x22,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_cvt_f32_ubyte0_e32 v255, 0xaf123456 ; encoding: [0xff,0x22,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cvt_f32_ubyte1 v5, v1
-// GFX12: v_cvt_f32_ubyte1_e32 v5, v1             ; encoding: [0x01,0x25,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte1_e32 v5, v1 ; encoding: [0x01,0x25,0x0a,0x7e]
 
 v_cvt_f32_ubyte1 v5, v255
-// GFX12: v_cvt_f32_ubyte1_e32 v5, v255           ; encoding: [0xff,0x25,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte1_e32 v5, v255 ; encoding: [0xff,0x25,0x0a,0x7e]
 
 v_cvt_f32_ubyte1 v5, s1
-// GFX12: v_cvt_f32_ubyte1_e32 v5, s1             ; encoding: [0x01,0x24,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte1_e32 v5, s1 ; encoding: [0x01,0x24,0x0a,0x7e]
 
 v_cvt_f32_ubyte1 v5, s105
-// GFX12: v_cvt_f32_ubyte1_e32 v5, s105           ; encoding: [0x69,0x24,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte1_e32 v5, s105 ; encoding: [0x69,0x24,0x0a,0x7e]
 
 v_cvt_f32_ubyte1 v5, vcc_lo
-// GFX12: v_cvt_f32_ubyte1_e32 v5, vcc_lo         ; encoding: [0x6a,0x24,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte1_e32 v5, vcc_lo ; encoding: [0x6a,0x24,0x0a,0x7e]
 
 v_cvt_f32_ubyte1 v5, vcc_hi
-// GFX12: v_cvt_f32_ubyte1_e32 v5, vcc_hi         ; encoding: [0x6b,0x24,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte1_e32 v5, vcc_hi ; encoding: [0x6b,0x24,0x0a,0x7e]
 
 v_cvt_f32_ubyte1 v5, ttmp15
-// GFX12: v_cvt_f32_ubyte1_e32 v5, ttmp15         ; encoding: [0x7b,0x24,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte1_e32 v5, ttmp15 ; encoding: [0x7b,0x24,0x0a,0x7e]
 
 v_cvt_f32_ubyte1 v5, m0
-// GFX12: v_cvt_f32_ubyte1_e32 v5, m0             ; encoding: [0x7d,0x24,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte1_e32 v5, m0 ; encoding: [0x7d,0x24,0x0a,0x7e]
 
 v_cvt_f32_ubyte1 v5, exec_lo
-// GFX12: v_cvt_f32_ubyte1_e32 v5, exec_lo        ; encoding: [0x7e,0x24,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte1_e32 v5, exec_lo ; encoding: [0x7e,0x24,0x0a,0x7e]
 
 v_cvt_f32_ubyte1 v5, exec_hi
-// GFX12: v_cvt_f32_ubyte1_e32 v5, exec_hi        ; encoding: [0x7f,0x24,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte1_e32 v5, exec_hi ; encoding: [0x7f,0x24,0x0a,0x7e]
 
 v_cvt_f32_ubyte1 v5, null
-// GFX12: v_cvt_f32_ubyte1_e32 v5, null           ; encoding: [0x7c,0x24,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte1_e32 v5, null ; encoding: [0x7c,0x24,0x0a,0x7e]
 
 v_cvt_f32_ubyte1 v5, -1
-// GFX12: v_cvt_f32_ubyte1_e32 v5, -1             ; encoding: [0xc1,0x24,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte1_e32 v5, -1 ; encoding: [0xc1,0x24,0x0a,0x7e]
 
 v_cvt_f32_ubyte1 v5, 0.5
-// GFX12: v_cvt_f32_ubyte1_e32 v5, 0.5            ; encoding: [0xf0,0x24,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte1_e32 v5, 0.5 ; encoding: [0xf0,0x24,0x0a,0x7e]
 
 v_cvt_f32_ubyte1 v5, src_scc
-// GFX12: v_cvt_f32_ubyte1_e32 v5, src_scc        ; encoding: [0xfd,0x24,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte1_e32 v5, src_scc ; encoding: [0xfd,0x24,0x0a,0x7e]
 
 v_cvt_f32_ubyte1 v255, 0xaf123456
-// GFX12: v_cvt_f32_ubyte1_e32 v255, 0xaf123456   ; encoding: [0xff,0x24,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_cvt_f32_ubyte1_e32 v255, 0xaf123456 ; encoding: [0xff,0x24,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cvt_f32_ubyte2 v5, v1
-// GFX12: v_cvt_f32_ubyte2_e32 v5, v1             ; encoding: [0x01,0x27,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte2_e32 v5, v1 ; encoding: [0x01,0x27,0x0a,0x7e]
 
 v_cvt_f32_ubyte2 v5, v255
-// GFX12: v_cvt_f32_ubyte2_e32 v5, v255           ; encoding: [0xff,0x27,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte2_e32 v5, v255 ; encoding: [0xff,0x27,0x0a,0x7e]
 
 v_cvt_f32_ubyte2 v5, s1
-// GFX12: v_cvt_f32_ubyte2_e32 v5, s1             ; encoding: [0x01,0x26,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte2_e32 v5, s1 ; encoding: [0x01,0x26,0x0a,0x7e]
 
 v_cvt_f32_ubyte2 v5, s105
-// GFX12: v_cvt_f32_ubyte2_e32 v5, s105           ; encoding: [0x69,0x26,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte2_e32 v5, s105 ; encoding: [0x69,0x26,0x0a,0x7e]
 
 v_cvt_f32_ubyte2 v5, vcc_lo
-// GFX12: v_cvt_f32_ubyte2_e32 v5, vcc_lo         ; encoding: [0x6a,0x26,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte2_e32 v5, vcc_lo ; encoding: [0x6a,0x26,0x0a,0x7e]
 
 v_cvt_f32_ubyte2 v5, vcc_hi
-// GFX12: v_cvt_f32_ubyte2_e32 v5, vcc_hi         ; encoding: [0x6b,0x26,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte2_e32 v5, vcc_hi ; encoding: [0x6b,0x26,0x0a,0x7e]
 
 v_cvt_f32_ubyte2 v5, ttmp15
-// GFX12: v_cvt_f32_ubyte2_e32 v5, ttmp15         ; encoding: [0x7b,0x26,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte2_e32 v5, ttmp15 ; encoding: [0x7b,0x26,0x0a,0x7e]
 
 v_cvt_f32_ubyte2 v5, m0
-// GFX12: v_cvt_f32_ubyte2_e32 v5, m0             ; encoding: [0x7d,0x26,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte2_e32 v5, m0 ; encoding: [0x7d,0x26,0x0a,0x7e]
 
 v_cvt_f32_ubyte2 v5, exec_lo
-// GFX12: v_cvt_f32_ubyte2_e32 v5, exec_lo        ; encoding: [0x7e,0x26,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte2_e32 v5, exec_lo ; encoding: [0x7e,0x26,0x0a,0x7e]
 
 v_cvt_f32_ubyte2 v5, exec_hi
-// GFX12: v_cvt_f32_ubyte2_e32 v5, exec_hi        ; encoding: [0x7f,0x26,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte2_e32 v5, exec_hi ; encoding: [0x7f,0x26,0x0a,0x7e]
 
 v_cvt_f32_ubyte2 v5, null
-// GFX12: v_cvt_f32_ubyte2_e32 v5, null           ; encoding: [0x7c,0x26,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte2_e32 v5, null ; encoding: [0x7c,0x26,0x0a,0x7e]
 
 v_cvt_f32_ubyte2 v5, -1
-// GFX12: v_cvt_f32_ubyte2_e32 v5, -1             ; encoding: [0xc1,0x26,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte2_e32 v5, -1 ; encoding: [0xc1,0x26,0x0a,0x7e]
 
 v_cvt_f32_ubyte2 v5, 0.5
-// GFX12: v_cvt_f32_ubyte2_e32 v5, 0.5            ; encoding: [0xf0,0x26,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte2_e32 v5, 0.5 ; encoding: [0xf0,0x26,0x0a,0x7e]
 
 v_cvt_f32_ubyte2 v5, src_scc
-// GFX12: v_cvt_f32_ubyte2_e32 v5, src_scc        ; encoding: [0xfd,0x26,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte2_e32 v5, src_scc ; encoding: [0xfd,0x26,0x0a,0x7e]
 
 v_cvt_f32_ubyte2 v255, 0xaf123456
-// GFX12: v_cvt_f32_ubyte2_e32 v255, 0xaf123456   ; encoding: [0xff,0x26,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_cvt_f32_ubyte2_e32 v255, 0xaf123456 ; encoding: [0xff,0x26,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cvt_f32_ubyte3 v5, v1
-// GFX12: v_cvt_f32_ubyte3_e32 v5, v1             ; encoding: [0x01,0x29,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte3_e32 v5, v1 ; encoding: [0x01,0x29,0x0a,0x7e]
 
 v_cvt_f32_ubyte3 v5, v255
-// GFX12: v_cvt_f32_ubyte3_e32 v5, v255           ; encoding: [0xff,0x29,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte3_e32 v5, v255 ; encoding: [0xff,0x29,0x0a,0x7e]
 
 v_cvt_f32_ubyte3 v5, s1
-// GFX12: v_cvt_f32_ubyte3_e32 v5, s1             ; encoding: [0x01,0x28,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte3_e32 v5, s1 ; encoding: [0x01,0x28,0x0a,0x7e]
 
 v_cvt_f32_ubyte3 v5, s105
-// GFX12: v_cvt_f32_ubyte3_e32 v5, s105           ; encoding: [0x69,0x28,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte3_e32 v5, s105 ; encoding: [0x69,0x28,0x0a,0x7e]
 
 v_cvt_f32_ubyte3 v5, vcc_lo
-// GFX12: v_cvt_f32_ubyte3_e32 v5, vcc_lo         ; encoding: [0x6a,0x28,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte3_e32 v5, vcc_lo ; encoding: [0x6a,0x28,0x0a,0x7e]
 
 v_cvt_f32_ubyte3 v5, vcc_hi
-// GFX12: v_cvt_f32_ubyte3_e32 v5, vcc_hi         ; encoding: [0x6b,0x28,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte3_e32 v5, vcc_hi ; encoding: [0x6b,0x28,0x0a,0x7e]
 
 v_cvt_f32_ubyte3 v5, ttmp15
-// GFX12: v_cvt_f32_ubyte3_e32 v5, ttmp15         ; encoding: [0x7b,0x28,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte3_e32 v5, ttmp15 ; encoding: [0x7b,0x28,0x0a,0x7e]
 
 v_cvt_f32_ubyte3 v5, m0
-// GFX12: v_cvt_f32_ubyte3_e32 v5, m0             ; encoding: [0x7d,0x28,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte3_e32 v5, m0 ; encoding: [0x7d,0x28,0x0a,0x7e]
 
 v_cvt_f32_ubyte3 v5, exec_lo
-// GFX12: v_cvt_f32_ubyte3_e32 v5, exec_lo        ; encoding: [0x7e,0x28,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte3_e32 v5, exec_lo ; encoding: [0x7e,0x28,0x0a,0x7e]
 
 v_cvt_f32_ubyte3 v5, exec_hi
-// GFX12: v_cvt_f32_ubyte3_e32 v5, exec_hi        ; encoding: [0x7f,0x28,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte3_e32 v5, exec_hi ; encoding: [0x7f,0x28,0x0a,0x7e]
 
 v_cvt_f32_ubyte3 v5, null
-// GFX12: v_cvt_f32_ubyte3_e32 v5, null           ; encoding: [0x7c,0x28,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte3_e32 v5, null ; encoding: [0x7c,0x28,0x0a,0x7e]
 
 v_cvt_f32_ubyte3 v5, -1
-// GFX12: v_cvt_f32_ubyte3_e32 v5, -1             ; encoding: [0xc1,0x28,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte3_e32 v5, -1 ; encoding: [0xc1,0x28,0x0a,0x7e]
 
 v_cvt_f32_ubyte3 v5, 0.5
-// GFX12: v_cvt_f32_ubyte3_e32 v5, 0.5            ; encoding: [0xf0,0x28,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte3_e32 v5, 0.5 ; encoding: [0xf0,0x28,0x0a,0x7e]
 
 v_cvt_f32_ubyte3 v5, src_scc
-// GFX12: v_cvt_f32_ubyte3_e32 v5, src_scc        ; encoding: [0xfd,0x28,0x0a,0x7e]
+// GFX12: v_cvt_f32_ubyte3_e32 v5, src_scc ; encoding: [0xfd,0x28,0x0a,0x7e]
 
 v_cvt_f32_ubyte3 v255, 0xaf123456
-// GFX12: v_cvt_f32_ubyte3_e32 v255, 0xaf123456   ; encoding: [0xff,0x28,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_cvt_f32_ubyte3_e32 v255, 0xaf123456 ; encoding: [0xff,0x28,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cvt_f64_f32 v[5:6], v1
-// GFX12: v_cvt_f64_f32_e32 v[5:6], v1            ; encoding: [0x01,0x21,0x0a,0x7e]
+// GFX12: v_cvt_f64_f32_e32 v[5:6], v1 ; encoding: [0x01,0x21,0x0a,0x7e]
 
 v_cvt_f64_f32 v[5:6], v255
-// GFX12: v_cvt_f64_f32_e32 v[5:6], v255          ; encoding: [0xff,0x21,0x0a,0x7e]
+// GFX12: v_cvt_f64_f32_e32 v[5:6], v255 ; encoding: [0xff,0x21,0x0a,0x7e]
 
 v_cvt_f64_f32 v[5:6], s1
-// GFX12: v_cvt_f64_f32_e32 v[5:6], s1            ; encoding: [0x01,0x20,0x0a,0x7e]
+// GFX12: v_cvt_f64_f32_e32 v[5:6], s1 ; encoding: [0x01,0x20,0x0a,0x7e]
 
 v_cvt_f64_f32 v[5:6], s105
-// GFX12: v_cvt_f64_f32_e32 v[5:6], s105          ; encoding: [0x69,0x20,0x0a,0x7e]
+// GFX12: v_cvt_f64_f32_e32 v[5:6], s105 ; encoding: [0x69,0x20,0x0a,0x7e]
 
 v_cvt_f64_f32 v[5:6], vcc_lo
-// GFX12: v_cvt_f64_f32_e32 v[5:6], vcc_lo        ; encoding: [0x6a,0x20,0x0a,0x7e]
+// GFX12: v_cvt_f64_f32_e32 v[5:6], vcc_lo ; encoding: [0x6a,0x20,0x0a,0x7e]
 
 v_cvt_f64_f32 v[5:6], vcc_hi
-// GFX12: v_cvt_f64_f32_e32 v[5:6], vcc_hi        ; encoding: [0x6b,0x20,0x0a,0x7e]
+// GFX12: v_cvt_f64_f32_e32 v[5:6], vcc_hi ; encoding: [0x6b,0x20,0x0a,0x7e]
 
 v_cvt_f64_f32 v[5:6], ttmp15
-// GFX12: v_cvt_f64_f32_e32 v[5:6], ttmp15        ; encoding: [0x7b,0x20,0x0a,0x7e]
+// GFX12: v_cvt_f64_f32_e32 v[5:6], ttmp15 ; encoding: [0x7b,0x20,0x0a,0x7e]
 
 v_cvt_f64_f32 v[5:6], m0
-// GFX12: v_cvt_f64_f32_e32 v[5:6], m0            ; encoding: [0x7d,0x20,0x0a,0x7e]
+// GFX12: v_cvt_f64_f32_e32 v[5:6], m0 ; encoding: [0x7d,0x20,0x0a,0x7e]
 
 v_cvt_f64_f32 v[5:6], exec_lo
-// GFX12: v_cvt_f64_f32_e32 v[5:6], exec_lo       ; encoding: [0x7e,0x20,0x0a,0x7e]
+// GFX12: v_cvt_f64_f32_e32 v[5:6], exec_lo ; encoding: [0x7e,0x20,0x0a,0x7e]
 
 v_cvt_f64_f32 v[5:6], exec_hi
-// GFX12: v_cvt_f64_f32_e32 v[5:6], exec_hi       ; encoding: [0x7f,0x20,0x0a,0x7e]
+// GFX12: v_cvt_f64_f32_e32 v[5:6], exec_hi ; encoding: [0x7f,0x20,0x0a,0x7e]
 
 v_cvt_f64_f32 v[5:6], null
-// GFX12: v_cvt_f64_f32_e32 v[5:6], null          ; encoding: [0x7c,0x20,0x0a,0x7e]
+// GFX12: v_cvt_f64_f32_e32 v[5:6], null ; encoding: [0x7c,0x20,0x0a,0x7e]
 
 v_cvt_f64_f32 v[5:6], -1
-// GFX12: v_cvt_f64_f32_e32 v[5:6], -1            ; encoding: [0xc1,0x20,0x0a,0x7e]
+// GFX12: v_cvt_f64_f32_e32 v[5:6], -1 ; encoding: [0xc1,0x20,0x0a,0x7e]
 
 v_cvt_f64_f32 v[5:6], 0.5
-// GFX12: v_cvt_f64_f32_e32 v[5:6], 0.5           ; encoding: [0xf0,0x20,0x0a,0x7e]
+// GFX12: v_cvt_f64_f32_e32 v[5:6], 0.5 ; encoding: [0xf0,0x20,0x0a,0x7e]
 
 v_cvt_f64_f32 v[5:6], src_scc
-// GFX12: v_cvt_f64_f32_e32 v[5:6], src_scc       ; encoding: [0xfd,0x20,0x0a,0x7e]
+// GFX12: v_cvt_f64_f32_e32 v[5:6], src_scc ; encoding: [0xfd,0x20,0x0a,0x7e]
 
 v_cvt_f64_f32 v[254:255], 0xaf123456
 // GFX12: v_cvt_f64_f32_e32 v[254:255], 0xaf123456 ; encoding: [0xff,0x20,0xfc,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cvt_f64_i32 v[5:6], v1
-// GFX12: v_cvt_f64_i32_e32 v[5:6], v1            ; encoding: [0x01,0x09,0x0a,0x7e]
+// GFX12: v_cvt_f64_i32_e32 v[5:6], v1 ; encoding: [0x01,0x09,0x0a,0x7e]
 
 v_cvt_f64_i32 v[5:6], v255
-// GFX12: v_cvt_f64_i32_e32 v[5:6], v255          ; encoding: [0xff,0x09,0x0a,0x7e]
+// GFX12: v_cvt_f64_i32_e32 v[5:6], v255 ; encoding: [0xff,0x09,0x0a,0x7e]
 
 v_cvt_f64_i32 v[5:6], s1
-// GFX12: v_cvt_f64_i32_e32 v[5:6], s1            ; encoding: [0x01,0x08,0x0a,0x7e]
+// GFX12: v_cvt_f64_i32_e32 v[5:6], s1 ; encoding: [0x01,0x08,0x0a,0x7e]
 
 v_cvt_f64_i32 v[5:6], s105
-// GFX12: v_cvt_f64_i32_e32 v[5:6], s105          ; encoding: [0x69,0x08,0x0a,0x7e]
+// GFX12: v_cvt_f64_i32_e32 v[5:6], s105 ; encoding: [0x69,0x08,0x0a,0x7e]
 
 v_cvt_f64_i32 v[5:6], vcc_lo
-// GFX12: v_cvt_f64_i32_e32 v[5:6], vcc_lo        ; encoding: [0x6a,0x08,0x0a,0x7e]
+// GFX12: v_cvt_f64_i32_e32 v[5:6], vcc_lo ; encoding: [0x6a,0x08,0x0a,0x7e]
 
 v_cvt_f64_i32 v[5:6], vcc_hi
-// GFX12: v_cvt_f64_i32_e32 v[5:6], vcc_hi        ; encoding: [0x6b,0x08,0x0a,0x7e]
+// GFX12: v_cvt_f64_i32_e32 v[5:6], vcc_hi ; encoding: [0x6b,0x08,0x0a,0x7e]
 
 v_cvt_f64_i32 v[5:6], ttmp15
-// GFX12: v_cvt_f64_i32_e32 v[5:6], ttmp15        ; encoding: [0x7b,0x08,0x0a,0x7e]
+// GFX12: v_cvt_f64_i32_e32 v[5:6], ttmp15 ; encoding: [0x7b,0x08,0x0a,0x7e]
 
 v_cvt_f64_i32 v[5:6], m0
-// GFX12: v_cvt_f64_i32_e32 v[5:6], m0            ; encoding: [0x7d,0x08,0x0a,0x7e]
+// GFX12: v_cvt_f64_i32_e32 v[5:6], m0 ; encoding: [0x7d,0x08,0x0a,0x7e]
 
 v_cvt_f64_i32 v[5:6], exec_lo
-// GFX12: v_cvt_f64_i32_e32 v[5:6], exec_lo       ; encoding: [0x7e,0x08,0x0a,0x7e]
+// GFX12: v_cvt_f64_i32_e32 v[5:6], exec_lo ; encoding: [0x7e,0x08,0x0a,0x7e]
 
 v_cvt_f64_i32 v[5:6], exec_hi
-// GFX12: v_cvt_f64_i32_e32 v[5:6], exec_hi       ; encoding: [0x7f,0x08,0x0a,0x7e]
+// GFX12: v_cvt_f64_i32_e32 v[5:6], exec_hi ; encoding: [0x7f,0x08,0x0a,0x7e]
 
 v_cvt_f64_i32 v[5:6], null
-// GFX12: v_cvt_f64_i32_e32 v[5:6], null          ; encoding: [0x7c,0x08,0x0a,0x7e]
+// GFX12: v_cvt_f64_i32_e32 v[5:6], null ; encoding: [0x7c,0x08,0x0a,0x7e]
 
 v_cvt_f64_i32 v[5:6], -1
-// GFX12: v_cvt_f64_i32_e32 v[5:6], -1            ; encoding: [0xc1,0x08,0x0a,0x7e]
+// GFX12: v_cvt_f64_i32_e32 v[5:6], -1 ; encoding: [0xc1,0x08,0x0a,0x7e]
 
 v_cvt_f64_i32 v[5:6], 0.5
-// GFX12: v_cvt_f64_i32_e32 v[5:6], 0.5           ; encoding: [0xf0,0x08,0x0a,0x7e]
+// GFX12: v_cvt_f64_i32_e32 v[5:6], 0.5 ; encoding: [0xf0,0x08,0x0a,0x7e]
 
 v_cvt_f64_i32 v[5:6], src_scc
-// GFX12: v_cvt_f64_i32_e32 v[5:6], src_scc       ; encoding: [0xfd,0x08,0x0a,0x7e]
+// GFX12: v_cvt_f64_i32_e32 v[5:6], src_scc ; encoding: [0xfd,0x08,0x0a,0x7e]
 
 v_cvt_f64_i32 v[254:255], 0xaf123456
 // GFX12: v_cvt_f64_i32_e32 v[254:255], 0xaf123456 ; encoding: [0xff,0x08,0xfc,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cvt_f64_u32 v[5:6], v1
-// GFX12: v_cvt_f64_u32_e32 v[5:6], v1            ; encoding: [0x01,0x2d,0x0a,0x7e]
+// GFX12: v_cvt_f64_u32_e32 v[5:6], v1 ; encoding: [0x01,0x2d,0x0a,0x7e]
 
 v_cvt_f64_u32 v[5:6], v255
-// GFX12: v_cvt_f64_u32_e32 v[5:6], v255          ; encoding: [0xff,0x2d,0x0a,0x7e]
+// GFX12: v_cvt_f64_u32_e32 v[5:6], v255 ; encoding: [0xff,0x2d,0x0a,0x7e]
 
 v_cvt_f64_u32 v[5:6], s1
-// GFX12: v_cvt_f64_u32_e32 v[5:6], s1            ; encoding: [0x01,0x2c,0x0a,0x7e]
+// GFX12: v_cvt_f64_u32_e32 v[5:6], s1 ; encoding: [0x01,0x2c,0x0a,0x7e]
 
 v_cvt_f64_u32 v[5:6], s105
-// GFX12: v_cvt_f64_u32_e32 v[5:6], s105          ; encoding: [0x69,0x2c,0x0a,0x7e]
+// GFX12: v_cvt_f64_u32_e32 v[5:6], s105 ; encoding: [0x69,0x2c,0x0a,0x7e]
 
 v_cvt_f64_u32 v[5:6], vcc_lo
-// GFX12: v_cvt_f64_u32_e32 v[5:6], vcc_lo        ; encoding: [0x6a,0x2c,0x0a,0x7e]
+// GFX12: v_cvt_f64_u32_e32 v[5:6], vcc_lo ; encoding: [0x6a,0x2c,0x0a,0x7e]
 
 v_cvt_f64_u32 v[5:6], vcc_hi
-// GFX12: v_cvt_f64_u32_e32 v[5:6], vcc_hi        ; encoding: [0x6b,0x2c,0x0a,0x7e]
+// GFX12: v_cvt_f64_u32_e32 v[5:6], vcc_hi ; encoding: [0x6b,0x2c,0x0a,0x7e]
 
 v_cvt_f64_u32 v[5:6], ttmp15
-// GFX12: v_cvt_f64_u32_e32 v[5:6], ttmp15        ; encoding: [0x7b,0x2c,0x0a,0x7e]
+// GFX12: v_cvt_f64_u32_e32 v[5:6], ttmp15 ; encoding: [0x7b,0x2c,0x0a,0x7e]
 
 v_cvt_f64_u32 v[5:6], m0
-// GFX12: v_cvt_f64_u32_e32 v[5:6], m0            ; encoding: [0x7d,0x2c,0x0a,0x7e]
+// GFX12: v_cvt_f64_u32_e32 v[5:6], m0 ; encoding: [0x7d,0x2c,0x0a,0x7e]
 
 v_cvt_f64_u32 v[5:6], exec_lo
-// GFX12: v_cvt_f64_u32_e32 v[5:6], exec_lo       ; encoding: [0x7e,0x2c,0x0a,0x7e]
+// GFX12: v_cvt_f64_u32_e32 v[5:6], exec_lo ; encoding: [0x7e,0x2c,0x0a,0x7e]
 
 v_cvt_f64_u32 v[5:6], exec_hi
-// GFX12: v_cvt_f64_u32_e32 v[5:6], exec_hi       ; encoding: [0x7f,0x2c,0x0a,0x7e]
+// GFX12: v_cvt_f64_u32_e32 v[5:6], exec_hi ; encoding: [0x7f,0x2c,0x0a,0x7e]
 
 v_cvt_f64_u32 v[5:6], null
-// GFX12: v_cvt_f64_u32_e32 v[5:6], null          ; encoding: [0x7c,0x2c,0x0a,0x7e]
+// GFX12: v_cvt_f64_u32_e32 v[5:6], null ; encoding: [0x7c,0x2c,0x0a,0x7e]
 
 v_cvt_f64_u32 v[5:6], -1
-// GFX12: v_cvt_f64_u32_e32 v[5:6], -1            ; encoding: [0xc1,0x2c,0x0a,0x7e]
+// GFX12: v_cvt_f64_u32_e32 v[5:6], -1 ; encoding: [0xc1,0x2c,0x0a,0x7e]
 
 v_cvt_f64_u32 v[5:6], 0.5
-// GFX12: v_cvt_f64_u32_e32 v[5:6], 0.5           ; encoding: [0xf0,0x2c,0x0a,0x7e]
+// GFX12: v_cvt_f64_u32_e32 v[5:6], 0.5 ; encoding: [0xf0,0x2c,0x0a,0x7e]
 
 v_cvt_f64_u32 v[5:6], src_scc
-// GFX12: v_cvt_f64_u32_e32 v[5:6], src_scc       ; encoding: [0xfd,0x2c,0x0a,0x7e]
+// GFX12: v_cvt_f64_u32_e32 v[5:6], src_scc ; encoding: [0xfd,0x2c,0x0a,0x7e]
 
 v_cvt_f64_u32 v[254:255], 0xaf123456
 // GFX12: v_cvt_f64_u32_e32 v[254:255], 0xaf123456 ; encoding: [0xff,0x2c,0xfc,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cvt_floor_i32_f32 v5, v1
-// GFX12: v_cvt_floor_i32_f32_e32 v5, v1          ; encoding: [0x01,0x1b,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, v1 ; encoding: [0x01,0x1b,0x0a,0x7e]
 
 v_cvt_floor_i32_f32 v5, v255
-// GFX12: v_cvt_floor_i32_f32_e32 v5, v255        ; encoding: [0xff,0x1b,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, v255 ; encoding: [0xff,0x1b,0x0a,0x7e]
 
 v_cvt_floor_i32_f32 v5, s1
-// GFX12: v_cvt_floor_i32_f32_e32 v5, s1          ; encoding: [0x01,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, s1 ; encoding: [0x01,0x1a,0x0a,0x7e]
 
 v_cvt_floor_i32_f32 v5, s105
-// GFX12: v_cvt_floor_i32_f32_e32 v5, s105        ; encoding: [0x69,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, s105 ; encoding: [0x69,0x1a,0x0a,0x7e]
 
 v_cvt_floor_i32_f32 v5, vcc_lo
-// GFX12: v_cvt_floor_i32_f32_e32 v5, vcc_lo      ; encoding: [0x6a,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, vcc_lo ; encoding: [0x6a,0x1a,0x0a,0x7e]
 
 v_cvt_floor_i32_f32 v5, vcc_hi
-// GFX12: v_cvt_floor_i32_f32_e32 v5, vcc_hi      ; encoding: [0x6b,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, vcc_hi ; encoding: [0x6b,0x1a,0x0a,0x7e]
 
 v_cvt_floor_i32_f32 v5, ttmp15
-// GFX12: v_cvt_floor_i32_f32_e32 v5, ttmp15      ; encoding: [0x7b,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, ttmp15 ; encoding: [0x7b,0x1a,0x0a,0x7e]
 
 v_cvt_floor_i32_f32 v5, m0
-// GFX12: v_cvt_floor_i32_f32_e32 v5, m0          ; encoding: [0x7d,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, m0 ; encoding: [0x7d,0x1a,0x0a,0x7e]
 
 v_cvt_floor_i32_f32 v5, exec_lo
-// GFX12: v_cvt_floor_i32_f32_e32 v5, exec_lo     ; encoding: [0x7e,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, exec_lo ; encoding: [0x7e,0x1a,0x0a,0x7e]
 
 v_cvt_floor_i32_f32 v5, exec_hi
-// GFX12: v_cvt_floor_i32_f32_e32 v5, exec_hi     ; encoding: [0x7f,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, exec_hi ; encoding: [0x7f,0x1a,0x0a,0x7e]
 
 v_cvt_floor_i32_f32 v5, null
-// GFX12: v_cvt_floor_i32_f32_e32 v5, null        ; encoding: [0x7c,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, null ; encoding: [0x7c,0x1a,0x0a,0x7e]
 
 v_cvt_floor_i32_f32 v5, -1
-// GFX12: v_cvt_floor_i32_f32_e32 v5, -1          ; encoding: [0xc1,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, -1 ; encoding: [0xc1,0x1a,0x0a,0x7e]
 
 v_cvt_floor_i32_f32 v5, 0.5
-// GFX12: v_cvt_floor_i32_f32_e32 v5, 0.5         ; encoding: [0xf0,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, 0.5 ; encoding: [0xf0,0x1a,0x0a,0x7e]
 
 v_cvt_floor_i32_f32 v5, src_scc
-// GFX12: v_cvt_floor_i32_f32_e32 v5, src_scc     ; encoding: [0xfd,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, src_scc ; encoding: [0xfd,0x1a,0x0a,0x7e]
 
 v_cvt_floor_i32_f32 v255, 0xaf123456
 // GFX12: v_cvt_floor_i32_f32_e32 v255, 0xaf123456 ; encoding: [0xff,0x1a,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cvt_flr_i32_f32 v5, v1
-// GFX12: v_cvt_floor_i32_f32_e32 v5, v1          ; encoding: [0x01,0x1b,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, v1 ; encoding: [0x01,0x1b,0x0a,0x7e]
 
 v_cvt_flr_i32_f32 v5, v255
-// GFX12: v_cvt_floor_i32_f32_e32 v5, v255        ; encoding: [0xff,0x1b,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, v255 ; encoding: [0xff,0x1b,0x0a,0x7e]
 
 v_cvt_flr_i32_f32 v5, s1
-// GFX12: v_cvt_floor_i32_f32_e32 v5, s1          ; encoding: [0x01,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, s1 ; encoding: [0x01,0x1a,0x0a,0x7e]
 
 v_cvt_flr_i32_f32 v5, s105
-// GFX12: v_cvt_floor_i32_f32_e32 v5, s105        ; encoding: [0x69,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, s105 ; encoding: [0x69,0x1a,0x0a,0x7e]
 
 v_cvt_flr_i32_f32 v5, vcc_lo
-// GFX12: v_cvt_floor_i32_f32_e32 v5, vcc_lo      ; encoding: [0x6a,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, vcc_lo ; encoding: [0x6a,0x1a,0x0a,0x7e]
 
 v_cvt_flr_i32_f32 v5, vcc_hi
-// GFX12: v_cvt_floor_i32_f32_e32 v5, vcc_hi      ; encoding: [0x6b,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, vcc_hi ; encoding: [0x6b,0x1a,0x0a,0x7e]
 
 v_cvt_flr_i32_f32 v5, ttmp15
-// GFX12: v_cvt_floor_i32_f32_e32 v5, ttmp15      ; encoding: [0x7b,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, ttmp15 ; encoding: [0x7b,0x1a,0x0a,0x7e]
 
 v_cvt_flr_i32_f32 v5, m0
-// GFX12: v_cvt_floor_i32_f32_e32 v5, m0          ; encoding: [0x7d,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, m0 ; encoding: [0x7d,0x1a,0x0a,0x7e]
 
 v_cvt_flr_i32_f32 v5, exec_lo
-// GFX12: v_cvt_floor_i32_f32_e32 v5, exec_lo     ; encoding: [0x7e,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, exec_lo ; encoding: [0x7e,0x1a,0x0a,0x7e]
 
 v_cvt_flr_i32_f32 v5, exec_hi
-// GFX12: v_cvt_floor_i32_f32_e32 v5, exec_hi     ; encoding: [0x7f,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, exec_hi ; encoding: [0x7f,0x1a,0x0a,0x7e]
 
 v_cvt_flr_i32_f32 v5, null
-// GFX12: v_cvt_floor_i32_f32_e32 v5, null        ; encoding: [0x7c,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, null ; encoding: [0x7c,0x1a,0x0a,0x7e]
 
 v_cvt_flr_i32_f32 v5, -1
-// GFX12: v_cvt_floor_i32_f32_e32 v5, -1          ; encoding: [0xc1,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, -1 ; encoding: [0xc1,0x1a,0x0a,0x7e]
 
 v_cvt_flr_i32_f32 v5, 0.5
-// GFX12: v_cvt_floor_i32_f32_e32 v5, 0.5         ; encoding: [0xf0,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, 0.5 ; encoding: [0xf0,0x1a,0x0a,0x7e]
 
 v_cvt_flr_i32_f32 v5, src_scc
-// GFX12: v_cvt_floor_i32_f32_e32 v5, src_scc     ; encoding: [0xfd,0x1a,0x0a,0x7e]
+// GFX12: v_cvt_floor_i32_f32_e32 v5, src_scc ; encoding: [0xfd,0x1a,0x0a,0x7e]
 
 v_cvt_flr_i32_f32 v255, 0xaf123456
 // GFX12: v_cvt_floor_i32_f32_e32 v255, 0xaf123456 ; encoding: [0xff,0x1a,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cvt_i16_f16 v5, v1
-// GFX12: v_cvt_i16_f16_e32 v5, v1                ; encoding: [0x01,0xa7,0x0a,0x7e]
+// GFX12: v_cvt_i16_f16_e32 v5, v1 ; encoding: [0x01,0xa7,0x0a,0x7e]
 
 v_cvt_i16_f16 v5, v127
-// GFX12: v_cvt_i16_f16_e32 v5, v127              ; encoding: [0x7f,0xa7,0x0a,0x7e]
+// GFX12: v_cvt_i16_f16_e32 v5, v127 ; encoding: [0x7f,0xa7,0x0a,0x7e]
 
 v_cvt_i16_f16 v5, s1
-// GFX12: v_cvt_i16_f16_e32 v5, s1                ; encoding: [0x01,0xa6,0x0a,0x7e]
+// GFX12: v_cvt_i16_f16_e32 v5, s1 ; encoding: [0x01,0xa6,0x0a,0x7e]
 
 v_cvt_i16_f16 v5, s105
-// GFX12: v_cvt_i16_f16_e32 v5, s105              ; encoding: [0x69,0xa6,0x0a,0x7e]
+// GFX12: v_cvt_i16_f16_e32 v5, s105 ; encoding: [0x69,0xa6,0x0a,0x7e]
 
 v_cvt_i16_f16 v5, vcc_lo
-// GFX12: v_cvt_i16_f16_e32 v5, vcc_lo            ; encoding: [0x6a,0xa6,0x0a,0x7e]
+// GFX12: v_cvt_i16_f16_e32 v5, vcc_lo ; encoding: [0x6a,0xa6,0x0a,0x7e]
 
 v_cvt_i16_f16 v5, vcc_hi
-// GFX12: v_cvt_i16_f16_e32 v5, vcc_hi            ; encoding: [0x6b,0xa6,0x0a,0x7e]
+// GFX12: v_cvt_i16_f16_e32 v5, vcc_hi ; encoding: [0x6b,0xa6,0x0a,0x7e]
 
 v_cvt_i16_f16 v5, ttmp15
-// GFX12: v_cvt_i16_f16_e32 v5, ttmp15            ; encoding: [0x7b,0xa6,0x0a,0x7e]
+// GFX12: v_cvt_i16_f16_e32 v5, ttmp15 ; encoding: [0x7b,0xa6,0x0a,0x7e]
 
 v_cvt_i16_f16 v5, m0
-// GFX12: v_cvt_i16_f16_e32 v5, m0                ; encoding: [0x7d,0xa6,0x0a,0x7e]
+// GFX12: v_cvt_i16_f16_e32 v5, m0 ; encoding: [0x7d,0xa6,0x0a,0x7e]
 
 v_cvt_i16_f16 v5, exec_lo
-// GFX12: v_cvt_i16_f16_e32 v5, exec_lo           ; encoding: [0x7e,0xa6,0x0a,0x7e]
+// GFX12: v_cvt_i16_f16_e32 v5, exec_lo ; encoding: [0x7e,0xa6,0x0a,0x7e]
 
 v_cvt_i16_f16 v5, exec_hi
-// GFX12: v_cvt_i16_f16_e32 v5, exec_hi           ; encoding: [0x7f,0xa6,0x0a,0x7e]
+// GFX12: v_cvt_i16_f16_e32 v5, exec_hi ; encoding: [0x7f,0xa6,0x0a,0x7e]
 
 v_cvt_i16_f16 v5, null
-// GFX12: v_cvt_i16_f16_e32 v5, null              ; encoding: [0x7c,0xa6,0x0a,0x7e]
+// GFX12: v_cvt_i16_f16_e32 v5, null ; encoding: [0x7c,0xa6,0x0a,0x7e]
 
 v_cvt_i16_f16 v5, -1
-// GFX12: v_cvt_i16_f16_e32 v5, -1                ; encoding: [0xc1,0xa6,0x0a,0x7e]
+// GFX12: v_cvt_i16_f16_e32 v5, -1 ; encoding: [0xc1,0xa6,0x0a,0x7e]
 
 v_cvt_i16_f16 v5, 0.5
-// GFX12: v_cvt_i16_f16_e32 v5, 0.5               ; encoding: [0xf0,0xa6,0x0a,0x7e]
+// GFX12: v_cvt_i16_f16_e32 v5, 0.5 ; encoding: [0xf0,0xa6,0x0a,0x7e]
 
 v_cvt_i16_f16 v5, src_scc
-// GFX12: v_cvt_i16_f16_e32 v5, src_scc           ; encoding: [0xfd,0xa6,0x0a,0x7e]
+// GFX12: v_cvt_i16_f16_e32 v5, src_scc ; encoding: [0xfd,0xa6,0x0a,0x7e]
 
 v_cvt_i16_f16 v127, 0xfe0b
-// GFX12: v_cvt_i16_f16_e32 v127, 0xfe0b          ; encoding: [0xff,0xa6,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_cvt_i16_f16_e32 v127, 0xfe0b ; encoding: [0xff,0xa6,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_cvt_i32_f32 v5, v1
-// GFX12: v_cvt_i32_f32_e32 v5, v1                ; encoding: [0x01,0x11,0x0a,0x7e]
+// GFX12: v_cvt_i32_f32_e32 v5, v1 ; encoding: [0x01,0x11,0x0a,0x7e]
 
 v_cvt_i32_f32 v5, v255
-// GFX12: v_cvt_i32_f32_e32 v5, v255              ; encoding: [0xff,0x11,0x0a,0x7e]
+// GFX12: v_cvt_i32_f32_e32 v5, v255 ; encoding: [0xff,0x11,0x0a,0x7e]
 
 v_cvt_i32_f32 v5, s1
-// GFX12: v_cvt_i32_f32_e32 v5, s1                ; encoding: [0x01,0x10,0x0a,0x7e]
+// GFX12: v_cvt_i32_f32_e32 v5, s1 ; encoding: [0x01,0x10,0x0a,0x7e]
 
 v_cvt_i32_f32 v5, s105
-// GFX12: v_cvt_i32_f32_e32 v5, s105              ; encoding: [0x69,0x10,0x0a,0x7e]
+// GFX12: v_cvt_i32_f32_e32 v5, s105 ; encoding: [0x69,0x10,0x0a,0x7e]
 
 v_cvt_i32_f32 v5, vcc_lo
-// GFX12: v_cvt_i32_f32_e32 v5, vcc_lo            ; encoding: [0x6a,0x10,0x0a,0x7e]
+// GFX12: v_cvt_i32_f32_e32 v5, vcc_lo ; encoding: [0x6a,0x10,0x0a,0x7e]
 
 v_cvt_i32_f32 v5, vcc_hi
-// GFX12: v_cvt_i32_f32_e32 v5, vcc_hi            ; encoding: [0x6b,0x10,0x0a,0x7e]
+// GFX12: v_cvt_i32_f32_e32 v5, vcc_hi ; encoding: [0x6b,0x10,0x0a,0x7e]
 
 v_cvt_i32_f32 v5, ttmp15
-// GFX12: v_cvt_i32_f32_e32 v5, ttmp15            ; encoding: [0x7b,0x10,0x0a,0x7e]
+// GFX12: v_cvt_i32_f32_e32 v5, ttmp15 ; encoding: [0x7b,0x10,0x0a,0x7e]
 
 v_cvt_i32_f32 v5, m0
-// GFX12: v_cvt_i32_f32_e32 v5, m0                ; encoding: [0x7d,0x10,0x0a,0x7e]
+// GFX12: v_cvt_i32_f32_e32 v5, m0 ; encoding: [0x7d,0x10,0x0a,0x7e]
 
 v_cvt_i32_f32 v5, exec_lo
-// GFX12: v_cvt_i32_f32_e32 v5, exec_lo           ; encoding: [0x7e,0x10,0x0a,0x7e]
+// GFX12: v_cvt_i32_f32_e32 v5, exec_lo ; encoding: [0x7e,0x10,0x0a,0x7e]
 
 v_cvt_i32_f32 v5, exec_hi
-// GFX12: v_cvt_i32_f32_e32 v5, exec_hi           ; encoding: [0x7f,0x10,0x0a,0x7e]
+// GFX12: v_cvt_i32_f32_e32 v5, exec_hi ; encoding: [0x7f,0x10,0x0a,0x7e]
 
 v_cvt_i32_f32 v5, null
-// GFX12: v_cvt_i32_f32_e32 v5, null              ; encoding: [0x7c,0x10,0x0a,0x7e]
+// GFX12: v_cvt_i32_f32_e32 v5, null ; encoding: [0x7c,0x10,0x0a,0x7e]
 
 v_cvt_i32_f32 v5, -1
-// GFX12: v_cvt_i32_f32_e32 v5, -1                ; encoding: [0xc1,0x10,0x0a,0x7e]
+// GFX12: v_cvt_i32_f32_e32 v5, -1 ; encoding: [0xc1,0x10,0x0a,0x7e]
 
 v_cvt_i32_f32 v5, 0.5
-// GFX12: v_cvt_i32_f32_e32 v5, 0.5               ; encoding: [0xf0,0x10,0x0a,0x7e]
+// GFX12: v_cvt_i32_f32_e32 v5, 0.5 ; encoding: [0xf0,0x10,0x0a,0x7e]
 
 v_cvt_i32_f32 v5, src_scc
-// GFX12: v_cvt_i32_f32_e32 v5, src_scc           ; encoding: [0xfd,0x10,0x0a,0x7e]
+// GFX12: v_cvt_i32_f32_e32 v5, src_scc ; encoding: [0xfd,0x10,0x0a,0x7e]
 
 v_cvt_i32_f32 v255, 0xaf123456
-// GFX12: v_cvt_i32_f32_e32 v255, 0xaf123456      ; encoding: [0xff,0x10,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_cvt_i32_f32_e32 v255, 0xaf123456 ; encoding: [0xff,0x10,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cvt_i32_f64 v5, v[1:2]
-// GFX12: v_cvt_i32_f64_e32 v5, v[1:2]            ; encoding: [0x01,0x07,0x0a,0x7e]
+// GFX12: v_cvt_i32_f64_e32 v5, v[1:2] ; encoding: [0x01,0x07,0x0a,0x7e]
 
 v_cvt_i32_f64 v5, v[254:255]
-// GFX12: v_cvt_i32_f64_e32 v5, v[254:255]        ; encoding: [0xfe,0x07,0x0a,0x7e]
+// GFX12: v_cvt_i32_f64_e32 v5, v[254:255] ; encoding: [0xfe,0x07,0x0a,0x7e]
 
 v_cvt_i32_f64 v5, s[2:3]
-// GFX12: v_cvt_i32_f64_e32 v5, s[2:3]            ; encoding: [0x02,0x06,0x0a,0x7e]
+// GFX12: v_cvt_i32_f64_e32 v5, s[2:3] ; encoding: [0x02,0x06,0x0a,0x7e]
 
 v_cvt_i32_f64 v5, s[104:105]
-// GFX12: v_cvt_i32_f64_e32 v5, s[104:105]        ; encoding: [0x68,0x06,0x0a,0x7e]
+// GFX12: v_cvt_i32_f64_e32 v5, s[104:105] ; encoding: [0x68,0x06,0x0a,0x7e]
 
 v_cvt_i32_f64 v5, vcc
-// GFX12: v_cvt_i32_f64_e32 v5, vcc               ; encoding: [0x6a,0x06,0x0a,0x7e]
+// GFX12: v_cvt_i32_f64_e32 v5, vcc ; encoding: [0x6a,0x06,0x0a,0x7e]
 
 v_cvt_i32_f64 v5, ttmp[14:15]
-// GFX12: v_cvt_i32_f64_e32 v5, ttmp[14:15]       ; encoding: [0x7a,0x06,0x0a,0x7e]
+// GFX12: v_cvt_i32_f64_e32 v5, ttmp[14:15] ; encoding: [0x7a,0x06,0x0a,0x7e]
 
 v_cvt_i32_f64 v5, exec
-// GFX12: v_cvt_i32_f64_e32 v5, exec              ; encoding: [0x7e,0x06,0x0a,0x7e]
+// GFX12: v_cvt_i32_f64_e32 v5, exec ; encoding: [0x7e,0x06,0x0a,0x7e]
 
 v_cvt_i32_f64 v5, null
-// GFX12: v_cvt_i32_f64_e32 v5, null              ; encoding: [0x7c,0x06,0x0a,0x7e]
+// GFX12: v_cvt_i32_f64_e32 v5, null ; encoding: [0x7c,0x06,0x0a,0x7e]
 
 v_cvt_i32_f64 v5, -1
-// GFX12: v_cvt_i32_f64_e32 v5, -1                ; encoding: [0xc1,0x06,0x0a,0x7e]
+// GFX12: v_cvt_i32_f64_e32 v5, -1 ; encoding: [0xc1,0x06,0x0a,0x7e]
 
 v_cvt_i32_f64 v5, 0.5
-// GFX12: v_cvt_i32_f64_e32 v5, 0.5               ; encoding: [0xf0,0x06,0x0a,0x7e]
+// GFX12: v_cvt_i32_f64_e32 v5, 0.5 ; encoding: [0xf0,0x06,0x0a,0x7e]
 
 v_cvt_i32_f64 v5, src_scc
-// GFX12: v_cvt_i32_f64_e32 v5, src_scc           ; encoding: [0xfd,0x06,0x0a,0x7e]
+// GFX12: v_cvt_i32_f64_e32 v5, src_scc ; encoding: [0xfd,0x06,0x0a,0x7e]
 
 v_cvt_i32_f64 v255, 0xaf123456
-// GFX12: v_cvt_i32_f64_e32 v255, 0xaf123456      ; encoding: [0xff,0x06,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_cvt_i32_f64_e32 v255, 0xaf123456 ; encoding: [0xff,0x06,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cvt_i32_i16 v5, v1
-// GFX12: v_cvt_i32_i16_e32 v5, v1                ; encoding: [0x01,0xd5,0x0a,0x7e]
+// GFX12: v_cvt_i32_i16_e32 v5, v1 ; encoding: [0x01,0xd5,0x0a,0x7e]
 
 v_cvt_i32_i16 v5, v127
-// GFX12: v_cvt_i32_i16_e32 v5, v127              ; encoding: [0x7f,0xd5,0x0a,0x7e]
+// GFX12: v_cvt_i32_i16_e32 v5, v127 ; encoding: [0x7f,0xd5,0x0a,0x7e]
 
 v_cvt_i32_i16 v5, s1
-// GFX12: v_cvt_i32_i16_e32 v5, s1                ; encoding: [0x01,0xd4,0x0a,0x7e]
+// GFX12: v_cvt_i32_i16_e32 v5, s1 ; encoding: [0x01,0xd4,0x0a,0x7e]
 
 v_cvt_i32_i16 v5, s105
-// GFX12: v_cvt_i32_i16_e32 v5, s105              ; encoding: [0x69,0xd4,0x0a,0x7e]
+// GFX12: v_cvt_i32_i16_e32 v5, s105 ; encoding: [0x69,0xd4,0x0a,0x7e]
 
 v_cvt_i32_i16 v5, vcc_lo
-// GFX12: v_cvt_i32_i16_e32 v5, vcc_lo            ; encoding: [0x6a,0xd4,0x0a,0x7e]
+// GFX12: v_cvt_i32_i16_e32 v5, vcc_lo ; encoding: [0x6a,0xd4,0x0a,0x7e]
 
 v_cvt_i32_i16 v5, vcc_hi
-// GFX12: v_cvt_i32_i16_e32 v5, vcc_hi            ; encoding: [0x6b,0xd4,0x0a,0x7e]
+// GFX12: v_cvt_i32_i16_e32 v5, vcc_hi ; encoding: [0x6b,0xd4,0x0a,0x7e]
 
 v_cvt_i32_i16 v5, ttmp15
-// GFX12: v_cvt_i32_i16_e32 v5, ttmp15            ; encoding: [0x7b,0xd4,0x0a,0x7e]
+// GFX12: v_cvt_i32_i16_e32 v5, ttmp15 ; encoding: [0x7b,0xd4,0x0a,0x7e]
 
 v_cvt_i32_i16 v5, m0
-// GFX12: v_cvt_i32_i16_e32 v5, m0                ; encoding: [0x7d,0xd4,0x0a,0x7e]
+// GFX12: v_cvt_i32_i16_e32 v5, m0 ; encoding: [0x7d,0xd4,0x0a,0x7e]
 
 v_cvt_i32_i16 v5, exec_lo
-// GFX12: v_cvt_i32_i16_e32 v5, exec_lo           ; encoding: [0x7e,0xd4,0x0a,0x7e]
+// GFX12: v_cvt_i32_i16_e32 v5, exec_lo ; encoding: [0x7e,0xd4,0x0a,0x7e]
 
 v_cvt_i32_i16 v5, exec_hi
-// GFX12: v_cvt_i32_i16_e32 v5, exec_hi           ; encoding: [0x7f,0xd4,0x0a,0x7e]
+// GFX12: v_cvt_i32_i16_e32 v5, exec_hi ; encoding: [0x7f,0xd4,0x0a,0x7e]
 
 v_cvt_i32_i16 v5, null
-// GFX12: v_cvt_i32_i16_e32 v5, null              ; encoding: [0x7c,0xd4,0x0a,0x7e]
+// GFX12: v_cvt_i32_i16_e32 v5, null ; encoding: [0x7c,0xd4,0x0a,0x7e]
 
 v_cvt_i32_i16 v5, -1
-// GFX12: v_cvt_i32_i16_e32 v5, -1                ; encoding: [0xc1,0xd4,0x0a,0x7e]
+// GFX12: v_cvt_i32_i16_e32 v5, -1 ; encoding: [0xc1,0xd4,0x0a,0x7e]
 
 v_cvt_i32_i16 v5, 0.5
-// GFX12-ASM: v_cvt_i32_i16_e32 v5, 0.5               ; encoding: [0xf0,0xd4,0x0a,0x7e]
-// GFX12-DIS: v_cvt_i32_i16_e32 v5, 0x3800            ; encoding: [0xff,0xd4,0x0a,0x7e,0x00,0x38,0x00,0x00]
+// GFX12-ASM: v_cvt_i32_i16_e32 v5, 0.5 ; encoding: [0xf0,0xd4,0x0a,0x7e]
+// GFX12-DIS: v_cvt_i32_i16_e32 v5, 0x3800 ; encoding: [0xff,0xd4,0x0a,0x7e,0x00,0x38,0x00,0x00]
 
 v_cvt_i32_i16 v5, src_scc
-// GFX12: v_cvt_i32_i16_e32 v5, src_scc           ; encoding: [0xfd,0xd4,0x0a,0x7e]
+// GFX12: v_cvt_i32_i16_e32 v5, src_scc ; encoding: [0xfd,0xd4,0x0a,0x7e]
 
 v_cvt_i32_i16 v255, 0xfe0b
-// GFX12: v_cvt_i32_i16_e32 v255, 0xfe0b          ; encoding: [0xff,0xd4,0xfe,0x7f,0x0b,0xfe,0x00,0x00]
+// GFX12: v_cvt_i32_i16_e32 v255, 0xfe0b ; encoding: [0xff,0xd4,0xfe,0x7f,0x0b,0xfe,0x00,0x00]
 
 v_cvt_nearest_i32_f32 v5, v1
-// GFX12: v_cvt_nearest_i32_f32_e32 v5, v1        ; encoding: [0x01,0x19,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, v1 ; encoding: [0x01,0x19,0x0a,0x7e]
 
 v_cvt_nearest_i32_f32 v5, v255
-// GFX12: v_cvt_nearest_i32_f32_e32 v5, v255      ; encoding: [0xff,0x19,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, v255 ; encoding: [0xff,0x19,0x0a,0x7e]
 
 v_cvt_nearest_i32_f32 v5, s1
-// GFX12: v_cvt_nearest_i32_f32_e32 v5, s1        ; encoding: [0x01,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, s1 ; encoding: [0x01,0x18,0x0a,0x7e]
 
 v_cvt_nearest_i32_f32 v5, s105
-// GFX12: v_cvt_nearest_i32_f32_e32 v5, s105      ; encoding: [0x69,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, s105 ; encoding: [0x69,0x18,0x0a,0x7e]
 
 v_cvt_nearest_i32_f32 v5, vcc_lo
-// GFX12: v_cvt_nearest_i32_f32_e32 v5, vcc_lo    ; encoding: [0x6a,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, vcc_lo ; encoding: [0x6a,0x18,0x0a,0x7e]
 
 v_cvt_nearest_i32_f32 v5, vcc_hi
-// GFX12: v_cvt_nearest_i32_f32_e32 v5, vcc_hi    ; encoding: [0x6b,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, vcc_hi ; encoding: [0x6b,0x18,0x0a,0x7e]
 
 v_cvt_nearest_i32_f32 v5, ttmp15
-// GFX12: v_cvt_nearest_i32_f32_e32 v5, ttmp15    ; encoding: [0x7b,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, ttmp15 ; encoding: [0x7b,0x18,0x0a,0x7e]
 
 v_cvt_nearest_i32_f32 v5, m0
-// GFX12: v_cvt_nearest_i32_f32_e32 v5, m0        ; encoding: [0x7d,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, m0 ; encoding: [0x7d,0x18,0x0a,0x7e]
 
 v_cvt_nearest_i32_f32 v5, exec_lo
-// GFX12: v_cvt_nearest_i32_f32_e32 v5, exec_lo   ; encoding: [0x7e,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, exec_lo ; encoding: [0x7e,0x18,0x0a,0x7e]
 
 v_cvt_nearest_i32_f32 v5, exec_hi
-// GFX12: v_cvt_nearest_i32_f32_e32 v5, exec_hi   ; encoding: [0x7f,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, exec_hi ; encoding: [0x7f,0x18,0x0a,0x7e]
 
 v_cvt_nearest_i32_f32 v5, null
-// GFX12: v_cvt_nearest_i32_f32_e32 v5, null      ; encoding: [0x7c,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, null ; encoding: [0x7c,0x18,0x0a,0x7e]
 
 v_cvt_nearest_i32_f32 v5, -1
-// GFX12: v_cvt_nearest_i32_f32_e32 v5, -1        ; encoding: [0xc1,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, -1 ; encoding: [0xc1,0x18,0x0a,0x7e]
 
 v_cvt_nearest_i32_f32 v5, 0.5
-// GFX12: v_cvt_nearest_i32_f32_e32 v5, 0.5       ; encoding: [0xf0,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, 0.5 ; encoding: [0xf0,0x18,0x0a,0x7e]
 
 v_cvt_nearest_i32_f32 v5, src_scc
-// GFX12: v_cvt_nearest_i32_f32_e32 v5, src_scc   ; encoding: [0xfd,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, src_scc ; encoding: [0xfd,0x18,0x0a,0x7e]
 
 v_cvt_nearest_i32_f32 v255, 0xaf123456
 // GFX12: v_cvt_nearest_i32_f32_e32 v255, 0xaf123456 ; encoding: [0xff,0x18,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cvt_norm_i16_f16 v5, v1
-// GFX12: v_cvt_norm_i16_f16_e32 v5, v1           ; encoding: [0x01,0xc7,0x0a,0x7e]
+// GFX12: v_cvt_norm_i16_f16_e32 v5, v1 ; encoding: [0x01,0xc7,0x0a,0x7e]
 
 v_cvt_norm_i16_f16 v5, v127
-// GFX12: v_cvt_norm_i16_f16_e32 v5, v127         ; encoding: [0x7f,0xc7,0x0a,0x7e]
+// GFX12: v_cvt_norm_i16_f16_e32 v5, v127 ; encoding: [0x7f,0xc7,0x0a,0x7e]
 
 v_cvt_norm_i16_f16 v5, s1
-// GFX12: v_cvt_norm_i16_f16_e32 v5, s1           ; encoding: [0x01,0xc6,0x0a,0x7e]
+// GFX12: v_cvt_norm_i16_f16_e32 v5, s1 ; encoding: [0x01,0xc6,0x0a,0x7e]
 
 v_cvt_norm_i16_f16 v5, s105
-// GFX12: v_cvt_norm_i16_f16_e32 v5, s105         ; encoding: [0x69,0xc6,0x0a,0x7e]
+// GFX12: v_cvt_norm_i16_f16_e32 v5, s105 ; encoding: [0x69,0xc6,0x0a,0x7e]
 
 v_cvt_norm_i16_f16 v5, vcc_lo
-// GFX12: v_cvt_norm_i16_f16_e32 v5, vcc_lo       ; encoding: [0x6a,0xc6,0x0a,0x7e]
+// GFX12: v_cvt_norm_i16_f16_e32 v5, vcc_lo ; encoding: [0x6a,0xc6,0x0a,0x7e]
 
 v_cvt_norm_i16_f16 v5, vcc_hi
-// GFX12: v_cvt_norm_i16_f16_e32 v5, vcc_hi       ; encoding: [0x6b,0xc6,0x0a,0x7e]
+// GFX12: v_cvt_norm_i16_f16_e32 v5, vcc_hi ; encoding: [0x6b,0xc6,0x0a,0x7e]
 
 v_cvt_norm_i16_f16 v5, ttmp15
-// GFX12: v_cvt_norm_i16_f16_e32 v5, ttmp15       ; encoding: [0x7b,0xc6,0x0a,0x7e]
+// GFX12: v_cvt_norm_i16_f16_e32 v5, ttmp15 ; encoding: [0x7b,0xc6,0x0a,0x7e]
 
 v_cvt_norm_i16_f16 v5, m0
-// GFX12: v_cvt_norm_i16_f16_e32 v5, m0           ; encoding: [0x7d,0xc6,0x0a,0x7e]
+// GFX12: v_cvt_norm_i16_f16_e32 v5, m0 ; encoding: [0x7d,0xc6,0x0a,0x7e]
 
 v_cvt_norm_i16_f16 v5, exec_lo
-// GFX12: v_cvt_norm_i16_f16_e32 v5, exec_lo      ; encoding: [0x7e,0xc6,0x0a,0x7e]
+// GFX12: v_cvt_norm_i16_f16_e32 v5, exec_lo ; encoding: [0x7e,0xc6,0x0a,0x7e]
 
 v_cvt_norm_i16_f16 v5, exec_hi
-// GFX12: v_cvt_norm_i16_f16_e32 v5, exec_hi      ; encoding: [0x7f,0xc6,0x0a,0x7e]
+// GFX12: v_cvt_norm_i16_f16_e32 v5, exec_hi ; encoding: [0x7f,0xc6,0x0a,0x7e]
 
 v_cvt_norm_i16_f16 v5, null
-// GFX12: v_cvt_norm_i16_f16_e32 v5, null         ; encoding: [0x7c,0xc6,0x0a,0x7e]
+// GFX12: v_cvt_norm_i16_f16_e32 v5, null ; encoding: [0x7c,0xc6,0x0a,0x7e]
 
 v_cvt_norm_i16_f16 v5, -1
-// GFX12: v_cvt_norm_i16_f16_e32 v5, -1           ; encoding: [0xc1,0xc6,0x0a,0x7e]
+// GFX12: v_cvt_norm_i16_f16_e32 v5, -1 ; encoding: [0xc1,0xc6,0x0a,0x7e]
 
 v_cvt_norm_i16_f16 v5, 0.5
-// GFX12: v_cvt_norm_i16_f16_e32 v5, 0.5          ; encoding: [0xf0,0xc6,0x0a,0x7e]
+// GFX12: v_cvt_norm_i16_f16_e32 v5, 0.5 ; encoding: [0xf0,0xc6,0x0a,0x7e]
 
 v_cvt_norm_i16_f16 v5, src_scc
-// GFX12: v_cvt_norm_i16_f16_e32 v5, src_scc      ; encoding: [0xfd,0xc6,0x0a,0x7e]
+// GFX12: v_cvt_norm_i16_f16_e32 v5, src_scc ; encoding: [0xfd,0xc6,0x0a,0x7e]
 
 v_cvt_norm_i16_f16 v127, 0xfe0b
-// GFX12: v_cvt_norm_i16_f16_e32 v127, 0xfe0b     ; encoding: [0xff,0xc6,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_cvt_norm_i16_f16_e32 v127, 0xfe0b ; encoding: [0xff,0xc6,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_cvt_norm_u16_f16 v5, v1
-// GFX12: v_cvt_norm_u16_f16_e32 v5, v1           ; encoding: [0x01,0xc9,0x0a,0x7e]
+// GFX12: v_cvt_norm_u16_f16_e32 v5, v1 ; encoding: [0x01,0xc9,0x0a,0x7e]
 
 v_cvt_norm_u16_f16 v5, v127
-// GFX12: v_cvt_norm_u16_f16_e32 v5, v127         ; encoding: [0x7f,0xc9,0x0a,0x7e]
+// GFX12: v_cvt_norm_u16_f16_e32 v5, v127 ; encoding: [0x7f,0xc9,0x0a,0x7e]
 
 v_cvt_norm_u16_f16 v5, s1
-// GFX12: v_cvt_norm_u16_f16_e32 v5, s1           ; encoding: [0x01,0xc8,0x0a,0x7e]
+// GFX12: v_cvt_norm_u16_f16_e32 v5, s1 ; encoding: [0x01,0xc8,0x0a,0x7e]
 
 v_cvt_norm_u16_f16 v5, s105
-// GFX12: v_cvt_norm_u16_f16_e32 v5, s105         ; encoding: [0x69,0xc8,0x0a,0x7e]
+// GFX12: v_cvt_norm_u16_f16_e32 v5, s105 ; encoding: [0x69,0xc8,0x0a,0x7e]
 
 v_cvt_norm_u16_f16 v5, vcc_lo
-// GFX12: v_cvt_norm_u16_f16_e32 v5, vcc_lo       ; encoding: [0x6a,0xc8,0x0a,0x7e]
+// GFX12: v_cvt_norm_u16_f16_e32 v5, vcc_lo ; encoding: [0x6a,0xc8,0x0a,0x7e]
 
 v_cvt_norm_u16_f16 v5, vcc_hi
-// GFX12: v_cvt_norm_u16_f16_e32 v5, vcc_hi       ; encoding: [0x6b,0xc8,0x0a,0x7e]
+// GFX12: v_cvt_norm_u16_f16_e32 v5, vcc_hi ; encoding: [0x6b,0xc8,0x0a,0x7e]
 
 v_cvt_norm_u16_f16 v5, ttmp15
-// GFX12: v_cvt_norm_u16_f16_e32 v5, ttmp15       ; encoding: [0x7b,0xc8,0x0a,0x7e]
+// GFX12: v_cvt_norm_u16_f16_e32 v5, ttmp15 ; encoding: [0x7b,0xc8,0x0a,0x7e]
 
 v_cvt_norm_u16_f16 v5, m0
-// GFX12: v_cvt_norm_u16_f16_e32 v5, m0           ; encoding: [0x7d,0xc8,0x0a,0x7e]
+// GFX12: v_cvt_norm_u16_f16_e32 v5, m0 ; encoding: [0x7d,0xc8,0x0a,0x7e]
 
 v_cvt_norm_u16_f16 v5, exec_lo
-// GFX12: v_cvt_norm_u16_f16_e32 v5, exec_lo      ; encoding: [0x7e,0xc8,0x0a,0x7e]
+// GFX12: v_cvt_norm_u16_f16_e32 v5, exec_lo ; encoding: [0x7e,0xc8,0x0a,0x7e]
 
 v_cvt_norm_u16_f16 v5, exec_hi
-// GFX12: v_cvt_norm_u16_f16_e32 v5, exec_hi      ; encoding: [0x7f,0xc8,0x0a,0x7e]
+// GFX12: v_cvt_norm_u16_f16_e32 v5, exec_hi ; encoding: [0x7f,0xc8,0x0a,0x7e]
 
 v_cvt_norm_u16_f16 v5, null
-// GFX12: v_cvt_norm_u16_f16_e32 v5, null         ; encoding: [0x7c,0xc8,0x0a,0x7e]
+// GFX12: v_cvt_norm_u16_f16_e32 v5, null ; encoding: [0x7c,0xc8,0x0a,0x7e]
 
 v_cvt_norm_u16_f16 v5, -1
-// GFX12: v_cvt_norm_u16_f16_e32 v5, -1           ; encoding: [0xc1,0xc8,0x0a,0x7e]
+// GFX12: v_cvt_norm_u16_f16_e32 v5, -1 ; encoding: [0xc1,0xc8,0x0a,0x7e]
 
 v_cvt_norm_u16_f16 v5, 0.5
-// GFX12: v_cvt_norm_u16_f16_e32 v5, 0.5          ; encoding: [0xf0,0xc8,0x0a,0x7e]
+// GFX12: v_cvt_norm_u16_f16_e32 v5, 0.5 ; encoding: [0xf0,0xc8,0x0a,0x7e]
 
 v_cvt_norm_u16_f16 v5, src_scc
-// GFX12: v_cvt_norm_u16_f16_e32 v5, src_scc      ; encoding: [0xfd,0xc8,0x0a,0x7e]
+// GFX12: v_cvt_norm_u16_f16_e32 v5, src_scc ; encoding: [0xfd,0xc8,0x0a,0x7e]
 
 v_cvt_norm_u16_f16 v127, 0xfe0b
-// GFX12: v_cvt_norm_u16_f16_e32 v127, 0xfe0b     ; encoding: [0xff,0xc8,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_cvt_norm_u16_f16_e32 v127, 0xfe0b ; encoding: [0xff,0xc8,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_cvt_off_f32_i4 v5, v1
-// GFX12: v_cvt_off_f32_i4_e32 v5, v1             ; encoding: [0x01,0x1d,0x0a,0x7e]
+// GFX12: v_cvt_off_f32_i4_e32 v5, v1 ; encoding: [0x01,0x1d,0x0a,0x7e]
 
 v_cvt_off_f32_i4 v5, v255
-// GFX12: v_cvt_off_f32_i4_e32 v5, v255           ; encoding: [0xff,0x1d,0x0a,0x7e]
+// GFX12: v_cvt_off_f32_i4_e32 v5, v255 ; encoding: [0xff,0x1d,0x0a,0x7e]
 
 v_cvt_off_f32_i4 v5, s1
-// GFX12: v_cvt_off_f32_i4_e32 v5, s1             ; encoding: [0x01,0x1c,0x0a,0x7e]
+// GFX12: v_cvt_off_f32_i4_e32 v5, s1 ; encoding: [0x01,0x1c,0x0a,0x7e]
 
 v_cvt_off_f32_i4 v5, s105
-// GFX12: v_cvt_off_f32_i4_e32 v5, s105           ; encoding: [0x69,0x1c,0x0a,0x7e]
+// GFX12: v_cvt_off_f32_i4_e32 v5, s105 ; encoding: [0x69,0x1c,0x0a,0x7e]
 
 v_cvt_off_f32_i4 v5, vcc_lo
-// GFX12: v_cvt_off_f32_i4_e32 v5, vcc_lo         ; encoding: [0x6a,0x1c,0x0a,0x7e]
+// GFX12: v_cvt_off_f32_i4_e32 v5, vcc_lo ; encoding: [0x6a,0x1c,0x0a,0x7e]
 
 v_cvt_off_f32_i4 v5, vcc_hi
-// GFX12: v_cvt_off_f32_i4_e32 v5, vcc_hi         ; encoding: [0x6b,0x1c,0x0a,0x7e]
+// GFX12: v_cvt_off_f32_i4_e32 v5, vcc_hi ; encoding: [0x6b,0x1c,0x0a,0x7e]
 
 v_cvt_off_f32_i4 v5, ttmp15
-// GFX12: v_cvt_off_f32_i4_e32 v5, ttmp15         ; encoding: [0x7b,0x1c,0x0a,0x7e]
+// GFX12: v_cvt_off_f32_i4_e32 v5, ttmp15 ; encoding: [0x7b,0x1c,0x0a,0x7e]
 
 v_cvt_off_f32_i4 v5, m0
-// GFX12: v_cvt_off_f32_i4_e32 v5, m0             ; encoding: [0x7d,0x1c,0x0a,0x7e]
+// GFX12: v_cvt_off_f32_i4_e32 v5, m0 ; encoding: [0x7d,0x1c,0x0a,0x7e]
 
 v_cvt_off_f32_i4 v5, exec_lo
-// GFX12: v_cvt_off_f32_i4_e32 v5, exec_lo        ; encoding: [0x7e,0x1c,0x0a,0x7e]
+// GFX12: v_cvt_off_f32_i4_e32 v5, exec_lo ; encoding: [0x7e,0x1c,0x0a,0x7e]
 
 v_cvt_off_f32_i4 v5, exec_hi
-// GFX12: v_cvt_off_f32_i4_e32 v5, exec_hi        ; encoding: [0x7f,0x1c,0x0a,0x7e]
+// GFX12: v_cvt_off_f32_i4_e32 v5, exec_hi ; encoding: [0x7f,0x1c,0x0a,0x7e]
 
 v_cvt_off_f32_i4 v5, null
-// GFX12: v_cvt_off_f32_i4_e32 v5, null           ; encoding: [0x7c,0x1c,0x0a,0x7e]
+// GFX12: v_cvt_off_f32_i4_e32 v5, null ; encoding: [0x7c,0x1c,0x0a,0x7e]
 
 v_cvt_off_f32_i4 v5, -1
-// GFX12: v_cvt_off_f32_i4_e32 v5, -1             ; encoding: [0xc1,0x1c,0x0a,0x7e]
+// GFX12: v_cvt_off_f32_i4_e32 v5, -1 ; encoding: [0xc1,0x1c,0x0a,0x7e]
 
 v_cvt_off_f32_i4 v5, 0.5
-// GFX12: v_cvt_off_f32_i4_e32 v5, 0.5            ; encoding: [0xf0,0x1c,0x0a,0x7e]
+// GFX12: v_cvt_off_f32_i4_e32 v5, 0.5 ; encoding: [0xf0,0x1c,0x0a,0x7e]
 
 v_cvt_off_f32_i4 v5, src_scc
-// GFX12: v_cvt_off_f32_i4_e32 v5, src_scc        ; encoding: [0xfd,0x1c,0x0a,0x7e]
+// GFX12: v_cvt_off_f32_i4_e32 v5, src_scc ; encoding: [0xfd,0x1c,0x0a,0x7e]
 
 v_cvt_off_f32_i4 v255, 0x4f
-// GFX12: v_cvt_off_f32_i4_e32 v255, 0x4f         ; encoding: [0xff,0x1c,0xfe,0x7f,0x4f,0x00,0x00,0x00]
+// GFX12: v_cvt_off_f32_i4_e32 v255, 0x4f ; encoding: [0xff,0x1c,0xfe,0x7f,0x4f,0x00,0x00,0x00]
 
 v_cvt_rpi_i32_f32 v5, v1
-// GFX12: v_cvt_nearest_i32_f32_e32 v5, v1        ; encoding: [0x01,0x19,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, v1 ; encoding: [0x01,0x19,0x0a,0x7e]
 
 v_cvt_rpi_i32_f32 v5, v255
-// GFX12: v_cvt_nearest_i32_f32_e32 v5, v255      ; encoding: [0xff,0x19,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, v255 ; encoding: [0xff,0x19,0x0a,0x7e]
 
 v_cvt_rpi_i32_f32 v5, s1
-// GFX12: v_cvt_nearest_i32_f32_e32 v5, s1        ; encoding: [0x01,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, s1 ; encoding: [0x01,0x18,0x0a,0x7e]
 
 v_cvt_rpi_i32_f32 v5, s105
-// GFX12: v_cvt_nearest_i32_f32_e32 v5, s105      ; encoding: [0x69,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, s105 ; encoding: [0x69,0x18,0x0a,0x7e]
 
 v_cvt_rpi_i32_f32 v5, vcc_lo
-// GFX12: v_cvt_nearest_i32_f32_e32 v5, vcc_lo    ; encoding: [0x6a,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, vcc_lo ; encoding: [0x6a,0x18,0x0a,0x7e]
 
 v_cvt_rpi_i32_f32 v5, vcc_hi
-// GFX12: v_cvt_nearest_i32_f32_e32 v5, vcc_hi    ; encoding: [0x6b,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, vcc_hi ; encoding: [0x6b,0x18,0x0a,0x7e]
 
 v_cvt_rpi_i32_f32 v5, ttmp15
-// GFX12: v_cvt_nearest_i32_f32_e32 v5, ttmp15    ; encoding: [0x7b,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, ttmp15 ; encoding: [0x7b,0x18,0x0a,0x7e]
 
 v_cvt_rpi_i32_f32 v5, m0
-// GFX12: v_cvt_nearest_i32_f32_e32 v5, m0        ; encoding: [0x7d,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, m0 ; encoding: [0x7d,0x18,0x0a,0x7e]
 
 v_cvt_rpi_i32_f32 v5, exec_lo
-// GFX12: v_cvt_nearest_i32_f32_e32 v5, exec_lo   ; encoding: [0x7e,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, exec_lo ; encoding: [0x7e,0x18,0x0a,0x7e]
 
 v_cvt_rpi_i32_f32 v5, exec_hi
-// GFX12: v_cvt_nearest_i32_f32_e32 v5, exec_hi   ; encoding: [0x7f,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, exec_hi ; encoding: [0x7f,0x18,0x0a,0x7e]
 
 v_cvt_rpi_i32_f32 v5, null
-// GFX12: v_cvt_nearest_i32_f32_e32 v5, null      ; encoding: [0x7c,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, null ; encoding: [0x7c,0x18,0x0a,0x7e]
 
 v_cvt_rpi_i32_f32 v5, -1
-// GFX12: v_cvt_nearest_i32_f32_e32 v5, -1        ; encoding: [0xc1,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, -1 ; encoding: [0xc1,0x18,0x0a,0x7e]
 
 v_cvt_rpi_i32_f32 v5, 0.5
-// GFX12: v_cvt_nearest_i32_f32_e32 v5, 0.5       ; encoding: [0xf0,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, 0.5 ; encoding: [0xf0,0x18,0x0a,0x7e]
 
 v_cvt_rpi_i32_f32 v5, src_scc
-// GFX12: v_cvt_nearest_i32_f32_e32 v5, src_scc   ; encoding: [0xfd,0x18,0x0a,0x7e]
+// GFX12: v_cvt_nearest_i32_f32_e32 v5, src_scc ; encoding: [0xfd,0x18,0x0a,0x7e]
 
 v_cvt_rpi_i32_f32 v255, 0xaf123456
 // GFX12: v_cvt_nearest_i32_f32_e32 v255, 0xaf123456 ; encoding: [0xff,0x18,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cvt_u16_f16 v5, v1
-// GFX12: v_cvt_u16_f16_e32 v5, v1                ; encoding: [0x01,0xa5,0x0a,0x7e]
+// GFX12: v_cvt_u16_f16_e32 v5, v1 ; encoding: [0x01,0xa5,0x0a,0x7e]
 
 v_cvt_u16_f16 v5, v127
-// GFX12: v_cvt_u16_f16_e32 v5, v127              ; encoding: [0x7f,0xa5,0x0a,0x7e]
+// GFX12: v_cvt_u16_f16_e32 v5, v127 ; encoding: [0x7f,0xa5,0x0a,0x7e]
 
 v_cvt_u16_f16 v5, s1
-// GFX12: v_cvt_u16_f16_e32 v5, s1                ; encoding: [0x01,0xa4,0x0a,0x7e]
+// GFX12: v_cvt_u16_f16_e32 v5, s1 ; encoding: [0x01,0xa4,0x0a,0x7e]
 
 v_cvt_u16_f16 v5, s105
-// GFX12: v_cvt_u16_f16_e32 v5, s105              ; encoding: [0x69,0xa4,0x0a,0x7e]
+// GFX12: v_cvt_u16_f16_e32 v5, s105 ; encoding: [0x69,0xa4,0x0a,0x7e]
 
 v_cvt_u16_f16 v5, vcc_lo
-// GFX12: v_cvt_u16_f16_e32 v5, vcc_lo            ; encoding: [0x6a,0xa4,0x0a,0x7e]
+// GFX12: v_cvt_u16_f16_e32 v5, vcc_lo ; encoding: [0x6a,0xa4,0x0a,0x7e]
 
 v_cvt_u16_f16 v5, vcc_hi
-// GFX12: v_cvt_u16_f16_e32 v5, vcc_hi            ; encoding: [0x6b,0xa4,0x0a,0x7e]
+// GFX12: v_cvt_u16_f16_e32 v5, vcc_hi ; encoding: [0x6b,0xa4,0x0a,0x7e]
 
 v_cvt_u16_f16 v5, ttmp15
-// GFX12: v_cvt_u16_f16_e32 v5, ttmp15            ; encoding: [0x7b,0xa4,0x0a,0x7e]
+// GFX12: v_cvt_u16_f16_e32 v5, ttmp15 ; encoding: [0x7b,0xa4,0x0a,0x7e]
 
 v_cvt_u16_f16 v5, m0
-// GFX12: v_cvt_u16_f16_e32 v5, m0                ; encoding: [0x7d,0xa4,0x0a,0x7e]
+// GFX12: v_cvt_u16_f16_e32 v5, m0 ; encoding: [0x7d,0xa4,0x0a,0x7e]
 
 v_cvt_u16_f16 v5, exec_lo
-// GFX12: v_cvt_u16_f16_e32 v5, exec_lo           ; encoding: [0x7e,0xa4,0x0a,0x7e]
+// GFX12: v_cvt_u16_f16_e32 v5, exec_lo ; encoding: [0x7e,0xa4,0x0a,0x7e]
 
 v_cvt_u16_f16 v5, exec_hi
-// GFX12: v_cvt_u16_f16_e32 v5, exec_hi           ; encoding: [0x7f,0xa4,0x0a,0x7e]
+// GFX12: v_cvt_u16_f16_e32 v5, exec_hi ; encoding: [0x7f,0xa4,0x0a,0x7e]
 
 v_cvt_u16_f16 v5, null
-// GFX12: v_cvt_u16_f16_e32 v5, null              ; encoding: [0x7c,0xa4,0x0a,0x7e]
+// GFX12: v_cvt_u16_f16_e32 v5, null ; encoding: [0x7c,0xa4,0x0a,0x7e]
 
 v_cvt_u16_f16 v5, -1
-// GFX12: v_cvt_u16_f16_e32 v5, -1                ; encoding: [0xc1,0xa4,0x0a,0x7e]
+// GFX12: v_cvt_u16_f16_e32 v5, -1 ; encoding: [0xc1,0xa4,0x0a,0x7e]
 
 v_cvt_u16_f16 v5, 0.5
-// GFX12: v_cvt_u16_f16_e32 v5, 0.5               ; encoding: [0xf0,0xa4,0x0a,0x7e]
+// GFX12: v_cvt_u16_f16_e32 v5, 0.5 ; encoding: [0xf0,0xa4,0x0a,0x7e]
 
 v_cvt_u16_f16 v5, src_scc
-// GFX12: v_cvt_u16_f16_e32 v5, src_scc           ; encoding: [0xfd,0xa4,0x0a,0x7e]
+// GFX12: v_cvt_u16_f16_e32 v5, src_scc ; encoding: [0xfd,0xa4,0x0a,0x7e]
 
 v_cvt_u16_f16 v127, 0xfe0b
-// GFX12: v_cvt_u16_f16_e32 v127, 0xfe0b          ; encoding: [0xff,0xa4,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_cvt_u16_f16_e32 v127, 0xfe0b ; encoding: [0xff,0xa4,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_cvt_u32_f32 v5, v1
-// GFX12: v_cvt_u32_f32_e32 v5, v1                ; encoding: [0x01,0x0f,0x0a,0x7e]
+// GFX12: v_cvt_u32_f32_e32 v5, v1 ; encoding: [0x01,0x0f,0x0a,0x7e]
 
 v_cvt_u32_f32 v5, v255
-// GFX12: v_cvt_u32_f32_e32 v5, v255              ; encoding: [0xff,0x0f,0x0a,0x7e]
+// GFX12: v_cvt_u32_f32_e32 v5, v255 ; encoding: [0xff,0x0f,0x0a,0x7e]
 
 v_cvt_u32_f32 v5, s1
-// GFX12: v_cvt_u32_f32_e32 v5, s1                ; encoding: [0x01,0x0e,0x0a,0x7e]
+// GFX12: v_cvt_u32_f32_e32 v5, s1 ; encoding: [0x01,0x0e,0x0a,0x7e]
 
 v_cvt_u32_f32 v5, s105
-// GFX12: v_cvt_u32_f32_e32 v5, s105              ; encoding: [0x69,0x0e,0x0a,0x7e]
+// GFX12: v_cvt_u32_f32_e32 v5, s105 ; encoding: [0x69,0x0e,0x0a,0x7e]
 
 v_cvt_u32_f32 v5, vcc_lo
-// GFX12: v_cvt_u32_f32_e32 v5, vcc_lo            ; encoding: [0x6a,0x0e,0x0a,0x7e]
+// GFX12: v_cvt_u32_f32_e32 v5, vcc_lo ; encoding: [0x6a,0x0e,0x0a,0x7e]
 
 v_cvt_u32_f32 v5, vcc_hi
-// GFX12: v_cvt_u32_f32_e32 v5, vcc_hi            ; encoding: [0x6b,0x0e,0x0a,0x7e]
+// GFX12: v_cvt_u32_f32_e32 v5, vcc_hi ; encoding: [0x6b,0x0e,0x0a,0x7e]
 
 v_cvt_u32_f32 v5, ttmp15
-// GFX12: v_cvt_u32_f32_e32 v5, ttmp15            ; encoding: [0x7b,0x0e,0x0a,0x7e]
+// GFX12: v_cvt_u32_f32_e32 v5, ttmp15 ; encoding: [0x7b,0x0e,0x0a,0x7e]
 
 v_cvt_u32_f32 v5, m0
-// GFX12: v_cvt_u32_f32_e32 v5, m0                ; encoding: [0x7d,0x0e,0x0a,0x7e]
+// GFX12: v_cvt_u32_f32_e32 v5, m0 ; encoding: [0x7d,0x0e,0x0a,0x7e]
 
 v_cvt_u32_f32 v5, exec_lo
-// GFX12: v_cvt_u32_f32_e32 v5, exec_lo           ; encoding: [0x7e,0x0e,0x0a,0x7e]
+// GFX12: v_cvt_u32_f32_e32 v5, exec_lo ; encoding: [0x7e,0x0e,0x0a,0x7e]
 
 v_cvt_u32_f32 v5, exec_hi
-// GFX12: v_cvt_u32_f32_e32 v5, exec_hi           ; encoding: [0x7f,0x0e,0x0a,0x7e]
+// GFX12: v_cvt_u32_f32_e32 v5, exec_hi ; encoding: [0x7f,0x0e,0x0a,0x7e]
 
 v_cvt_u32_f32 v5, null
-// GFX12: v_cvt_u32_f32_e32 v5, null              ; encoding: [0x7c,0x0e,0x0a,0x7e]
+// GFX12: v_cvt_u32_f32_e32 v5, null ; encoding: [0x7c,0x0e,0x0a,0x7e]
 
 v_cvt_u32_f32 v5, -1
-// GFX12: v_cvt_u32_f32_e32 v5, -1                ; encoding: [0xc1,0x0e,0x0a,0x7e]
+// GFX12: v_cvt_u32_f32_e32 v5, -1 ; encoding: [0xc1,0x0e,0x0a,0x7e]
 
 v_cvt_u32_f32 v5, 0.5
-// GFX12: v_cvt_u32_f32_e32 v5, 0.5               ; encoding: [0xf0,0x0e,0x0a,0x7e]
+// GFX12: v_cvt_u32_f32_e32 v5, 0.5 ; encoding: [0xf0,0x0e,0x0a,0x7e]
 
 v_cvt_u32_f32 v5, src_scc
-// GFX12: v_cvt_u32_f32_e32 v5, src_scc           ; encoding: [0xfd,0x0e,0x0a,0x7e]
+// GFX12: v_cvt_u32_f32_e32 v5, src_scc ; encoding: [0xfd,0x0e,0x0a,0x7e]
 
 v_cvt_u32_f32 v255, 0xaf123456
-// GFX12: v_cvt_u32_f32_e32 v255, 0xaf123456      ; encoding: [0xff,0x0e,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_cvt_u32_f32_e32 v255, 0xaf123456 ; encoding: [0xff,0x0e,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cvt_u32_f64 v5, v[1:2]
-// GFX12: v_cvt_u32_f64_e32 v5, v[1:2]            ; encoding: [0x01,0x2b,0x0a,0x7e]
+// GFX12: v_cvt_u32_f64_e32 v5, v[1:2] ; encoding: [0x01,0x2b,0x0a,0x7e]
 
 v_cvt_u32_f64 v5, v[254:255]
-// GFX12: v_cvt_u32_f64_e32 v5, v[254:255]        ; encoding: [0xfe,0x2b,0x0a,0x7e]
+// GFX12: v_cvt_u32_f64_e32 v5, v[254:255] ; encoding: [0xfe,0x2b,0x0a,0x7e]
 
 v_cvt_u32_f64 v5, s[2:3]
-// GFX12: v_cvt_u32_f64_e32 v5, s[2:3]            ; encoding: [0x02,0x2a,0x0a,0x7e]
+// GFX12: v_cvt_u32_f64_e32 v5, s[2:3] ; encoding: [0x02,0x2a,0x0a,0x7e]
 
 v_cvt_u32_f64 v5, s[104:105]
-// GFX12: v_cvt_u32_f64_e32 v5, s[104:105]        ; encoding: [0x68,0x2a,0x0a,0x7e]
+// GFX12: v_cvt_u32_f64_e32 v5, s[104:105] ; encoding: [0x68,0x2a,0x0a,0x7e]
 
 v_cvt_u32_f64 v5, vcc
-// GFX12: v_cvt_u32_f64_e32 v5, vcc               ; encoding: [0x6a,0x2a,0x0a,0x7e]
+// GFX12: v_cvt_u32_f64_e32 v5, vcc ; encoding: [0x6a,0x2a,0x0a,0x7e]
 
 v_cvt_u32_f64 v5, ttmp[14:15]
-// GFX12: v_cvt_u32_f64_e32 v5, ttmp[14:15]       ; encoding: [0x7a,0x2a,0x0a,0x7e]
+// GFX12: v_cvt_u32_f64_e32 v5, ttmp[14:15] ; encoding: [0x7a,0x2a,0x0a,0x7e]
 
 v_cvt_u32_f64 v5, exec
-// GFX12: v_cvt_u32_f64_e32 v5, exec              ; encoding: [0x7e,0x2a,0x0a,0x7e]
+// GFX12: v_cvt_u32_f64_e32 v5, exec ; encoding: [0x7e,0x2a,0x0a,0x7e]
 
 v_cvt_u32_f64 v5, null
-// GFX12: v_cvt_u32_f64_e32 v5, null              ; encoding: [0x7c,0x2a,0x0a,0x7e]
+// GFX12: v_cvt_u32_f64_e32 v5, null ; encoding: [0x7c,0x2a,0x0a,0x7e]
 
 v_cvt_u32_f64 v5, -1
-// GFX12: v_cvt_u32_f64_e32 v5, -1                ; encoding: [0xc1,0x2a,0x0a,0x7e]
+// GFX12: v_cvt_u32_f64_e32 v5, -1 ; encoding: [0xc1,0x2a,0x0a,0x7e]
 
 v_cvt_u32_f64 v5, 0.5
-// GFX12: v_cvt_u32_f64_e32 v5, 0.5               ; encoding: [0xf0,0x2a,0x0a,0x7e]
+// GFX12: v_cvt_u32_f64_e32 v5, 0.5 ; encoding: [0xf0,0x2a,0x0a,0x7e]
 
 v_cvt_u32_f64 v5, src_scc
-// GFX12: v_cvt_u32_f64_e32 v5, src_scc           ; encoding: [0xfd,0x2a,0x0a,0x7e]
+// GFX12: v_cvt_u32_f64_e32 v5, src_scc ; encoding: [0xfd,0x2a,0x0a,0x7e]
 
 v_cvt_u32_f64 v255, 0xaf123456
-// GFX12: v_cvt_u32_f64_e32 v255, 0xaf123456      ; encoding: [0xff,0x2a,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_cvt_u32_f64_e32 v255, 0xaf123456 ; encoding: [0xff,0x2a,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_cvt_u32_u16 v5, v1
-// GFX12: v_cvt_u32_u16_e32 v5, v1                ; encoding: [0x01,0xd7,0x0a,0x7e]
+// GFX12: v_cvt_u32_u16_e32 v5, v1 ; encoding: [0x01,0xd7,0x0a,0x7e]
 
 v_cvt_u32_u16 v5, v127
-// GFX12: v_cvt_u32_u16_e32 v5, v127              ; encoding: [0x7f,0xd7,0x0a,0x7e]
+// GFX12: v_cvt_u32_u16_e32 v5, v127 ; encoding: [0x7f,0xd7,0x0a,0x7e]
 
 v_cvt_u32_u16 v5, s1
-// GFX12: v_cvt_u32_u16_e32 v5, s1                ; encoding: [0x01,0xd6,0x0a,0x7e]
+// GFX12: v_cvt_u32_u16_e32 v5, s1 ; encoding: [0x01,0xd6,0x0a,0x7e]
 
 v_cvt_u32_u16 v5, s105
-// GFX12: v_cvt_u32_u16_e32 v5, s105              ; encoding: [0x69,0xd6,0x0a,0x7e]
+// GFX12: v_cvt_u32_u16_e32 v5, s105 ; encoding: [0x69,0xd6,0x0a,0x7e]
 
 v_cvt_u32_u16 v5, vcc_lo
-// GFX12: v_cvt_u32_u16_e32 v5, vcc_lo            ; encoding: [0x6a,0xd6,0x0a,0x7e]
+// GFX12: v_cvt_u32_u16_e32 v5, vcc_lo ; encoding: [0x6a,0xd6,0x0a,0x7e]
 
 v_cvt_u32_u16 v5, vcc_hi
-// GFX12: v_cvt_u32_u16_e32 v5, vcc_hi            ; encoding: [0x6b,0xd6,0x0a,0x7e]
+// GFX12: v_cvt_u32_u16_e32 v5, vcc_hi ; encoding: [0x6b,0xd6,0x0a,0x7e]
 
 v_cvt_u32_u16 v5, ttmp15
-// GFX12: v_cvt_u32_u16_e32 v5, ttmp15            ; encoding: [0x7b,0xd6,0x0a,0x7e]
+// GFX12: v_cvt_u32_u16_e32 v5, ttmp15 ; encoding: [0x7b,0xd6,0x0a,0x7e]
 
 v_cvt_u32_u16 v5, m0
-// GFX12: v_cvt_u32_u16_e32 v5, m0                ; encoding: [0x7d,0xd6,0x0a,0x7e]
+// GFX12: v_cvt_u32_u16_e32 v5, m0 ; encoding: [0x7d,0xd6,0x0a,0x7e]
 
 v_cvt_u32_u16 v5, exec_lo
-// GFX12: v_cvt_u32_u16_e32 v5, exec_lo           ; encoding: [0x7e,0xd6,0x0a,0x7e]
+// GFX12: v_cvt_u32_u16_e32 v5, exec_lo ; encoding: [0x7e,0xd6,0x0a,0x7e]
 
 v_cvt_u32_u16 v5, exec_hi
-// GFX12: v_cvt_u32_u16_e32 v5, exec_hi           ; encoding: [0x7f,0xd6,0x0a,0x7e]
+// GFX12: v_cvt_u32_u16_e32 v5, exec_hi ; encoding: [0x7f,0xd6,0x0a,0x7e]
 
 v_cvt_u32_u16 v5, null
-// GFX12: v_cvt_u32_u16_e32 v5, null              ; encoding: [0x7c,0xd6,0x0a,0x7e]
+// GFX12: v_cvt_u32_u16_e32 v5, null ; encoding: [0x7c,0xd6,0x0a,0x7e]
 
 v_cvt_u32_u16 v5, -1
-// GFX12: v_cvt_u32_u16_e32 v5, -1                ; encoding: [0xc1,0xd6,0x0a,0x7e]
+// GFX12: v_cvt_u32_u16_e32 v5, -1 ; encoding: [0xc1,0xd6,0x0a,0x7e]
 
 v_cvt_u32_u16 v5, 0.5
-// GFX12-ASM: v_cvt_u32_u16_e32 v5, 0.5               ; encoding: [0xf0,0xd6,0x0a,0x7e]
-// GFX12-DIS: v_cvt_u32_u16_e32 v5, 0x3800            ; encoding: [0xff,0xd6,0x0a,0x7e,0x00,0x38,0x00,0x00]
+// GFX12-ASM: v_cvt_u32_u16_e32 v5, 0.5 ; encoding: [0xf0,0xd6,0x0a,0x7e]
+// GFX12-DIS: v_cvt_u32_u16_e32 v5, 0x3800 ; encoding: [0xff,0xd6,0x0a,0x7e,0x00,0x38,0x00,0x00]
 
 v_cvt_u32_u16 v5, src_scc
-// GFX12: v_cvt_u32_u16_e32 v5, src_scc           ; encoding: [0xfd,0xd6,0x0a,0x7e]
+// GFX12: v_cvt_u32_u16_e32 v5, src_scc ; encoding: [0xfd,0xd6,0x0a,0x7e]
 
 v_cvt_u32_u16 v255, 0xfe0b
-// GFX12: v_cvt_u32_u16_e32 v255, 0xfe0b          ; encoding: [0xff,0xd6,0xfe,0x7f,0x0b,0xfe,0x00,0x00]
+// GFX12: v_cvt_u32_u16_e32 v255, 0xfe0b ; encoding: [0xff,0xd6,0xfe,0x7f,0x0b,0xfe,0x00,0x00]
 
 v_exp_f16 v5, v1
-// GFX12: v_exp_f16_e32 v5, v1                    ; encoding: [0x01,0xb1,0x0a,0x7e]
+// GFX12: v_exp_f16_e32 v5, v1 ; encoding: [0x01,0xb1,0x0a,0x7e]
 
 v_exp_f16 v5, v127
-// GFX12: v_exp_f16_e32 v5, v127                  ; encoding: [0x7f,0xb1,0x0a,0x7e]
+// GFX12: v_exp_f16_e32 v5, v127 ; encoding: [0x7f,0xb1,0x0a,0x7e]
 
 v_exp_f16 v5, s1
-// GFX12: v_exp_f16_e32 v5, s1                    ; encoding: [0x01,0xb0,0x0a,0x7e]
+// GFX12: v_exp_f16_e32 v5, s1 ; encoding: [0x01,0xb0,0x0a,0x7e]
 
 v_exp_f16 v5, s105
-// GFX12: v_exp_f16_e32 v5, s105                  ; encoding: [0x69,0xb0,0x0a,0x7e]
+// GFX12: v_exp_f16_e32 v5, s105 ; encoding: [0x69,0xb0,0x0a,0x7e]
 
 v_exp_f16 v5, vcc_lo
-// GFX12: v_exp_f16_e32 v5, vcc_lo                ; encoding: [0x6a,0xb0,0x0a,0x7e]
+// GFX12: v_exp_f16_e32 v5, vcc_lo ; encoding: [0x6a,0xb0,0x0a,0x7e]
 
 v_exp_f16 v5, vcc_hi
-// GFX12: v_exp_f16_e32 v5, vcc_hi                ; encoding: [0x6b,0xb0,0x0a,0x7e]
+// GFX12: v_exp_f16_e32 v5, vcc_hi ; encoding: [0x6b,0xb0,0x0a,0x7e]
 
 v_exp_f16 v5, ttmp15
-// GFX12: v_exp_f16_e32 v5, ttmp15                ; encoding: [0x7b,0xb0,0x0a,0x7e]
+// GFX12: v_exp_f16_e32 v5, ttmp15 ; encoding: [0x7b,0xb0,0x0a,0x7e]
 
 v_exp_f16 v5, m0
-// GFX12: v_exp_f16_e32 v5, m0                    ; encoding: [0x7d,0xb0,0x0a,0x7e]
+// GFX12: v_exp_f16_e32 v5, m0 ; encoding: [0x7d,0xb0,0x0a,0x7e]
 
 v_exp_f16 v5, exec_lo
-// GFX12: v_exp_f16_e32 v5, exec_lo               ; encoding: [0x7e,0xb0,0x0a,0x7e]
+// GFX12: v_exp_f16_e32 v5, exec_lo ; encoding: [0x7e,0xb0,0x0a,0x7e]
 
 v_exp_f16 v5, exec_hi
-// GFX12: v_exp_f16_e32 v5, exec_hi               ; encoding: [0x7f,0xb0,0x0a,0x7e]
+// GFX12: v_exp_f16_e32 v5, exec_hi ; encoding: [0x7f,0xb0,0x0a,0x7e]
 
 v_exp_f16 v5, null
-// GFX12: v_exp_f16_e32 v5, null                  ; encoding: [0x7c,0xb0,0x0a,0x7e]
+// GFX12: v_exp_f16_e32 v5, null ; encoding: [0x7c,0xb0,0x0a,0x7e]
 
 v_exp_f16 v5, -1
-// GFX12: v_exp_f16_e32 v5, -1                    ; encoding: [0xc1,0xb0,0x0a,0x7e]
+// GFX12: v_exp_f16_e32 v5, -1 ; encoding: [0xc1,0xb0,0x0a,0x7e]
 
 v_exp_f16 v5, 0.5
-// GFX12: v_exp_f16_e32 v5, 0.5                   ; encoding: [0xf0,0xb0,0x0a,0x7e]
+// GFX12: v_exp_f16_e32 v5, 0.5 ; encoding: [0xf0,0xb0,0x0a,0x7e]
 
 v_exp_f16 v5, src_scc
-// GFX12: v_exp_f16_e32 v5, src_scc               ; encoding: [0xfd,0xb0,0x0a,0x7e]
+// GFX12: v_exp_f16_e32 v5, src_scc ; encoding: [0xfd,0xb0,0x0a,0x7e]
 
 v_exp_f16 v127, 0xfe0b
-// GFX12: v_exp_f16_e32 v127, 0xfe0b              ; encoding: [0xff,0xb0,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_exp_f16_e32 v127, 0xfe0b ; encoding: [0xff,0xb0,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_exp_f32 v5, v1
-// GFX12: v_exp_f32_e32 v5, v1                    ; encoding: [0x01,0x4b,0x0a,0x7e]
+// GFX12: v_exp_f32_e32 v5, v1 ; encoding: [0x01,0x4b,0x0a,0x7e]
 
 v_exp_f32 v5, v255
-// GFX12: v_exp_f32_e32 v5, v255                  ; encoding: [0xff,0x4b,0x0a,0x7e]
+// GFX12: v_exp_f32_e32 v5, v255 ; encoding: [0xff,0x4b,0x0a,0x7e]
 
 v_exp_f32 v5, s1
-// GFX12: v_exp_f32_e32 v5, s1                    ; encoding: [0x01,0x4a,0x0a,0x7e]
+// GFX12: v_exp_f32_e32 v5, s1 ; encoding: [0x01,0x4a,0x0a,0x7e]
 
 v_exp_f32 v5, s105
-// GFX12: v_exp_f32_e32 v5, s105                  ; encoding: [0x69,0x4a,0x0a,0x7e]
+// GFX12: v_exp_f32_e32 v5, s105 ; encoding: [0x69,0x4a,0x0a,0x7e]
 
 v_exp_f32 v5, vcc_lo
-// GFX12: v_exp_f32_e32 v5, vcc_lo                ; encoding: [0x6a,0x4a,0x0a,0x7e]
+// GFX12: v_exp_f32_e32 v5, vcc_lo ; encoding: [0x6a,0x4a,0x0a,0x7e]
 
 v_exp_f32 v5, vcc_hi
-// GFX12: v_exp_f32_e32 v5, vcc_hi                ; encoding: [0x6b,0x4a,0x0a,0x7e]
+// GFX12: v_exp_f32_e32 v5, vcc_hi ; encoding: [0x6b,0x4a,0x0a,0x7e]
 
 v_exp_f32 v5, ttmp15
-// GFX12: v_exp_f32_e32 v5, ttmp15                ; encoding: [0x7b,0x4a,0x0a,0x7e]
+// GFX12: v_exp_f32_e32 v5, ttmp15 ; encoding: [0x7b,0x4a,0x0a,0x7e]
 
 v_exp_f32 v5, m0
-// GFX12: v_exp_f32_e32 v5, m0                    ; encoding: [0x7d,0x4a,0x0a,0x7e]
+// GFX12: v_exp_f32_e32 v5, m0 ; encoding: [0x7d,0x4a,0x0a,0x7e]
 
 v_exp_f32 v5, exec_lo
-// GFX12: v_exp_f32_e32 v5, exec_lo               ; encoding: [0x7e,0x4a,0x0a,0x7e]
+// GFX12: v_exp_f32_e32 v5, exec_lo ; encoding: [0x7e,0x4a,0x0a,0x7e]
 
 v_exp_f32 v5, exec_hi
-// GFX12: v_exp_f32_e32 v5, exec_hi               ; encoding: [0x7f,0x4a,0x0a,0x7e]
+// GFX12: v_exp_f32_e32 v5, exec_hi ; encoding: [0x7f,0x4a,0x0a,0x7e]
 
 v_exp_f32 v5, null
-// GFX12: v_exp_f32_e32 v5, null                  ; encoding: [0x7c,0x4a,0x0a,0x7e]
+// GFX12: v_exp_f32_e32 v5, null ; encoding: [0x7c,0x4a,0x0a,0x7e]
 
 v_exp_f32 v5, -1
-// GFX12: v_exp_f32_e32 v5, -1                    ; encoding: [0xc1,0x4a,0x0a,0x7e]
+// GFX12: v_exp_f32_e32 v5, -1 ; encoding: [0xc1,0x4a,0x0a,0x7e]
 
 v_exp_f32 v5, 0.5
-// GFX12: v_exp_f32_e32 v5, 0.5                   ; encoding: [0xf0,0x4a,0x0a,0x7e]
+// GFX12: v_exp_f32_e32 v5, 0.5 ; encoding: [0xf0,0x4a,0x0a,0x7e]
 
 v_exp_f32 v5, src_scc
-// GFX12: v_exp_f32_e32 v5, src_scc               ; encoding: [0xfd,0x4a,0x0a,0x7e]
+// GFX12: v_exp_f32_e32 v5, src_scc ; encoding: [0xfd,0x4a,0x0a,0x7e]
 
 v_exp_f32 v255, 0xaf123456
-// GFX12: v_exp_f32_e32 v255, 0xaf123456          ; encoding: [0xff,0x4a,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_exp_f32_e32 v255, 0xaf123456 ; encoding: [0xff,0x4a,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_ffbh_i32 v5, v1
-// GFX12: v_cls_i32_e32 v5, v1                    ; encoding: [0x01,0x77,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, v1 ; encoding: [0x01,0x77,0x0a,0x7e]
 
 v_ffbh_i32 v5, v255
-// GFX12: v_cls_i32_e32 v5, v255                  ; encoding: [0xff,0x77,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, v255 ; encoding: [0xff,0x77,0x0a,0x7e]
 
 v_ffbh_i32 v5, s1
-// GFX12: v_cls_i32_e32 v5, s1                    ; encoding: [0x01,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, s1 ; encoding: [0x01,0x76,0x0a,0x7e]
 
 v_ffbh_i32 v5, s105
-// GFX12: v_cls_i32_e32 v5, s105                  ; encoding: [0x69,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, s105 ; encoding: [0x69,0x76,0x0a,0x7e]
 
 v_ffbh_i32 v5, vcc_lo
-// GFX12: v_cls_i32_e32 v5, vcc_lo                ; encoding: [0x6a,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, vcc_lo ; encoding: [0x6a,0x76,0x0a,0x7e]
 
 v_ffbh_i32 v5, vcc_hi
-// GFX12: v_cls_i32_e32 v5, vcc_hi                ; encoding: [0x6b,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, vcc_hi ; encoding: [0x6b,0x76,0x0a,0x7e]
 
 v_ffbh_i32 v5, ttmp15
-// GFX12: v_cls_i32_e32 v5, ttmp15                ; encoding: [0x7b,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, ttmp15 ; encoding: [0x7b,0x76,0x0a,0x7e]
 
 v_ffbh_i32 v5, m0
-// GFX12: v_cls_i32_e32 v5, m0                    ; encoding: [0x7d,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, m0 ; encoding: [0x7d,0x76,0x0a,0x7e]
 
 v_ffbh_i32 v5, exec_lo
-// GFX12: v_cls_i32_e32 v5, exec_lo               ; encoding: [0x7e,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, exec_lo ; encoding: [0x7e,0x76,0x0a,0x7e]
 
 v_ffbh_i32 v5, exec_hi
-// GFX12: v_cls_i32_e32 v5, exec_hi               ; encoding: [0x7f,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, exec_hi ; encoding: [0x7f,0x76,0x0a,0x7e]
 
 v_ffbh_i32 v5, null
-// GFX12: v_cls_i32_e32 v5, null                  ; encoding: [0x7c,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, null ; encoding: [0x7c,0x76,0x0a,0x7e]
 
 v_ffbh_i32 v5, -1
-// GFX12: v_cls_i32_e32 v5, -1                    ; encoding: [0xc1,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, -1 ; encoding: [0xc1,0x76,0x0a,0x7e]
 
 v_ffbh_i32 v5, 0.5
-// GFX12: v_cls_i32_e32 v5, 0.5                   ; encoding: [0xf0,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, 0.5 ; encoding: [0xf0,0x76,0x0a,0x7e]
 
 v_ffbh_i32 v5, src_scc
-// GFX12: v_cls_i32_e32 v5, src_scc               ; encoding: [0xfd,0x76,0x0a,0x7e]
+// GFX12: v_cls_i32_e32 v5, src_scc ; encoding: [0xfd,0x76,0x0a,0x7e]
 
 v_ffbh_i32 v255, 0xaf123456
-// GFX12: v_cls_i32_e32 v255, 0xaf123456          ; encoding: [0xff,0x76,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_cls_i32_e32 v255, 0xaf123456 ; encoding: [0xff,0x76,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_ffbh_u32 v5, v1
-// GFX12: v_clz_i32_u32_e32 v5, v1                ; encoding: [0x01,0x73,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, v1 ; encoding: [0x01,0x73,0x0a,0x7e]
 
 v_ffbh_u32 v5, v255
-// GFX12: v_clz_i32_u32_e32 v5, v255              ; encoding: [0xff,0x73,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, v255 ; encoding: [0xff,0x73,0x0a,0x7e]
 
 v_ffbh_u32 v5, s1
-// GFX12: v_clz_i32_u32_e32 v5, s1                ; encoding: [0x01,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, s1 ; encoding: [0x01,0x72,0x0a,0x7e]
 
 v_ffbh_u32 v5, s105
-// GFX12: v_clz_i32_u32_e32 v5, s105              ; encoding: [0x69,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, s105 ; encoding: [0x69,0x72,0x0a,0x7e]
 
 v_ffbh_u32 v5, vcc_lo
-// GFX12: v_clz_i32_u32_e32 v5, vcc_lo            ; encoding: [0x6a,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, vcc_lo ; encoding: [0x6a,0x72,0x0a,0x7e]
 
 v_ffbh_u32 v5, vcc_hi
-// GFX12: v_clz_i32_u32_e32 v5, vcc_hi            ; encoding: [0x6b,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, vcc_hi ; encoding: [0x6b,0x72,0x0a,0x7e]
 
 v_ffbh_u32 v5, ttmp15
-// GFX12: v_clz_i32_u32_e32 v5, ttmp15            ; encoding: [0x7b,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, ttmp15 ; encoding: [0x7b,0x72,0x0a,0x7e]
 
 v_ffbh_u32 v5, m0
-// GFX12: v_clz_i32_u32_e32 v5, m0                ; encoding: [0x7d,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, m0 ; encoding: [0x7d,0x72,0x0a,0x7e]
 
 v_ffbh_u32 v5, exec_lo
-// GFX12: v_clz_i32_u32_e32 v5, exec_lo           ; encoding: [0x7e,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, exec_lo ; encoding: [0x7e,0x72,0x0a,0x7e]
 
 v_ffbh_u32 v5, exec_hi
-// GFX12: v_clz_i32_u32_e32 v5, exec_hi           ; encoding: [0x7f,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, exec_hi ; encoding: [0x7f,0x72,0x0a,0x7e]
 
 v_ffbh_u32 v5, null
-// GFX12: v_clz_i32_u32_e32 v5, null              ; encoding: [0x7c,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, null ; encoding: [0x7c,0x72,0x0a,0x7e]
 
 v_ffbh_u32 v5, -1
-// GFX12: v_clz_i32_u32_e32 v5, -1                ; encoding: [0xc1,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, -1 ; encoding: [0xc1,0x72,0x0a,0x7e]
 
 v_ffbh_u32 v5, 0.5
-// GFX12: v_clz_i32_u32_e32 v5, 0.5               ; encoding: [0xf0,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, 0.5 ; encoding: [0xf0,0x72,0x0a,0x7e]
 
 v_ffbh_u32 v5, src_scc
-// GFX12: v_clz_i32_u32_e32 v5, src_scc           ; encoding: [0xfd,0x72,0x0a,0x7e]
+// GFX12: v_clz_i32_u32_e32 v5, src_scc ; encoding: [0xfd,0x72,0x0a,0x7e]
 
 v_ffbh_u32 v255, 0xaf123456
-// GFX12: v_clz_i32_u32_e32 v255, 0xaf123456      ; encoding: [0xff,0x72,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_clz_i32_u32_e32 v255, 0xaf123456 ; encoding: [0xff,0x72,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_ffbl_b32 v5, v1
-// GFX12: v_ctz_i32_b32_e32 v5, v1                ; encoding: [0x01,0x75,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, v1 ; encoding: [0x01,0x75,0x0a,0x7e]
 
 v_ffbl_b32 v5, v255
-// GFX12: v_ctz_i32_b32_e32 v5, v255              ; encoding: [0xff,0x75,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, v255 ; encoding: [0xff,0x75,0x0a,0x7e]
 
 v_ffbl_b32 v5, s1
-// GFX12: v_ctz_i32_b32_e32 v5, s1                ; encoding: [0x01,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, s1 ; encoding: [0x01,0x74,0x0a,0x7e]
 
 v_ffbl_b32 v5, s105
-// GFX12: v_ctz_i32_b32_e32 v5, s105              ; encoding: [0x69,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, s105 ; encoding: [0x69,0x74,0x0a,0x7e]
 
 v_ffbl_b32 v5, vcc_lo
-// GFX12: v_ctz_i32_b32_e32 v5, vcc_lo            ; encoding: [0x6a,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, vcc_lo ; encoding: [0x6a,0x74,0x0a,0x7e]
 
 v_ffbl_b32 v5, vcc_hi
-// GFX12: v_ctz_i32_b32_e32 v5, vcc_hi            ; encoding: [0x6b,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, vcc_hi ; encoding: [0x6b,0x74,0x0a,0x7e]
 
 v_ffbl_b32 v5, ttmp15
-// GFX12: v_ctz_i32_b32_e32 v5, ttmp15            ; encoding: [0x7b,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, ttmp15 ; encoding: [0x7b,0x74,0x0a,0x7e]
 
 v_ffbl_b32 v5, m0
-// GFX12: v_ctz_i32_b32_e32 v5, m0                ; encoding: [0x7d,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, m0 ; encoding: [0x7d,0x74,0x0a,0x7e]
 
 v_ffbl_b32 v5, exec_lo
-// GFX12: v_ctz_i32_b32_e32 v5, exec_lo           ; encoding: [0x7e,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, exec_lo ; encoding: [0x7e,0x74,0x0a,0x7e]
 
 v_ffbl_b32 v5, exec_hi
-// GFX12: v_ctz_i32_b32_e32 v5, exec_hi           ; encoding: [0x7f,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, exec_hi ; encoding: [0x7f,0x74,0x0a,0x7e]
 
 v_ffbl_b32 v5, null
-// GFX12: v_ctz_i32_b32_e32 v5, null              ; encoding: [0x7c,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, null ; encoding: [0x7c,0x74,0x0a,0x7e]
 
 v_ffbl_b32 v5, -1
-// GFX12: v_ctz_i32_b32_e32 v5, -1                ; encoding: [0xc1,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, -1 ; encoding: [0xc1,0x74,0x0a,0x7e]
 
 v_ffbl_b32 v5, 0.5
-// GFX12: v_ctz_i32_b32_e32 v5, 0.5               ; encoding: [0xf0,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, 0.5 ; encoding: [0xf0,0x74,0x0a,0x7e]
 
 v_ffbl_b32 v5, src_scc
-// GFX12: v_ctz_i32_b32_e32 v5, src_scc           ; encoding: [0xfd,0x74,0x0a,0x7e]
+// GFX12: v_ctz_i32_b32_e32 v5, src_scc ; encoding: [0xfd,0x74,0x0a,0x7e]
 
 v_ffbl_b32 v255, 0xaf123456
-// GFX12: v_ctz_i32_b32_e32 v255, 0xaf123456      ; encoding: [0xff,0x74,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_ctz_i32_b32_e32 v255, 0xaf123456 ; encoding: [0xff,0x74,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_floor_f16 v5, v1
-// GFX12: v_floor_f16_e32 v5, v1                  ; encoding: [0x01,0xb7,0x0a,0x7e]
+// GFX12: v_floor_f16_e32 v5, v1 ; encoding: [0x01,0xb7,0x0a,0x7e]
 
 v_floor_f16 v5, v127
-// GFX12: v_floor_f16_e32 v5, v127                ; encoding: [0x7f,0xb7,0x0a,0x7e]
+// GFX12: v_floor_f16_e32 v5, v127 ; encoding: [0x7f,0xb7,0x0a,0x7e]
 
 v_floor_f16 v5, s1
-// GFX12: v_floor_f16_e32 v5, s1                  ; encoding: [0x01,0xb6,0x0a,0x7e]
+// GFX12: v_floor_f16_e32 v5, s1 ; encoding: [0x01,0xb6,0x0a,0x7e]
 
 v_floor_f16 v5, s105
-// GFX12: v_floor_f16_e32 v5, s105                ; encoding: [0x69,0xb6,0x0a,0x7e]
+// GFX12: v_floor_f16_e32 v5, s105 ; encoding: [0x69,0xb6,0x0a,0x7e]
 
 v_floor_f16 v5, vcc_lo
-// GFX12: v_floor_f16_e32 v5, vcc_lo              ; encoding: [0x6a,0xb6,0x0a,0x7e]
+// GFX12: v_floor_f16_e32 v5, vcc_lo ; encoding: [0x6a,0xb6,0x0a,0x7e]
 
 v_floor_f16 v5, vcc_hi
-// GFX12: v_floor_f16_e32 v5, vcc_hi              ; encoding: [0x6b,0xb6,0x0a,0x7e]
+// GFX12: v_floor_f16_e32 v5, vcc_hi ; encoding: [0x6b,0xb6,0x0a,0x7e]
 
 v_floor_f16 v5, ttmp15
-// GFX12: v_floor_f16_e32 v5, ttmp15              ; encoding: [0x7b,0xb6,0x0a,0x7e]
+// GFX12: v_floor_f16_e32 v5, ttmp15 ; encoding: [0x7b,0xb6,0x0a,0x7e]
 
 v_floor_f16 v5, m0
-// GFX12: v_floor_f16_e32 v5, m0                  ; encoding: [0x7d,0xb6,0x0a,0x7e]
+// GFX12: v_floor_f16_e32 v5, m0 ; encoding: [0x7d,0xb6,0x0a,0x7e]
 
 v_floor_f16 v5, exec_lo
-// GFX12: v_floor_f16_e32 v5, exec_lo             ; encoding: [0x7e,0xb6,0x0a,0x7e]
+// GFX12: v_floor_f16_e32 v5, exec_lo ; encoding: [0x7e,0xb6,0x0a,0x7e]
 
 v_floor_f16 v5, exec_hi
-// GFX12: v_floor_f16_e32 v5, exec_hi             ; encoding: [0x7f,0xb6,0x0a,0x7e]
+// GFX12: v_floor_f16_e32 v5, exec_hi ; encoding: [0x7f,0xb6,0x0a,0x7e]
 
 v_floor_f16 v5, null
-// GFX12: v_floor_f16_e32 v5, null                ; encoding: [0x7c,0xb6,0x0a,0x7e]
+// GFX12: v_floor_f16_e32 v5, null ; encoding: [0x7c,0xb6,0x0a,0x7e]
 
 v_floor_f16 v5, -1
-// GFX12: v_floor_f16_e32 v5, -1                  ; encoding: [0xc1,0xb6,0x0a,0x7e]
+// GFX12: v_floor_f16_e32 v5, -1 ; encoding: [0xc1,0xb6,0x0a,0x7e]
 
 v_floor_f16 v5, 0.5
-// GFX12: v_floor_f16_e32 v5, 0.5                 ; encoding: [0xf0,0xb6,0x0a,0x7e]
+// GFX12: v_floor_f16_e32 v5, 0.5 ; encoding: [0xf0,0xb6,0x0a,0x7e]
 
 v_floor_f16 v5, src_scc
-// GFX12: v_floor_f16_e32 v5, src_scc             ; encoding: [0xfd,0xb6,0x0a,0x7e]
+// GFX12: v_floor_f16_e32 v5, src_scc ; encoding: [0xfd,0xb6,0x0a,0x7e]
 
 v_floor_f16 v127, 0xfe0b
-// GFX12: v_floor_f16_e32 v127, 0xfe0b            ; encoding: [0xff,0xb6,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_floor_f16_e32 v127, 0xfe0b ; encoding: [0xff,0xb6,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_floor_f32 v5, v1
-// GFX12: v_floor_f32_e32 v5, v1                  ; encoding: [0x01,0x49,0x0a,0x7e]
+// GFX12: v_floor_f32_e32 v5, v1 ; encoding: [0x01,0x49,0x0a,0x7e]
 
 v_floor_f32 v5, v255
-// GFX12: v_floor_f32_e32 v5, v255                ; encoding: [0xff,0x49,0x0a,0x7e]
+// GFX12: v_floor_f32_e32 v5, v255 ; encoding: [0xff,0x49,0x0a,0x7e]
 
 v_floor_f32 v5, s1
-// GFX12: v_floor_f32_e32 v5, s1                  ; encoding: [0x01,0x48,0x0a,0x7e]
+// GFX12: v_floor_f32_e32 v5, s1 ; encoding: [0x01,0x48,0x0a,0x7e]
 
 v_floor_f32 v5, s105
-// GFX12: v_floor_f32_e32 v5, s105                ; encoding: [0x69,0x48,0x0a,0x7e]
+// GFX12: v_floor_f32_e32 v5, s105 ; encoding: [0x69,0x48,0x0a,0x7e]
 
 v_floor_f32 v5, vcc_lo
-// GFX12: v_floor_f32_e32 v5, vcc_lo              ; encoding: [0x6a,0x48,0x0a,0x7e]
+// GFX12: v_floor_f32_e32 v5, vcc_lo ; encoding: [0x6a,0x48,0x0a,0x7e]
 
 v_floor_f32 v5, vcc_hi
-// GFX12: v_floor_f32_e32 v5, vcc_hi              ; encoding: [0x6b,0x48,0x0a,0x7e]
+// GFX12: v_floor_f32_e32 v5, vcc_hi ; encoding: [0x6b,0x48,0x0a,0x7e]
 
 v_floor_f32 v5, ttmp15
-// GFX12: v_floor_f32_e32 v5, ttmp15              ; encoding: [0x7b,0x48,0x0a,0x7e]
+// GFX12: v_floor_f32_e32 v5, ttmp15 ; encoding: [0x7b,0x48,0x0a,0x7e]
 
 v_floor_f32 v5, m0
-// GFX12: v_floor_f32_e32 v5, m0                  ; encoding: [0x7d,0x48,0x0a,0x7e]
+// GFX12: v_floor_f32_e32 v5, m0 ; encoding: [0x7d,0x48,0x0a,0x7e]
 
 v_floor_f32 v5, exec_lo
-// GFX12: v_floor_f32_e32 v5, exec_lo             ; encoding: [0x7e,0x48,0x0a,0x7e]
+// GFX12: v_floor_f32_e32 v5, exec_lo ; encoding: [0x7e,0x48,0x0a,0x7e]
 
 v_floor_f32 v5, exec_hi
-// GFX12: v_floor_f32_e32 v5, exec_hi             ; encoding: [0x7f,0x48,0x0a,0x7e]
+// GFX12: v_floor_f32_e32 v5, exec_hi ; encoding: [0x7f,0x48,0x0a,0x7e]
 
 v_floor_f32 v5, null
-// GFX12: v_floor_f32_e32 v5, null                ; encoding: [0x7c,0x48,0x0a,0x7e]
+// GFX12: v_floor_f32_e32 v5, null ; encoding: [0x7c,0x48,0x0a,0x7e]
 
 v_floor_f32 v5, -1
-// GFX12: v_floor_f32_e32 v5, -1                  ; encoding: [0xc1,0x48,0x0a,0x7e]
+// GFX12: v_floor_f32_e32 v5, -1 ; encoding: [0xc1,0x48,0x0a,0x7e]
 
 v_floor_f32 v5, 0.5
-// GFX12: v_floor_f32_e32 v5, 0.5                 ; encoding: [0xf0,0x48,0x0a,0x7e]
+// GFX12: v_floor_f32_e32 v5, 0.5 ; encoding: [0xf0,0x48,0x0a,0x7e]
 
 v_floor_f32 v5, src_scc
-// GFX12: v_floor_f32_e32 v5, src_scc             ; encoding: [0xfd,0x48,0x0a,0x7e]
+// GFX12: v_floor_f32_e32 v5, src_scc ; encoding: [0xfd,0x48,0x0a,0x7e]
 
 v_floor_f32 v255, 0xaf123456
-// GFX12: v_floor_f32_e32 v255, 0xaf123456        ; encoding: [0xff,0x48,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_floor_f32_e32 v255, 0xaf123456 ; encoding: [0xff,0x48,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_floor_f64 v[5:6], v[1:2]
-// GFX12: v_floor_f64_e32 v[5:6], v[1:2]          ; encoding: [0x01,0x35,0x0a,0x7e]
+// GFX12: v_floor_f64_e32 v[5:6], v[1:2] ; encoding: [0x01,0x35,0x0a,0x7e]
 
 v_floor_f64 v[5:6], v[254:255]
-// GFX12: v_floor_f64_e32 v[5:6], v[254:255]      ; encoding: [0xfe,0x35,0x0a,0x7e]
+// GFX12: v_floor_f64_e32 v[5:6], v[254:255] ; encoding: [0xfe,0x35,0x0a,0x7e]
 
 v_floor_f64 v[5:6], s[2:3]
-// GFX12: v_floor_f64_e32 v[5:6], s[2:3]          ; encoding: [0x02,0x34,0x0a,0x7e]
+// GFX12: v_floor_f64_e32 v[5:6], s[2:3] ; encoding: [0x02,0x34,0x0a,0x7e]
 
 v_floor_f64 v[5:6], s[104:105]
-// GFX12: v_floor_f64_e32 v[5:6], s[104:105]      ; encoding: [0x68,0x34,0x0a,0x7e]
+// GFX12: v_floor_f64_e32 v[5:6], s[104:105] ; encoding: [0x68,0x34,0x0a,0x7e]
 
 v_floor_f64 v[5:6], vcc
-// GFX12: v_floor_f64_e32 v[5:6], vcc             ; encoding: [0x6a,0x34,0x0a,0x7e]
+// GFX12: v_floor_f64_e32 v[5:6], vcc ; encoding: [0x6a,0x34,0x0a,0x7e]
 
 v_floor_f64 v[5:6], ttmp[14:15]
-// GFX12: v_floor_f64_e32 v[5:6], ttmp[14:15]     ; encoding: [0x7a,0x34,0x0a,0x7e]
+// GFX12: v_floor_f64_e32 v[5:6], ttmp[14:15] ; encoding: [0x7a,0x34,0x0a,0x7e]
 
 v_floor_f64 v[5:6], exec
-// GFX12: v_floor_f64_e32 v[5:6], exec            ; encoding: [0x7e,0x34,0x0a,0x7e]
+// GFX12: v_floor_f64_e32 v[5:6], exec ; encoding: [0x7e,0x34,0x0a,0x7e]
 
 v_floor_f64 v[5:6], null
-// GFX12: v_floor_f64_e32 v[5:6], null            ; encoding: [0x7c,0x34,0x0a,0x7e]
+// GFX12: v_floor_f64_e32 v[5:6], null ; encoding: [0x7c,0x34,0x0a,0x7e]
 
 v_floor_f64 v[5:6], -1
-// GFX12: v_floor_f64_e32 v[5:6], -1              ; encoding: [0xc1,0x34,0x0a,0x7e]
+// GFX12: v_floor_f64_e32 v[5:6], -1 ; encoding: [0xc1,0x34,0x0a,0x7e]
 
 v_floor_f64 v[5:6], 0.5
-// GFX12: v_floor_f64_e32 v[5:6], 0.5             ; encoding: [0xf0,0x34,0x0a,0x7e]
+// GFX12: v_floor_f64_e32 v[5:6], 0.5 ; encoding: [0xf0,0x34,0x0a,0x7e]
 
 v_floor_f64 v[5:6], src_scc
-// GFX12: v_floor_f64_e32 v[5:6], src_scc         ; encoding: [0xfd,0x34,0x0a,0x7e]
+// GFX12: v_floor_f64_e32 v[5:6], src_scc ; encoding: [0xfd,0x34,0x0a,0x7e]
 
 v_floor_f64 v[254:255], 0xaf123456
-// GFX12: v_floor_f64_e32 v[254:255], 0xaf123456  ; encoding: [0xff,0x34,0xfc,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_floor_f64_e32 v[254:255], 0xaf123456 ; encoding: [0xff,0x34,0xfc,0x7f,0x56,0x34,0x12,0xaf]
 
 v_fract_f16 v5, v1
-// GFX12: v_fract_f16_e32 v5, v1                  ; encoding: [0x01,0xbf,0x0a,0x7e]
+// GFX12: v_fract_f16_e32 v5, v1 ; encoding: [0x01,0xbf,0x0a,0x7e]
 
 v_fract_f16 v5, v127
-// GFX12: v_fract_f16_e32 v5, v127                ; encoding: [0x7f,0xbf,0x0a,0x7e]
+// GFX12: v_fract_f16_e32 v5, v127 ; encoding: [0x7f,0xbf,0x0a,0x7e]
 
 v_fract_f16 v5, s1
-// GFX12: v_fract_f16_e32 v5, s1                  ; encoding: [0x01,0xbe,0x0a,0x7e]
+// GFX12: v_fract_f16_e32 v5, s1 ; encoding: [0x01,0xbe,0x0a,0x7e]
 
 v_fract_f16 v5, s105
-// GFX12: v_fract_f16_e32 v5, s105                ; encoding: [0x69,0xbe,0x0a,0x7e]
+// GFX12: v_fract_f16_e32 v5, s105 ; encoding: [0x69,0xbe,0x0a,0x7e]
 
 v_fract_f16 v5, vcc_lo
-// GFX12: v_fract_f16_e32 v5, vcc_lo              ; encoding: [0x6a,0xbe,0x0a,0x7e]
+// GFX12: v_fract_f16_e32 v5, vcc_lo ; encoding: [0x6a,0xbe,0x0a,0x7e]
 
 v_fract_f16 v5, vcc_hi
-// GFX12: v_fract_f16_e32 v5, vcc_hi              ; encoding: [0x6b,0xbe,0x0a,0x7e]
+// GFX12: v_fract_f16_e32 v5, vcc_hi ; encoding: [0x6b,0xbe,0x0a,0x7e]
 
 v_fract_f16 v5, ttmp15
-// GFX12: v_fract_f16_e32 v5, ttmp15              ; encoding: [0x7b,0xbe,0x0a,0x7e]
+// GFX12: v_fract_f16_e32 v5, ttmp15 ; encoding: [0x7b,0xbe,0x0a,0x7e]
 
 v_fract_f16 v5, m0
-// GFX12: v_fract_f16_e32 v5, m0                  ; encoding: [0x7d,0xbe,0x0a,0x7e]
+// GFX12: v_fract_f16_e32 v5, m0 ; encoding: [0x7d,0xbe,0x0a,0x7e]
 
 v_fract_f16 v5, exec_lo
-// GFX12: v_fract_f16_e32 v5, exec_lo             ; encoding: [0x7e,0xbe,0x0a,0x7e]
+// GFX12: v_fract_f16_e32 v5, exec_lo ; encoding: [0x7e,0xbe,0x0a,0x7e]
 
 v_fract_f16 v5, exec_hi
-// GFX12: v_fract_f16_e32 v5, exec_hi             ; encoding: [0x7f,0xbe,0x0a,0x7e]
+// GFX12: v_fract_f16_e32 v5, exec_hi ; encoding: [0x7f,0xbe,0x0a,0x7e]
 
 v_fract_f16 v5, null
-// GFX12: v_fract_f16_e32 v5, null                ; encoding: [0x7c,0xbe,0x0a,0x7e]
+// GFX12: v_fract_f16_e32 v5, null ; encoding: [0x7c,0xbe,0x0a,0x7e]
 
 v_fract_f16 v5, -1
-// GFX12: v_fract_f16_e32 v5, -1                  ; encoding: [0xc1,0xbe,0x0a,0x7e]
+// GFX12: v_fract_f16_e32 v5, -1 ; encoding: [0xc1,0xbe,0x0a,0x7e]
 
 v_fract_f16 v5, 0.5
-// GFX12: v_fract_f16_e32 v5, 0.5                 ; encoding: [0xf0,0xbe,0x0a,0x7e]
+// GFX12: v_fract_f16_e32 v5, 0.5 ; encoding: [0xf0,0xbe,0x0a,0x7e]
 
 v_fract_f16 v5, src_scc
-// GFX12: v_fract_f16_e32 v5, src_scc             ; encoding: [0xfd,0xbe,0x0a,0x7e]
+// GFX12: v_fract_f16_e32 v5, src_scc ; encoding: [0xfd,0xbe,0x0a,0x7e]
 
 v_fract_f16 v127, 0xfe0b
-// GFX12: v_fract_f16_e32 v127, 0xfe0b            ; encoding: [0xff,0xbe,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_fract_f16_e32 v127, 0xfe0b ; encoding: [0xff,0xbe,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_fract_f32 v5, v1
-// GFX12: v_fract_f32_e32 v5, v1                  ; encoding: [0x01,0x41,0x0a,0x7e]
+// GFX12: v_fract_f32_e32 v5, v1 ; encoding: [0x01,0x41,0x0a,0x7e]
 
 v_fract_f32 v5, v255
-// GFX12: v_fract_f32_e32 v5, v255                ; encoding: [0xff,0x41,0x0a,0x7e]
+// GFX12: v_fract_f32_e32 v5, v255 ; encoding: [0xff,0x41,0x0a,0x7e]
 
 v_fract_f32 v5, s1
-// GFX12: v_fract_f32_e32 v5, s1                  ; encoding: [0x01,0x40,0x0a,0x7e]
+// GFX12: v_fract_f32_e32 v5, s1 ; encoding: [0x01,0x40,0x0a,0x7e]
 
 v_fract_f32 v5, s105
-// GFX12: v_fract_f32_e32 v5, s105                ; encoding: [0x69,0x40,0x0a,0x7e]
+// GFX12: v_fract_f32_e32 v5, s105 ; encoding: [0x69,0x40,0x0a,0x7e]
 
 v_fract_f32 v5, vcc_lo
-// GFX12: v_fract_f32_e32 v5, vcc_lo              ; encoding: [0x6a,0x40,0x0a,0x7e]
+// GFX12: v_fract_f32_e32 v5, vcc_lo ; encoding: [0x6a,0x40,0x0a,0x7e]
 
 v_fract_f32 v5, vcc_hi
-// GFX12: v_fract_f32_e32 v5, vcc_hi              ; encoding: [0x6b,0x40,0x0a,0x7e]
+// GFX12: v_fract_f32_e32 v5, vcc_hi ; encoding: [0x6b,0x40,0x0a,0x7e]
 
 v_fract_f32 v5, ttmp15
-// GFX12: v_fract_f32_e32 v5, ttmp15              ; encoding: [0x7b,0x40,0x0a,0x7e]
+// GFX12: v_fract_f32_e32 v5, ttmp15 ; encoding: [0x7b,0x40,0x0a,0x7e]
 
 v_fract_f32 v5, m0
-// GFX12: v_fract_f32_e32 v5, m0                  ; encoding: [0x7d,0x40,0x0a,0x7e]
+// GFX12: v_fract_f32_e32 v5, m0 ; encoding: [0x7d,0x40,0x0a,0x7e]
 
 v_fract_f32 v5, exec_lo
-// GFX12: v_fract_f32_e32 v5, exec_lo             ; encoding: [0x7e,0x40,0x0a,0x7e]
+// GFX12: v_fract_f32_e32 v5, exec_lo ; encoding: [0x7e,0x40,0x0a,0x7e]
 
 v_fract_f32 v5, exec_hi
-// GFX12: v_fract_f32_e32 v5, exec_hi             ; encoding: [0x7f,0x40,0x0a,0x7e]
+// GFX12: v_fract_f32_e32 v5, exec_hi ; encoding: [0x7f,0x40,0x0a,0x7e]
 
 v_fract_f32 v5, null
-// GFX12: v_fract_f32_e32 v5, null                ; encoding: [0x7c,0x40,0x0a,0x7e]
+// GFX12: v_fract_f32_e32 v5, null ; encoding: [0x7c,0x40,0x0a,0x7e]
 
 v_fract_f32 v5, -1
-// GFX12: v_fract_f32_e32 v5, -1                  ; encoding: [0xc1,0x40,0x0a,0x7e]
+// GFX12: v_fract_f32_e32 v5, -1 ; encoding: [0xc1,0x40,0x0a,0x7e]
 
 v_fract_f32 v5, 0.5
-// GFX12: v_fract_f32_e32 v5, 0.5                 ; encoding: [0xf0,0x40,0x0a,0x7e]
+// GFX12: v_fract_f32_e32 v5, 0.5 ; encoding: [0xf0,0x40,0x0a,0x7e]
 
 v_fract_f32 v5, src_scc
-// GFX12: v_fract_f32_e32 v5, src_scc             ; encoding: [0xfd,0x40,0x0a,0x7e]
+// GFX12: v_fract_f32_e32 v5, src_scc ; encoding: [0xfd,0x40,0x0a,0x7e]
 
 v_fract_f32 v255, 0xaf123456
-// GFX12: v_fract_f32_e32 v255, 0xaf123456        ; encoding: [0xff,0x40,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_fract_f32_e32 v255, 0xaf123456 ; encoding: [0xff,0x40,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_fract_f64 v[5:6], v[1:2]
-// GFX12: v_fract_f64_e32 v[5:6], v[1:2]          ; encoding: [0x01,0x7d,0x0a,0x7e]
+// GFX12: v_fract_f64_e32 v[5:6], v[1:2] ; encoding: [0x01,0x7d,0x0a,0x7e]
 
 v_fract_f64 v[5:6], v[254:255]
-// GFX12: v_fract_f64_e32 v[5:6], v[254:255]      ; encoding: [0xfe,0x7d,0x0a,0x7e]
+// GFX12: v_fract_f64_e32 v[5:6], v[254:255] ; encoding: [0xfe,0x7d,0x0a,0x7e]
 
 v_fract_f64 v[5:6], s[2:3]
-// GFX12: v_fract_f64_e32 v[5:6], s[2:3]          ; encoding: [0x02,0x7c,0x0a,0x7e]
+// GFX12: v_fract_f64_e32 v[5:6], s[2:3] ; encoding: [0x02,0x7c,0x0a,0x7e]
 
 v_fract_f64 v[5:6], s[104:105]
-// GFX12: v_fract_f64_e32 v[5:6], s[104:105]      ; encoding: [0x68,0x7c,0x0a,0x7e]
+// GFX12: v_fract_f64_e32 v[5:6], s[104:105] ; encoding: [0x68,0x7c,0x0a,0x7e]
 
 v_fract_f64 v[5:6], vcc
-// GFX12: v_fract_f64_e32 v[5:6], vcc             ; encoding: [0x6a,0x7c,0x0a,0x7e]
+// GFX12: v_fract_f64_e32 v[5:6], vcc ; encoding: [0x6a,0x7c,0x0a,0x7e]
 
 v_fract_f64 v[5:6], ttmp[14:15]
-// GFX12: v_fract_f64_e32 v[5:6], ttmp[14:15]     ; encoding: [0x7a,0x7c,0x0a,0x7e]
+// GFX12: v_fract_f64_e32 v[5:6], ttmp[14:15] ; encoding: [0x7a,0x7c,0x0a,0x7e]
 
 v_fract_f64 v[5:6], exec
-// GFX12: v_fract_f64_e32 v[5:6], exec            ; encoding: [0x7e,0x7c,0x0a,0x7e]
+// GFX12: v_fract_f64_e32 v[5:6], exec ; encoding: [0x7e,0x7c,0x0a,0x7e]
 
 v_fract_f64 v[5:6], null
-// GFX12: v_fract_f64_e32 v[5:6], null            ; encoding: [0x7c,0x7c,0x0a,0x7e]
+// GFX12: v_fract_f64_e32 v[5:6], null ; encoding: [0x7c,0x7c,0x0a,0x7e]
 
 v_fract_f64 v[5:6], -1
-// GFX12: v_fract_f64_e32 v[5:6], -1              ; encoding: [0xc1,0x7c,0x0a,0x7e]
+// GFX12: v_fract_f64_e32 v[5:6], -1 ; encoding: [0xc1,0x7c,0x0a,0x7e]
 
 v_fract_f64 v[5:6], 0.5
-// GFX12: v_fract_f64_e32 v[5:6], 0.5             ; encoding: [0xf0,0x7c,0x0a,0x7e]
+// GFX12: v_fract_f64_e32 v[5:6], 0.5 ; encoding: [0xf0,0x7c,0x0a,0x7e]
 
 v_fract_f64 v[5:6], src_scc
-// GFX12: v_fract_f64_e32 v[5:6], src_scc         ; encoding: [0xfd,0x7c,0x0a,0x7e]
+// GFX12: v_fract_f64_e32 v[5:6], src_scc ; encoding: [0xfd,0x7c,0x0a,0x7e]
 
 v_fract_f64 v[254:255], 0xaf123456
-// GFX12: v_fract_f64_e32 v[254:255], 0xaf123456  ; encoding: [0xff,0x7c,0xfc,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_fract_f64_e32 v[254:255], 0xaf123456 ; encoding: [0xff,0x7c,0xfc,0x7f,0x56,0x34,0x12,0xaf]
 
 v_frexp_exp_i16_f16 v5, v1
-// GFX12: v_frexp_exp_i16_f16_e32 v5, v1          ; encoding: [0x01,0xb5,0x0a,0x7e]
+// GFX12: v_frexp_exp_i16_f16_e32 v5, v1 ; encoding: [0x01,0xb5,0x0a,0x7e]
 
 v_frexp_exp_i16_f16 v5, v127
-// GFX12: v_frexp_exp_i16_f16_e32 v5, v127        ; encoding: [0x7f,0xb5,0x0a,0x7e]
+// GFX12: v_frexp_exp_i16_f16_e32 v5, v127 ; encoding: [0x7f,0xb5,0x0a,0x7e]
 
 v_frexp_exp_i16_f16 v5, s1
-// GFX12: v_frexp_exp_i16_f16_e32 v5, s1          ; encoding: [0x01,0xb4,0x0a,0x7e]
+// GFX12: v_frexp_exp_i16_f16_e32 v5, s1 ; encoding: [0x01,0xb4,0x0a,0x7e]
 
 v_frexp_exp_i16_f16 v5, s105
-// GFX12: v_frexp_exp_i16_f16_e32 v5, s105        ; encoding: [0x69,0xb4,0x0a,0x7e]
+// GFX12: v_frexp_exp_i16_f16_e32 v5, s105 ; encoding: [0x69,0xb4,0x0a,0x7e]
 
 v_frexp_exp_i16_f16 v5, vcc_lo
-// GFX12: v_frexp_exp_i16_f16_e32 v5, vcc_lo      ; encoding: [0x6a,0xb4,0x0a,0x7e]
+// GFX12: v_frexp_exp_i16_f16_e32 v5, vcc_lo ; encoding: [0x6a,0xb4,0x0a,0x7e]
 
 v_frexp_exp_i16_f16 v5, vcc_hi
-// GFX12: v_frexp_exp_i16_f16_e32 v5, vcc_hi      ; encoding: [0x6b,0xb4,0x0a,0x7e]
+// GFX12: v_frexp_exp_i16_f16_e32 v5, vcc_hi ; encoding: [0x6b,0xb4,0x0a,0x7e]
 
 v_frexp_exp_i16_f16 v5, ttmp15
-// GFX12: v_frexp_exp_i16_f16_e32 v5, ttmp15      ; encoding: [0x7b,0xb4,0x0a,0x7e]
+// GFX12: v_frexp_exp_i16_f16_e32 v5, ttmp15 ; encoding: [0x7b,0xb4,0x0a,0x7e]
 
 v_frexp_exp_i16_f16 v5, m0
-// GFX12: v_frexp_exp_i16_f16_e32 v5, m0          ; encoding: [0x7d,0xb4,0x0a,0x7e]
+// GFX12: v_frexp_exp_i16_f16_e32 v5, m0 ; encoding: [0x7d,0xb4,0x0a,0x7e]
 
 v_frexp_exp_i16_f16 v5, exec_lo
-// GFX12: v_frexp_exp_i16_f16_e32 v5, exec_lo     ; encoding: [0x7e,0xb4,0x0a,0x7e]
+// GFX12: v_frexp_exp_i16_f16_e32 v5, exec_lo ; encoding: [0x7e,0xb4,0x0a,0x7e]
 
 v_frexp_exp_i16_f16 v5, exec_hi
-// GFX12: v_frexp_exp_i16_f16_e32 v5, exec_hi     ; encoding: [0x7f,0xb4,0x0a,0x7e]
+// GFX12: v_frexp_exp_i16_f16_e32 v5, exec_hi ; encoding: [0x7f,0xb4,0x0a,0x7e]
 
 v_frexp_exp_i16_f16 v5, null
-// GFX12: v_frexp_exp_i16_f16_e32 v5, null        ; encoding: [0x7c,0xb4,0x0a,0x7e]
+// GFX12: v_frexp_exp_i16_f16_e32 v5, null ; encoding: [0x7c,0xb4,0x0a,0x7e]
 
 v_frexp_exp_i16_f16 v5, -1
-// GFX12: v_frexp_exp_i16_f16_e32 v5, -1          ; encoding: [0xc1,0xb4,0x0a,0x7e]
+// GFX12: v_frexp_exp_i16_f16_e32 v5, -1 ; encoding: [0xc1,0xb4,0x0a,0x7e]
 
 v_frexp_exp_i16_f16 v5, 0.5
-// GFX12: v_frexp_exp_i16_f16_e32 v5, 0.5         ; encoding: [0xf0,0xb4,0x0a,0x7e]
+// GFX12: v_frexp_exp_i16_f16_e32 v5, 0.5 ; encoding: [0xf0,0xb4,0x0a,0x7e]
 
 v_frexp_exp_i16_f16 v5, src_scc
-// GFX12: v_frexp_exp_i16_f16_e32 v5, src_scc     ; encoding: [0xfd,0xb4,0x0a,0x7e]
+// GFX12: v_frexp_exp_i16_f16_e32 v5, src_scc ; encoding: [0xfd,0xb4,0x0a,0x7e]
 
 v_frexp_exp_i16_f16 v127, 0xfe0b
-// GFX12: v_frexp_exp_i16_f16_e32 v127, 0xfe0b    ; encoding: [0xff,0xb4,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_frexp_exp_i16_f16_e32 v127, 0xfe0b ; encoding: [0xff,0xb4,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_frexp_exp_i32_f32 v5, v1
-// GFX12: v_frexp_exp_i32_f32_e32 v5, v1          ; encoding: [0x01,0x7f,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f32_e32 v5, v1 ; encoding: [0x01,0x7f,0x0a,0x7e]
 
 v_frexp_exp_i32_f32 v5, v255
-// GFX12: v_frexp_exp_i32_f32_e32 v5, v255        ; encoding: [0xff,0x7f,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f32_e32 v5, v255 ; encoding: [0xff,0x7f,0x0a,0x7e]
 
 v_frexp_exp_i32_f32 v5, s1
-// GFX12: v_frexp_exp_i32_f32_e32 v5, s1          ; encoding: [0x01,0x7e,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f32_e32 v5, s1 ; encoding: [0x01,0x7e,0x0a,0x7e]
 
 v_frexp_exp_i32_f32 v5, s105
-// GFX12: v_frexp_exp_i32_f32_e32 v5, s105        ; encoding: [0x69,0x7e,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f32_e32 v5, s105 ; encoding: [0x69,0x7e,0x0a,0x7e]
 
 v_frexp_exp_i32_f32 v5, vcc_lo
-// GFX12: v_frexp_exp_i32_f32_e32 v5, vcc_lo      ; encoding: [0x6a,0x7e,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f32_e32 v5, vcc_lo ; encoding: [0x6a,0x7e,0x0a,0x7e]
 
 v_frexp_exp_i32_f32 v5, vcc_hi
-// GFX12: v_frexp_exp_i32_f32_e32 v5, vcc_hi      ; encoding: [0x6b,0x7e,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f32_e32 v5, vcc_hi ; encoding: [0x6b,0x7e,0x0a,0x7e]
 
 v_frexp_exp_i32_f32 v5, ttmp15
-// GFX12: v_frexp_exp_i32_f32_e32 v5, ttmp15      ; encoding: [0x7b,0x7e,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f32_e32 v5, ttmp15 ; encoding: [0x7b,0x7e,0x0a,0x7e]
 
 v_frexp_exp_i32_f32 v5, m0
-// GFX12: v_frexp_exp_i32_f32_e32 v5, m0          ; encoding: [0x7d,0x7e,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f32_e32 v5, m0 ; encoding: [0x7d,0x7e,0x0a,0x7e]
 
 v_frexp_exp_i32_f32 v5, exec_lo
-// GFX12: v_frexp_exp_i32_f32_e32 v5, exec_lo     ; encoding: [0x7e,0x7e,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f32_e32 v5, exec_lo ; encoding: [0x7e,0x7e,0x0a,0x7e]
 
 v_frexp_exp_i32_f32 v5, exec_hi
-// GFX12: v_frexp_exp_i32_f32_e32 v5, exec_hi     ; encoding: [0x7f,0x7e,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f32_e32 v5, exec_hi ; encoding: [0x7f,0x7e,0x0a,0x7e]
 
 v_frexp_exp_i32_f32 v5, null
-// GFX12: v_frexp_exp_i32_f32_e32 v5, null        ; encoding: [0x7c,0x7e,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f32_e32 v5, null ; encoding: [0x7c,0x7e,0x0a,0x7e]
 
 v_frexp_exp_i32_f32 v5, -1
-// GFX12: v_frexp_exp_i32_f32_e32 v5, -1          ; encoding: [0xc1,0x7e,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f32_e32 v5, -1 ; encoding: [0xc1,0x7e,0x0a,0x7e]
 
 v_frexp_exp_i32_f32 v5, 0.5
-// GFX12: v_frexp_exp_i32_f32_e32 v5, 0.5         ; encoding: [0xf0,0x7e,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f32_e32 v5, 0.5 ; encoding: [0xf0,0x7e,0x0a,0x7e]
 
 v_frexp_exp_i32_f32 v5, src_scc
-// GFX12: v_frexp_exp_i32_f32_e32 v5, src_scc     ; encoding: [0xfd,0x7e,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f32_e32 v5, src_scc ; encoding: [0xfd,0x7e,0x0a,0x7e]
 
 v_frexp_exp_i32_f32 v255, 0xaf123456
 // GFX12: v_frexp_exp_i32_f32_e32 v255, 0xaf123456 ; encoding: [0xff,0x7e,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_frexp_exp_i32_f64 v5, v[1:2]
-// GFX12: v_frexp_exp_i32_f64_e32 v5, v[1:2]      ; encoding: [0x01,0x79,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f64_e32 v5, v[1:2] ; encoding: [0x01,0x79,0x0a,0x7e]
 
 v_frexp_exp_i32_f64 v5, v[254:255]
-// GFX12: v_frexp_exp_i32_f64_e32 v5, v[254:255]  ; encoding: [0xfe,0x79,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f64_e32 v5, v[254:255] ; encoding: [0xfe,0x79,0x0a,0x7e]
 
 v_frexp_exp_i32_f64 v5, s[2:3]
-// GFX12: v_frexp_exp_i32_f64_e32 v5, s[2:3]      ; encoding: [0x02,0x78,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f64_e32 v5, s[2:3] ; encoding: [0x02,0x78,0x0a,0x7e]
 
 v_frexp_exp_i32_f64 v5, s[104:105]
-// GFX12: v_frexp_exp_i32_f64_e32 v5, s[104:105]  ; encoding: [0x68,0x78,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f64_e32 v5, s[104:105] ; encoding: [0x68,0x78,0x0a,0x7e]
 
 v_frexp_exp_i32_f64 v5, vcc
-// GFX12: v_frexp_exp_i32_f64_e32 v5, vcc         ; encoding: [0x6a,0x78,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f64_e32 v5, vcc ; encoding: [0x6a,0x78,0x0a,0x7e]
 
 v_frexp_exp_i32_f64 v5, ttmp[14:15]
 // GFX12: v_frexp_exp_i32_f64_e32 v5, ttmp[14:15] ; encoding: [0x7a,0x78,0x0a,0x7e]
 
 v_frexp_exp_i32_f64 v5, exec
-// GFX12: v_frexp_exp_i32_f64_e32 v5, exec        ; encoding: [0x7e,0x78,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f64_e32 v5, exec ; encoding: [0x7e,0x78,0x0a,0x7e]
 
 v_frexp_exp_i32_f64 v5, null
-// GFX12: v_frexp_exp_i32_f64_e32 v5, null        ; encoding: [0x7c,0x78,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f64_e32 v5, null ; encoding: [0x7c,0x78,0x0a,0x7e]
 
 v_frexp_exp_i32_f64 v5, -1
-// GFX12: v_frexp_exp_i32_f64_e32 v5, -1          ; encoding: [0xc1,0x78,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f64_e32 v5, -1 ; encoding: [0xc1,0x78,0x0a,0x7e]
 
 v_frexp_exp_i32_f64 v5, 0.5
-// GFX12: v_frexp_exp_i32_f64_e32 v5, 0.5         ; encoding: [0xf0,0x78,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f64_e32 v5, 0.5 ; encoding: [0xf0,0x78,0x0a,0x7e]
 
 v_frexp_exp_i32_f64 v5, src_scc
-// GFX12: v_frexp_exp_i32_f64_e32 v5, src_scc     ; encoding: [0xfd,0x78,0x0a,0x7e]
+// GFX12: v_frexp_exp_i32_f64_e32 v5, src_scc ; encoding: [0xfd,0x78,0x0a,0x7e]
 
 v_frexp_exp_i32_f64 v255, 0xaf123456
 // GFX12: v_frexp_exp_i32_f64_e32 v255, 0xaf123456 ; encoding: [0xff,0x78,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_frexp_mant_f16 v5, v1
-// GFX12: v_frexp_mant_f16_e32 v5, v1             ; encoding: [0x01,0xb3,0x0a,0x7e]
+// GFX12: v_frexp_mant_f16_e32 v5, v1 ; encoding: [0x01,0xb3,0x0a,0x7e]
 
 v_frexp_mant_f16 v5, v127
-// GFX12: v_frexp_mant_f16_e32 v5, v127           ; encoding: [0x7f,0xb3,0x0a,0x7e]
+// GFX12: v_frexp_mant_f16_e32 v5, v127 ; encoding: [0x7f,0xb3,0x0a,0x7e]
 
 v_frexp_mant_f16 v5, s1
-// GFX12: v_frexp_mant_f16_e32 v5, s1             ; encoding: [0x01,0xb2,0x0a,0x7e]
+// GFX12: v_frexp_mant_f16_e32 v5, s1 ; encoding: [0x01,0xb2,0x0a,0x7e]
 
 v_frexp_mant_f16 v5, s105
-// GFX12: v_frexp_mant_f16_e32 v5, s105           ; encoding: [0x69,0xb2,0x0a,0x7e]
+// GFX12: v_frexp_mant_f16_e32 v5, s105 ; encoding: [0x69,0xb2,0x0a,0x7e]
 
 v_frexp_mant_f16 v5, vcc_lo
-// GFX12: v_frexp_mant_f16_e32 v5, vcc_lo         ; encoding: [0x6a,0xb2,0x0a,0x7e]
+// GFX12: v_frexp_mant_f16_e32 v5, vcc_lo ; encoding: [0x6a,0xb2,0x0a,0x7e]
 
 v_frexp_mant_f16 v5, vcc_hi
-// GFX12: v_frexp_mant_f16_e32 v5, vcc_hi         ; encoding: [0x6b,0xb2,0x0a,0x7e]
+// GFX12: v_frexp_mant_f16_e32 v5, vcc_hi ; encoding: [0x6b,0xb2,0x0a,0x7e]
 
 v_frexp_mant_f16 v5, ttmp15
-// GFX12: v_frexp_mant_f16_e32 v5, ttmp15         ; encoding: [0x7b,0xb2,0x0a,0x7e]
+// GFX12: v_frexp_mant_f16_e32 v5, ttmp15 ; encoding: [0x7b,0xb2,0x0a,0x7e]
 
 v_frexp_mant_f16 v5, m0
-// GFX12: v_frexp_mant_f16_e32 v5, m0             ; encoding: [0x7d,0xb2,0x0a,0x7e]
+// GFX12: v_frexp_mant_f16_e32 v5, m0 ; encoding: [0x7d,0xb2,0x0a,0x7e]
 
 v_frexp_mant_f16 v5, exec_lo
-// GFX12: v_frexp_mant_f16_e32 v5, exec_lo        ; encoding: [0x7e,0xb2,0x0a,0x7e]
+// GFX12: v_frexp_mant_f16_e32 v5, exec_lo ; encoding: [0x7e,0xb2,0x0a,0x7e]
 
 v_frexp_mant_f16 v5, exec_hi
-// GFX12: v_frexp_mant_f16_e32 v5, exec_hi        ; encoding: [0x7f,0xb2,0x0a,0x7e]
+// GFX12: v_frexp_mant_f16_e32 v5, exec_hi ; encoding: [0x7f,0xb2,0x0a,0x7e]
 
 v_frexp_mant_f16 v5, null
-// GFX12: v_frexp_mant_f16_e32 v5, null           ; encoding: [0x7c,0xb2,0x0a,0x7e]
+// GFX12: v_frexp_mant_f16_e32 v5, null ; encoding: [0x7c,0xb2,0x0a,0x7e]
 
 v_frexp_mant_f16 v5, -1
-// GFX12: v_frexp_mant_f16_e32 v5, -1             ; encoding: [0xc1,0xb2,0x0a,0x7e]
+// GFX12: v_frexp_mant_f16_e32 v5, -1 ; encoding: [0xc1,0xb2,0x0a,0x7e]
 
 v_frexp_mant_f16 v5, 0.5
-// GFX12: v_frexp_mant_f16_e32 v5, 0.5            ; encoding: [0xf0,0xb2,0x0a,0x7e]
+// GFX12: v_frexp_mant_f16_e32 v5, 0.5 ; encoding: [0xf0,0xb2,0x0a,0x7e]
 
 v_frexp_mant_f16 v5, src_scc
-// GFX12: v_frexp_mant_f16_e32 v5, src_scc        ; encoding: [0xfd,0xb2,0x0a,0x7e]
+// GFX12: v_frexp_mant_f16_e32 v5, src_scc ; encoding: [0xfd,0xb2,0x0a,0x7e]
 
 v_frexp_mant_f16 v127, 0xfe0b
-// GFX12: v_frexp_mant_f16_e32 v127, 0xfe0b       ; encoding: [0xff,0xb2,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_frexp_mant_f16_e32 v127, 0xfe0b ; encoding: [0xff,0xb2,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_frexp_mant_f32 v5, v1
-// GFX12: v_frexp_mant_f32_e32 v5, v1             ; encoding: [0x01,0x81,0x0a,0x7e]
+// GFX12: v_frexp_mant_f32_e32 v5, v1 ; encoding: [0x01,0x81,0x0a,0x7e]
 
 v_frexp_mant_f32 v5, v255
-// GFX12: v_frexp_mant_f32_e32 v5, v255           ; encoding: [0xff,0x81,0x0a,0x7e]
+// GFX12: v_frexp_mant_f32_e32 v5, v255 ; encoding: [0xff,0x81,0x0a,0x7e]
 
 v_frexp_mant_f32 v5, s1
-// GFX12: v_frexp_mant_f32_e32 v5, s1             ; encoding: [0x01,0x80,0x0a,0x7e]
+// GFX12: v_frexp_mant_f32_e32 v5, s1 ; encoding: [0x01,0x80,0x0a,0x7e]
 
 v_frexp_mant_f32 v5, s105
-// GFX12: v_frexp_mant_f32_e32 v5, s105           ; encoding: [0x69,0x80,0x0a,0x7e]
+// GFX12: v_frexp_mant_f32_e32 v5, s105 ; encoding: [0x69,0x80,0x0a,0x7e]
 
 v_frexp_mant_f32 v5, vcc_lo
-// GFX12: v_frexp_mant_f32_e32 v5, vcc_lo         ; encoding: [0x6a,0x80,0x0a,0x7e]
+// GFX12: v_frexp_mant_f32_e32 v5, vcc_lo ; encoding: [0x6a,0x80,0x0a,0x7e]
 
 v_frexp_mant_f32 v5, vcc_hi
-// GFX12: v_frexp_mant_f32_e32 v5, vcc_hi         ; encoding: [0x6b,0x80,0x0a,0x7e]
+// GFX12: v_frexp_mant_f32_e32 v5, vcc_hi ; encoding: [0x6b,0x80,0x0a,0x7e]
 
 v_frexp_mant_f32 v5, ttmp15
-// GFX12: v_frexp_mant_f32_e32 v5, ttmp15         ; encoding: [0x7b,0x80,0x0a,0x7e]
+// GFX12: v_frexp_mant_f32_e32 v5, ttmp15 ; encoding: [0x7b,0x80,0x0a,0x7e]
 
 v_frexp_mant_f32 v5, m0
-// GFX12: v_frexp_mant_f32_e32 v5, m0             ; encoding: [0x7d,0x80,0x0a,0x7e]
+// GFX12: v_frexp_mant_f32_e32 v5, m0 ; encoding: [0x7d,0x80,0x0a,0x7e]
 
 v_frexp_mant_f32 v5, exec_lo
-// GFX12: v_frexp_mant_f32_e32 v5, exec_lo        ; encoding: [0x7e,0x80,0x0a,0x7e]
+// GFX12: v_frexp_mant_f32_e32 v5, exec_lo ; encoding: [0x7e,0x80,0x0a,0x7e]
 
 v_frexp_mant_f32 v5, exec_hi
-// GFX12: v_frexp_mant_f32_e32 v5, exec_hi        ; encoding: [0x7f,0x80,0x0a,0x7e]
+// GFX12: v_frexp_mant_f32_e32 v5, exec_hi ; encoding: [0x7f,0x80,0x0a,0x7e]
 
 v_frexp_mant_f32 v5, null
-// GFX12: v_frexp_mant_f32_e32 v5, null           ; encoding: [0x7c,0x80,0x0a,0x7e]
+// GFX12: v_frexp_mant_f32_e32 v5, null ; encoding: [0x7c,0x80,0x0a,0x7e]
 
 v_frexp_mant_f32 v5, -1
-// GFX12: v_frexp_mant_f32_e32 v5, -1             ; encoding: [0xc1,0x80,0x0a,0x7e]
+// GFX12: v_frexp_mant_f32_e32 v5, -1 ; encoding: [0xc1,0x80,0x0a,0x7e]
 
 v_frexp_mant_f32 v5, 0.5
-// GFX12: v_frexp_mant_f32_e32 v5, 0.5            ; encoding: [0xf0,0x80,0x0a,0x7e]
+// GFX12: v_frexp_mant_f32_e32 v5, 0.5 ; encoding: [0xf0,0x80,0x0a,0x7e]
 
 v_frexp_mant_f32 v5, src_scc
-// GFX12: v_frexp_mant_f32_e32 v5, src_scc        ; encoding: [0xfd,0x80,0x0a,0x7e]
+// GFX12: v_frexp_mant_f32_e32 v5, src_scc ; encoding: [0xfd,0x80,0x0a,0x7e]
 
 v_frexp_mant_f32 v255, 0xaf123456
-// GFX12: v_frexp_mant_f32_e32 v255, 0xaf123456   ; encoding: [0xff,0x80,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_frexp_mant_f32_e32 v255, 0xaf123456 ; encoding: [0xff,0x80,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_frexp_mant_f64 v[5:6], v[1:2]
-// GFX12: v_frexp_mant_f64_e32 v[5:6], v[1:2]     ; encoding: [0x01,0x7b,0x0a,0x7e]
+// GFX12: v_frexp_mant_f64_e32 v[5:6], v[1:2] ; encoding: [0x01,0x7b,0x0a,0x7e]
 
 v_frexp_mant_f64 v[5:6], v[254:255]
 // GFX12: v_frexp_mant_f64_e32 v[5:6], v[254:255] ; encoding: [0xfe,0x7b,0x0a,0x7e]
 
 v_frexp_mant_f64 v[5:6], s[2:3]
-// GFX12: v_frexp_mant_f64_e32 v[5:6], s[2:3]     ; encoding: [0x02,0x7a,0x0a,0x7e]
+// GFX12: v_frexp_mant_f64_e32 v[5:6], s[2:3] ; encoding: [0x02,0x7a,0x0a,0x7e]
 
 v_frexp_mant_f64 v[5:6], s[104:105]
 // GFX12: v_frexp_mant_f64_e32 v[5:6], s[104:105] ; encoding: [0x68,0x7a,0x0a,0x7e]
 
 v_frexp_mant_f64 v[5:6], vcc
-// GFX12: v_frexp_mant_f64_e32 v[5:6], vcc        ; encoding: [0x6a,0x7a,0x0a,0x7e]
+// GFX12: v_frexp_mant_f64_e32 v[5:6], vcc ; encoding: [0x6a,0x7a,0x0a,0x7e]
 
 v_frexp_mant_f64 v[5:6], ttmp[14:15]
 // GFX12: v_frexp_mant_f64_e32 v[5:6], ttmp[14:15] ; encoding: [0x7a,0x7a,0x0a,0x7e]
 
 v_frexp_mant_f64 v[5:6], exec
-// GFX12: v_frexp_mant_f64_e32 v[5:6], exec       ; encoding: [0x7e,0x7a,0x0a,0x7e]
+// GFX12: v_frexp_mant_f64_e32 v[5:6], exec ; encoding: [0x7e,0x7a,0x0a,0x7e]
 
 v_frexp_mant_f64 v[5:6], null
-// GFX12: v_frexp_mant_f64_e32 v[5:6], null       ; encoding: [0x7c,0x7a,0x0a,0x7e]
+// GFX12: v_frexp_mant_f64_e32 v[5:6], null ; encoding: [0x7c,0x7a,0x0a,0x7e]
 
 v_frexp_mant_f64 v[5:6], -1
-// GFX12: v_frexp_mant_f64_e32 v[5:6], -1         ; encoding: [0xc1,0x7a,0x0a,0x7e]
+// GFX12: v_frexp_mant_f64_e32 v[5:6], -1 ; encoding: [0xc1,0x7a,0x0a,0x7e]
 
 v_frexp_mant_f64 v[5:6], 0.5
-// GFX12: v_frexp_mant_f64_e32 v[5:6], 0.5        ; encoding: [0xf0,0x7a,0x0a,0x7e]
+// GFX12: v_frexp_mant_f64_e32 v[5:6], 0.5 ; encoding: [0xf0,0x7a,0x0a,0x7e]
 
 v_frexp_mant_f64 v[5:6], src_scc
-// GFX12: v_frexp_mant_f64_e32 v[5:6], src_scc    ; encoding: [0xfd,0x7a,0x0a,0x7e]
+// GFX12: v_frexp_mant_f64_e32 v[5:6], src_scc ; encoding: [0xfd,0x7a,0x0a,0x7e]
 
 v_frexp_mant_f64 v[254:255], 0xaf123456
 // GFX12: v_frexp_mant_f64_e32 v[254:255], 0xaf123456 ; encoding: [0xff,0x7a,0xfc,0x7f,0x56,0x34,0x12,0xaf]
 
 v_log_f16 v5, v1
-// GFX12: v_log_f16_e32 v5, v1                    ; encoding: [0x01,0xaf,0x0a,0x7e]
+// GFX12: v_log_f16_e32 v5, v1 ; encoding: [0x01,0xaf,0x0a,0x7e]
 
 v_log_f16 v5, v127
-// GFX12: v_log_f16_e32 v5, v127                  ; encoding: [0x7f,0xaf,0x0a,0x7e]
+// GFX12: v_log_f16_e32 v5, v127 ; encoding: [0x7f,0xaf,0x0a,0x7e]
 
 v_log_f16 v5, s1
-// GFX12: v_log_f16_e32 v5, s1                    ; encoding: [0x01,0xae,0x0a,0x7e]
+// GFX12: v_log_f16_e32 v5, s1 ; encoding: [0x01,0xae,0x0a,0x7e]
 
 v_log_f16 v5, s105
-// GFX12: v_log_f16_e32 v5, s105                  ; encoding: [0x69,0xae,0x0a,0x7e]
+// GFX12: v_log_f16_e32 v5, s105 ; encoding: [0x69,0xae,0x0a,0x7e]
 
 v_log_f16 v5, vcc_lo
-// GFX12: v_log_f16_e32 v5, vcc_lo                ; encoding: [0x6a,0xae,0x0a,0x7e]
+// GFX12: v_log_f16_e32 v5, vcc_lo ; encoding: [0x6a,0xae,0x0a,0x7e]
 
 v_log_f16 v5, vcc_hi
-// GFX12: v_log_f16_e32 v5, vcc_hi                ; encoding: [0x6b,0xae,0x0a,0x7e]
+// GFX12: v_log_f16_e32 v5, vcc_hi ; encoding: [0x6b,0xae,0x0a,0x7e]
 
 v_log_f16 v5, ttmp15
-// GFX12: v_log_f16_e32 v5, ttmp15                ; encoding: [0x7b,0xae,0x0a,0x7e]
+// GFX12: v_log_f16_e32 v5, ttmp15 ; encoding: [0x7b,0xae,0x0a,0x7e]
 
 v_log_f16 v5, m0
-// GFX12: v_log_f16_e32 v5, m0                    ; encoding: [0x7d,0xae,0x0a,0x7e]
+// GFX12: v_log_f16_e32 v5, m0 ; encoding: [0x7d,0xae,0x0a,0x7e]
 
 v_log_f16 v5, exec_lo
-// GFX12: v_log_f16_e32 v5, exec_lo               ; encoding: [0x7e,0xae,0x0a,0x7e]
+// GFX12: v_log_f16_e32 v5, exec_lo ; encoding: [0x7e,0xae,0x0a,0x7e]
 
 v_log_f16 v5, exec_hi
-// GFX12: v_log_f16_e32 v5, exec_hi               ; encoding: [0x7f,0xae,0x0a,0x7e]
+// GFX12: v_log_f16_e32 v5, exec_hi ; encoding: [0x7f,0xae,0x0a,0x7e]
 
 v_log_f16 v5, null
-// GFX12: v_log_f16_e32 v5, null                  ; encoding: [0x7c,0xae,0x0a,0x7e]
+// GFX12: v_log_f16_e32 v5, null ; encoding: [0x7c,0xae,0x0a,0x7e]
 
 v_log_f16 v5, -1
-// GFX12: v_log_f16_e32 v5, -1                    ; encoding: [0xc1,0xae,0x0a,0x7e]
+// GFX12: v_log_f16_e32 v5, -1 ; encoding: [0xc1,0xae,0x0a,0x7e]
 
 v_log_f16 v5, 0.5
-// GFX12: v_log_f16_e32 v5, 0.5                   ; encoding: [0xf0,0xae,0x0a,0x7e]
+// GFX12: v_log_f16_e32 v5, 0.5 ; encoding: [0xf0,0xae,0x0a,0x7e]
 
 v_log_f16 v5, src_scc
-// GFX12: v_log_f16_e32 v5, src_scc               ; encoding: [0xfd,0xae,0x0a,0x7e]
+// GFX12: v_log_f16_e32 v5, src_scc ; encoding: [0xfd,0xae,0x0a,0x7e]
 
 v_log_f16 v127, 0xfe0b
-// GFX12: v_log_f16_e32 v127, 0xfe0b              ; encoding: [0xff,0xae,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_log_f16_e32 v127, 0xfe0b ; encoding: [0xff,0xae,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_log_f32 v5, v1
-// GFX12: v_log_f32_e32 v5, v1                    ; encoding: [0x01,0x4f,0x0a,0x7e]
+// GFX12: v_log_f32_e32 v5, v1 ; encoding: [0x01,0x4f,0x0a,0x7e]
 
 v_log_f32 v5, v255
-// GFX12: v_log_f32_e32 v5, v255                  ; encoding: [0xff,0x4f,0x0a,0x7e]
+// GFX12: v_log_f32_e32 v5, v255 ; encoding: [0xff,0x4f,0x0a,0x7e]
 
 v_log_f32 v5, s1
-// GFX12: v_log_f32_e32 v5, s1                    ; encoding: [0x01,0x4e,0x0a,0x7e]
+// GFX12: v_log_f32_e32 v5, s1 ; encoding: [0x01,0x4e,0x0a,0x7e]
 
 v_log_f32 v5, s105
-// GFX12: v_log_f32_e32 v5, s105                  ; encoding: [0x69,0x4e,0x0a,0x7e]
+// GFX12: v_log_f32_e32 v5, s105 ; encoding: [0x69,0x4e,0x0a,0x7e]
 
 v_log_f32 v5, vcc_lo
-// GFX12: v_log_f32_e32 v5, vcc_lo                ; encoding: [0x6a,0x4e,0x0a,0x7e]
+// GFX12: v_log_f32_e32 v5, vcc_lo ; encoding: [0x6a,0x4e,0x0a,0x7e]
 
 v_log_f32 v5, vcc_hi
-// GFX12: v_log_f32_e32 v5, vcc_hi                ; encoding: [0x6b,0x4e,0x0a,0x7e]
+// GFX12: v_log_f32_e32 v5, vcc_hi ; encoding: [0x6b,0x4e,0x0a,0x7e]
 
 v_log_f32 v5, ttmp15
-// GFX12: v_log_f32_e32 v5, ttmp15                ; encoding: [0x7b,0x4e,0x0a,0x7e]
+// GFX12: v_log_f32_e32 v5, ttmp15 ; encoding: [0x7b,0x4e,0x0a,0x7e]
 
 v_log_f32 v5, m0
-// GFX12: v_log_f32_e32 v5, m0                    ; encoding: [0x7d,0x4e,0x0a,0x7e]
+// GFX12: v_log_f32_e32 v5, m0 ; encoding: [0x7d,0x4e,0x0a,0x7e]
 
 v_log_f32 v5, exec_lo
-// GFX12: v_log_f32_e32 v5, exec_lo               ; encoding: [0x7e,0x4e,0x0a,0x7e]
+// GFX12: v_log_f32_e32 v5, exec_lo ; encoding: [0x7e,0x4e,0x0a,0x7e]
 
 v_log_f32 v5, exec_hi
-// GFX12: v_log_f32_e32 v5, exec_hi               ; encoding: [0x7f,0x4e,0x0a,0x7e]
+// GFX12: v_log_f32_e32 v5, exec_hi ; encoding: [0x7f,0x4e,0x0a,0x7e]
 
 v_log_f32 v5, null
-// GFX12: v_log_f32_e32 v5, null                  ; encoding: [0x7c,0x4e,0x0a,0x7e]
+// GFX12: v_log_f32_e32 v5, null ; encoding: [0x7c,0x4e,0x0a,0x7e]
 
 v_log_f32 v5, -1
-// GFX12: v_log_f32_e32 v5, -1                    ; encoding: [0xc1,0x4e,0x0a,0x7e]
+// GFX12: v_log_f32_e32 v5, -1 ; encoding: [0xc1,0x4e,0x0a,0x7e]
 
 v_log_f32 v5, 0.5
-// GFX12: v_log_f32_e32 v5, 0.5                   ; encoding: [0xf0,0x4e,0x0a,0x7e]
+// GFX12: v_log_f32_e32 v5, 0.5 ; encoding: [0xf0,0x4e,0x0a,0x7e]
 
 v_log_f32 v5, src_scc
-// GFX12: v_log_f32_e32 v5, src_scc               ; encoding: [0xfd,0x4e,0x0a,0x7e]
+// GFX12: v_log_f32_e32 v5, src_scc ; encoding: [0xfd,0x4e,0x0a,0x7e]
 
 v_log_f32 v255, 0xaf123456
-// GFX12: v_log_f32_e32 v255, 0xaf123456          ; encoding: [0xff,0x4e,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_log_f32_e32 v255, 0xaf123456 ; encoding: [0xff,0x4e,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_mov_b32 v5, v1
-// GFX12: v_mov_b32_e32 v5, v1                    ; encoding: [0x01,0x03,0x0a,0x7e]
+// GFX12: v_mov_b32_e32 v5, v1 ; encoding: [0x01,0x03,0x0a,0x7e]
 
 v_mov_b32 v5, v255
-// GFX12: v_mov_b32_e32 v5, v255                  ; encoding: [0xff,0x03,0x0a,0x7e]
+// GFX12: v_mov_b32_e32 v5, v255 ; encoding: [0xff,0x03,0x0a,0x7e]
 
 v_mov_b32 v5, s1
-// GFX12: v_mov_b32_e32 v5, s1                    ; encoding: [0x01,0x02,0x0a,0x7e]
+// GFX12: v_mov_b32_e32 v5, s1 ; encoding: [0x01,0x02,0x0a,0x7e]
 
 v_mov_b32 v5, s105
-// GFX12: v_mov_b32_e32 v5, s105                  ; encoding: [0x69,0x02,0x0a,0x7e]
+// GFX12: v_mov_b32_e32 v5, s105 ; encoding: [0x69,0x02,0x0a,0x7e]
 
 v_mov_b32 v5, vcc_lo
-// GFX12: v_mov_b32_e32 v5, vcc_lo                ; encoding: [0x6a,0x02,0x0a,0x7e]
+// GFX12: v_mov_b32_e32 v5, vcc_lo ; encoding: [0x6a,0x02,0x0a,0x7e]
 
 v_mov_b32 v5, vcc_hi
-// GFX12: v_mov_b32_e32 v5, vcc_hi                ; encoding: [0x6b,0x02,0x0a,0x7e]
+// GFX12: v_mov_b32_e32 v5, vcc_hi ; encoding: [0x6b,0x02,0x0a,0x7e]
 
 v_mov_b32 v5, ttmp15
-// GFX12: v_mov_b32_e32 v5, ttmp15                ; encoding: [0x7b,0x02,0x0a,0x7e]
+// GFX12: v_mov_b32_e32 v5, ttmp15 ; encoding: [0x7b,0x02,0x0a,0x7e]
 
 v_mov_b32 v5, m0
-// GFX12: v_mov_b32_e32 v5, m0                    ; encoding: [0x7d,0x02,0x0a,0x7e]
+// GFX12: v_mov_b32_e32 v5, m0 ; encoding: [0x7d,0x02,0x0a,0x7e]
 
 v_mov_b32 v5, exec_lo
-// GFX12: v_mov_b32_e32 v5, exec_lo               ; encoding: [0x7e,0x02,0x0a,0x7e]
+// GFX12: v_mov_b32_e32 v5, exec_lo ; encoding: [0x7e,0x02,0x0a,0x7e]
 
 v_mov_b32 v5, exec_hi
-// GFX12: v_mov_b32_e32 v5, exec_hi               ; encoding: [0x7f,0x02,0x0a,0x7e]
+// GFX12: v_mov_b32_e32 v5, exec_hi ; encoding: [0x7f,0x02,0x0a,0x7e]
 
 v_mov_b32 v5, null
-// GFX12: v_mov_b32_e32 v5, null                  ; encoding: [0x7c,0x02,0x0a,0x7e]
+// GFX12: v_mov_b32_e32 v5, null ; encoding: [0x7c,0x02,0x0a,0x7e]
 
 v_mov_b32 v5, -1
-// GFX12: v_mov_b32_e32 v5, -1                    ; encoding: [0xc1,0x02,0x0a,0x7e]
+// GFX12: v_mov_b32_e32 v5, -1 ; encoding: [0xc1,0x02,0x0a,0x7e]
 
 v_mov_b32 v5, 0.5
-// GFX12: v_mov_b32_e32 v5, 0.5                   ; encoding: [0xf0,0x02,0x0a,0x7e]
+// GFX12: v_mov_b32_e32 v5, 0.5 ; encoding: [0xf0,0x02,0x0a,0x7e]
 
 v_mov_b32 v5, src_scc
-// GFX12: v_mov_b32_e32 v5, src_scc               ; encoding: [0xfd,0x02,0x0a,0x7e]
+// GFX12: v_mov_b32_e32 v5, src_scc ; encoding: [0xfd,0x02,0x0a,0x7e]
 
 v_mov_b32 v255, 0xaf123456
-// GFX12: v_mov_b32_e32 v255, 0xaf123456          ; encoding: [0xff,0x02,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_mov_b32_e32 v255, 0xaf123456 ; encoding: [0xff,0x02,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_movreld_b32 v5, v1
-// GFX12: v_movreld_b32_e32 v5, v1                ; encoding: [0x01,0x85,0x0a,0x7e]
+// GFX12: v_movreld_b32_e32 v5, v1 ; encoding: [0x01,0x85,0x0a,0x7e]
 
 v_movreld_b32 v5, v255
-// GFX12: v_movreld_b32_e32 v5, v255              ; encoding: [0xff,0x85,0x0a,0x7e]
+// GFX12: v_movreld_b32_e32 v5, v255 ; encoding: [0xff,0x85,0x0a,0x7e]
 
 v_movreld_b32 v5, s1
-// GFX12: v_movreld_b32_e32 v5, s1                ; encoding: [0x01,0x84,0x0a,0x7e]
+// GFX12: v_movreld_b32_e32 v5, s1 ; encoding: [0x01,0x84,0x0a,0x7e]
 
 v_movreld_b32 v5, s105
-// GFX12: v_movreld_b32_e32 v5, s105              ; encoding: [0x69,0x84,0x0a,0x7e]
+// GFX12: v_movreld_b32_e32 v5, s105 ; encoding: [0x69,0x84,0x0a,0x7e]
 
 v_movreld_b32 v5, vcc_lo
-// GFX12: v_movreld_b32_e32 v5, vcc_lo            ; encoding: [0x6a,0x84,0x0a,0x7e]
+// GFX12: v_movreld_b32_e32 v5, vcc_lo ; encoding: [0x6a,0x84,0x0a,0x7e]
 
 v_movreld_b32 v5, vcc_hi
-// GFX12: v_movreld_b32_e32 v5, vcc_hi            ; encoding: [0x6b,0x84,0x0a,0x7e]
+// GFX12: v_movreld_b32_e32 v5, vcc_hi ; encoding: [0x6b,0x84,0x0a,0x7e]
 
 v_movreld_b32 v5, ttmp15
-// GFX12: v_movreld_b32_e32 v5, ttmp15            ; encoding: [0x7b,0x84,0x0a,0x7e]
+// GFX12: v_movreld_b32_e32 v5, ttmp15 ; encoding: [0x7b,0x84,0x0a,0x7e]
 
 v_movreld_b32 v5, m0
-// GFX12: v_movreld_b32_e32 v5, m0                ; encoding: [0x7d,0x84,0x0a,0x7e]
+// GFX12: v_movreld_b32_e32 v5, m0 ; encoding: [0x7d,0x84,0x0a,0x7e]
 
 v_movreld_b32 v5, exec_lo
-// GFX12: v_movreld_b32_e32 v5, exec_lo           ; encoding: [0x7e,0x84,0x0a,0x7e]
+// GFX12: v_movreld_b32_e32 v5, exec_lo ; encoding: [0x7e,0x84,0x0a,0x7e]
 
 v_movreld_b32 v5, exec_hi
-// GFX12: v_movreld_b32_e32 v5, exec_hi           ; encoding: [0x7f,0x84,0x0a,0x7e]
+// GFX12: v_movreld_b32_e32 v5, exec_hi ; encoding: [0x7f,0x84,0x0a,0x7e]
 
 v_movreld_b32 v5, null
-// GFX12: v_movreld_b32_e32 v5, null              ; encoding: [0x7c,0x84,0x0a,0x7e]
+// GFX12: v_movreld_b32_e32 v5, null ; encoding: [0x7c,0x84,0x0a,0x7e]
 
 v_movreld_b32 v5, -1
-// GFX12: v_movreld_b32_e32 v5, -1                ; encoding: [0xc1,0x84,0x0a,0x7e]
+// GFX12: v_movreld_b32_e32 v5, -1 ; encoding: [0xc1,0x84,0x0a,0x7e]
 
 v_movreld_b32 v5, 0.5
-// GFX12: v_movreld_b32_e32 v5, 0.5               ; encoding: [0xf0,0x84,0x0a,0x7e]
+// GFX12: v_movreld_b32_e32 v5, 0.5 ; encoding: [0xf0,0x84,0x0a,0x7e]
 
 v_movreld_b32 v5, src_scc
-// GFX12: v_movreld_b32_e32 v5, src_scc           ; encoding: [0xfd,0x84,0x0a,0x7e]
+// GFX12: v_movreld_b32_e32 v5, src_scc ; encoding: [0xfd,0x84,0x0a,0x7e]
 
 v_movreld_b32 v255, 0xaf123456
-// GFX12: v_movreld_b32_e32 v255, 0xaf123456      ; encoding: [0xff,0x84,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_movreld_b32_e32 v255, 0xaf123456 ; encoding: [0xff,0x84,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_movrels_b32 v5, v1
-// GFX12: v_movrels_b32_e32 v5, v1                ; encoding: [0x01,0x87,0x0a,0x7e]
+// GFX12: v_movrels_b32_e32 v5, v1 ; encoding: [0x01,0x87,0x0a,0x7e]
 
 v_movrels_b32 v255, v255
-// GFX12: v_movrels_b32_e32 v255, v255            ; encoding: [0xff,0x87,0xfe,0x7f]
+// GFX12: v_movrels_b32_e32 v255, v255 ; encoding: [0xff,0x87,0xfe,0x7f]
 
 v_movrelsd_2_b32 v5, v1
-// GFX12: v_movrelsd_2_b32_e32 v5, v1             ; encoding: [0x01,0x91,0x0a,0x7e]
+// GFX12: v_movrelsd_2_b32_e32 v5, v1 ; encoding: [0x01,0x91,0x0a,0x7e]
 
 v_movrelsd_2_b32 v255, v255
-// GFX12: v_movrelsd_2_b32_e32 v255, v255         ; encoding: [0xff,0x91,0xfe,0x7f]
+// GFX12: v_movrelsd_2_b32_e32 v255, v255 ; encoding: [0xff,0x91,0xfe,0x7f]
 
 v_movrelsd_b32 v5, v1
-// GFX12: v_movrelsd_b32_e32 v5, v1               ; encoding: [0x01,0x89,0x0a,0x7e]
+// GFX12: v_movrelsd_b32_e32 v5, v1 ; encoding: [0x01,0x89,0x0a,0x7e]
 
 v_movrelsd_b32 v255, v255
-// GFX12: v_movrelsd_b32_e32 v255, v255           ; encoding: [0xff,0x89,0xfe,0x7f]
+// GFX12: v_movrelsd_b32_e32 v255, v255 ; encoding: [0xff,0x89,0xfe,0x7f]
 
 v_nop
-// GFX12: v_nop                                   ; encoding: [0x00,0x00,0x00,0x7e]
+// GFX12: v_nop ; encoding: [0x00,0x00,0x00,0x7e]
 
 v_not_b16 v5, v1
-// GFX12: v_not_b16_e32 v5, v1                    ; encoding: [0x01,0xd3,0x0a,0x7e]
+// GFX12: v_not_b16_e32 v5, v1 ; encoding: [0x01,0xd3,0x0a,0x7e]
 
 v_not_b16 v5, v127
-// GFX12: v_not_b16_e32 v5, v127                  ; encoding: [0x7f,0xd3,0x0a,0x7e]
+// GFX12: v_not_b16_e32 v5, v127 ; encoding: [0x7f,0xd3,0x0a,0x7e]
 
 v_not_b16 v5, s1
-// GFX12: v_not_b16_e32 v5, s1                    ; encoding: [0x01,0xd2,0x0a,0x7e]
+// GFX12: v_not_b16_e32 v5, s1 ; encoding: [0x01,0xd2,0x0a,0x7e]
 
 v_not_b16 v5, s105
-// GFX12: v_not_b16_e32 v5, s105                  ; encoding: [0x69,0xd2,0x0a,0x7e]
+// GFX12: v_not_b16_e32 v5, s105 ; encoding: [0x69,0xd2,0x0a,0x7e]
 
 v_not_b16 v5, vcc_lo
-// GFX12: v_not_b16_e32 v5, vcc_lo                ; encoding: [0x6a,0xd2,0x0a,0x7e]
+// GFX12: v_not_b16_e32 v5, vcc_lo ; encoding: [0x6a,0xd2,0x0a,0x7e]
 
 v_not_b16 v5, vcc_hi
-// GFX12: v_not_b16_e32 v5, vcc_hi                ; encoding: [0x6b,0xd2,0x0a,0x7e]
+// GFX12: v_not_b16_e32 v5, vcc_hi ; encoding: [0x6b,0xd2,0x0a,0x7e]
 
 v_not_b16 v5, ttmp15
-// GFX12: v_not_b16_e32 v5, ttmp15                ; encoding: [0x7b,0xd2,0x0a,0x7e]
+// GFX12: v_not_b16_e32 v5, ttmp15 ; encoding: [0x7b,0xd2,0x0a,0x7e]
 
 v_not_b16 v5, m0
-// GFX12: v_not_b16_e32 v5, m0                    ; encoding: [0x7d,0xd2,0x0a,0x7e]
+// GFX12: v_not_b16_e32 v5, m0 ; encoding: [0x7d,0xd2,0x0a,0x7e]
 
 v_not_b16 v5, exec_lo
-// GFX12: v_not_b16_e32 v5, exec_lo               ; encoding: [0x7e,0xd2,0x0a,0x7e]
+// GFX12: v_not_b16_e32 v5, exec_lo ; encoding: [0x7e,0xd2,0x0a,0x7e]
 
 v_not_b16 v5, exec_hi
-// GFX12: v_not_b16_e32 v5, exec_hi               ; encoding: [0x7f,0xd2,0x0a,0x7e]
+// GFX12: v_not_b16_e32 v5, exec_hi ; encoding: [0x7f,0xd2,0x0a,0x7e]
 
 v_not_b16 v5, null
-// GFX12: v_not_b16_e32 v5, null                  ; encoding: [0x7c,0xd2,0x0a,0x7e]
+// GFX12: v_not_b16_e32 v5, null ; encoding: [0x7c,0xd2,0x0a,0x7e]
 
 v_not_b16 v5, -1
-// GFX12: v_not_b16_e32 v5, -1                    ; encoding: [0xc1,0xd2,0x0a,0x7e]
+// GFX12: v_not_b16_e32 v5, -1 ; encoding: [0xc1,0xd2,0x0a,0x7e]
 
 v_not_b16 v5, 0.5
-// GFX12-ASM: v_not_b16_e32 v5, 0.5                   ; encoding: [0xf0,0xd2,0x0a,0x7e]
-// GFX12-DIS: v_not_b16_e32 v5, 0x3800                ; encoding: [0xff,0xd2,0x0a,0x7e,0x00,0x38,0x00,0x00]
+// GFX12-ASM: v_not_b16_e32 v5, 0.5 ; encoding: [0xf0,0xd2,0x0a,0x7e]
+// GFX12-DIS: v_not_b16_e32 v5, 0x3800 ; encoding: [0xff,0xd2,0x0a,0x7e,0x00,0x38,0x00,0x00]
 
 v_not_b16 v5, src_scc
-// GFX12: v_not_b16_e32 v5, src_scc               ; encoding: [0xfd,0xd2,0x0a,0x7e]
+// GFX12: v_not_b16_e32 v5, src_scc ; encoding: [0xfd,0xd2,0x0a,0x7e]
 
 v_not_b16 v127, 0xfe0b
-// GFX12: v_not_b16_e32 v127, 0xfe0b              ; encoding: [0xff,0xd2,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_not_b16_e32 v127, 0xfe0b ; encoding: [0xff,0xd2,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_not_b32 v5, v1
-// GFX12: v_not_b32_e32 v5, v1                    ; encoding: [0x01,0x6f,0x0a,0x7e]
+// GFX12: v_not_b32_e32 v5, v1 ; encoding: [0x01,0x6f,0x0a,0x7e]
 
 v_not_b32 v5, v255
-// GFX12: v_not_b32_e32 v5, v255                  ; encoding: [0xff,0x6f,0x0a,0x7e]
+// GFX12: v_not_b32_e32 v5, v255 ; encoding: [0xff,0x6f,0x0a,0x7e]
 
 v_not_b32 v5, s1
-// GFX12: v_not_b32_e32 v5, s1                    ; encoding: [0x01,0x6e,0x0a,0x7e]
+// GFX12: v_not_b32_e32 v5, s1 ; encoding: [0x01,0x6e,0x0a,0x7e]
 
 v_not_b32 v5, s105
-// GFX12: v_not_b32_e32 v5, s105                  ; encoding: [0x69,0x6e,0x0a,0x7e]
+// GFX12: v_not_b32_e32 v5, s105 ; encoding: [0x69,0x6e,0x0a,0x7e]
 
 v_not_b32 v5, vcc_lo
-// GFX12: v_not_b32_e32 v5, vcc_lo                ; encoding: [0x6a,0x6e,0x0a,0x7e]
+// GFX12: v_not_b32_e32 v5, vcc_lo ; encoding: [0x6a,0x6e,0x0a,0x7e]
 
 v_not_b32 v5, vcc_hi
-// GFX12: v_not_b32_e32 v5, vcc_hi                ; encoding: [0x6b,0x6e,0x0a,0x7e]
+// GFX12: v_not_b32_e32 v5, vcc_hi ; encoding: [0x6b,0x6e,0x0a,0x7e]
 
 v_not_b32 v5, ttmp15
-// GFX12: v_not_b32_e32 v5, ttmp15                ; encoding: [0x7b,0x6e,0x0a,0x7e]
+// GFX12: v_not_b32_e32 v5, ttmp15 ; encoding: [0x7b,0x6e,0x0a,0x7e]
 
 v_not_b32 v5, m0
-// GFX12: v_not_b32_e32 v5, m0                    ; encoding: [0x7d,0x6e,0x0a,0x7e]
+// GFX12: v_not_b32_e32 v5, m0 ; encoding: [0x7d,0x6e,0x0a,0x7e]
 
 v_not_b32 v5, exec_lo
-// GFX12: v_not_b32_e32 v5, exec_lo               ; encoding: [0x7e,0x6e,0x0a,0x7e]
+// GFX12: v_not_b32_e32 v5, exec_lo ; encoding: [0x7e,0x6e,0x0a,0x7e]
 
 v_not_b32 v5, exec_hi
-// GFX12: v_not_b32_e32 v5, exec_hi               ; encoding: [0x7f,0x6e,0x0a,0x7e]
+// GFX12: v_not_b32_e32 v5, exec_hi ; encoding: [0x7f,0x6e,0x0a,0x7e]
 
 v_not_b32 v5, null
-// GFX12: v_not_b32_e32 v5, null                  ; encoding: [0x7c,0x6e,0x0a,0x7e]
+// GFX12: v_not_b32_e32 v5, null ; encoding: [0x7c,0x6e,0x0a,0x7e]
 
 v_not_b32 v5, -1
-// GFX12: v_not_b32_e32 v5, -1                    ; encoding: [0xc1,0x6e,0x0a,0x7e]
+// GFX12: v_not_b32_e32 v5, -1 ; encoding: [0xc1,0x6e,0x0a,0x7e]
 
 v_not_b32 v5, 0.5
-// GFX12: v_not_b32_e32 v5, 0.5                   ; encoding: [0xf0,0x6e,0x0a,0x7e]
+// GFX12: v_not_b32_e32 v5, 0.5 ; encoding: [0xf0,0x6e,0x0a,0x7e]
 
 v_not_b32 v5, src_scc
-// GFX12: v_not_b32_e32 v5, src_scc               ; encoding: [0xfd,0x6e,0x0a,0x7e]
+// GFX12: v_not_b32_e32 v5, src_scc ; encoding: [0xfd,0x6e,0x0a,0x7e]
 
 v_not_b32 v255, 0xaf123456
-// GFX12: v_not_b32_e32 v255, 0xaf123456          ; encoding: [0xff,0x6e,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_not_b32_e32 v255, 0xaf123456 ; encoding: [0xff,0x6e,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_permlane64_b32 v5, v1
-// GFX12: v_permlane64_b32 v5, v1                 ; encoding: [0x01,0xcf,0x0a,0x7e]
+// GFX12: v_permlane64_b32 v5, v1 ; encoding: [0x01,0xcf,0x0a,0x7e]
 
 v_permlane64_b32 v255, v255
-// GFX12: v_permlane64_b32 v255, v255             ; encoding: [0xff,0xcf,0xfe,0x7f]
+// GFX12: v_permlane64_b32 v255, v255 ; encoding: [0xff,0xcf,0xfe,0x7f]
 
 v_pipeflush
-// GFX12: v_pipeflush                             ; encoding: [0x00,0x36,0x00,0x7e]
+// GFX12: v_pipeflush ; encoding: [0x00,0x36,0x00,0x7e]
 
 v_rcp_f16 v5, v1
-// GFX12: v_rcp_f16_e32 v5, v1                    ; encoding: [0x01,0xa9,0x0a,0x7e]
+// GFX12: v_rcp_f16_e32 v5, v1 ; encoding: [0x01,0xa9,0x0a,0x7e]
 
 v_rcp_f16 v5, v127
-// GFX12: v_rcp_f16_e32 v5, v127                  ; encoding: [0x7f,0xa9,0x0a,0x7e]
+// GFX12: v_rcp_f16_e32 v5, v127 ; encoding: [0x7f,0xa9,0x0a,0x7e]
 
 v_rcp_f16 v5, s1
-// GFX12: v_rcp_f16_e32 v5, s1                    ; encoding: [0x01,0xa8,0x0a,0x7e]
+// GFX12: v_rcp_f16_e32 v5, s1 ; encoding: [0x01,0xa8,0x0a,0x7e]
 
 v_rcp_f16 v5, s105
-// GFX12: v_rcp_f16_e32 v5, s105                  ; encoding: [0x69,0xa8,0x0a,0x7e]
+// GFX12: v_rcp_f16_e32 v5, s105 ; encoding: [0x69,0xa8,0x0a,0x7e]
 
 v_rcp_f16 v5, vcc_lo
-// GFX12: v_rcp_f16_e32 v5, vcc_lo                ; encoding: [0x6a,0xa8,0x0a,0x7e]
+// GFX12: v_rcp_f16_e32 v5, vcc_lo ; encoding: [0x6a,0xa8,0x0a,0x7e]
 
 v_rcp_f16 v5, vcc_hi
-// GFX12: v_rcp_f16_e32 v5, vcc_hi                ; encoding: [0x6b,0xa8,0x0a,0x7e]
+// GFX12: v_rcp_f16_e32 v5, vcc_hi ; encoding: [0x6b,0xa8,0x0a,0x7e]
 
 v_rcp_f16 v5, ttmp15
-// GFX12: v_rcp_f16_e32 v5, ttmp15                ; encoding: [0x7b,0xa8,0x0a,0x7e]
+// GFX12: v_rcp_f16_e32 v5, ttmp15 ; encoding: [0x7b,0xa8,0x0a,0x7e]
 
 v_rcp_f16 v5, m0
-// GFX12: v_rcp_f16_e32 v5, m0                    ; encoding: [0x7d,0xa8,0x0a,0x7e]
+// GFX12: v_rcp_f16_e32 v5, m0 ; encoding: [0x7d,0xa8,0x0a,0x7e]
 
 v_rcp_f16 v5, exec_lo
-// GFX12: v_rcp_f16_e32 v5, exec_lo               ; encoding: [0x7e,0xa8,0x0a,0x7e]
+// GFX12: v_rcp_f16_e32 v5, exec_lo ; encoding: [0x7e,0xa8,0x0a,0x7e]
 
 v_rcp_f16 v5, exec_hi
-// GFX12: v_rcp_f16_e32 v5, exec_hi               ; encoding: [0x7f,0xa8,0x0a,0x7e]
+// GFX12: v_rcp_f16_e32 v5, exec_hi ; encoding: [0x7f,0xa8,0x0a,0x7e]
 
 v_rcp_f16 v5, null
-// GFX12: v_rcp_f16_e32 v5, null                  ; encoding: [0x7c,0xa8,0x0a,0x7e]
+// GFX12: v_rcp_f16_e32 v5, null ; encoding: [0x7c,0xa8,0x0a,0x7e]
 
 v_rcp_f16 v5, -1
-// GFX12: v_rcp_f16_e32 v5, -1                    ; encoding: [0xc1,0xa8,0x0a,0x7e]
+// GFX12: v_rcp_f16_e32 v5, -1 ; encoding: [0xc1,0xa8,0x0a,0x7e]
 
 v_rcp_f16 v5, 0.5
-// GFX12: v_rcp_f16_e32 v5, 0.5                   ; encoding: [0xf0,0xa8,0x0a,0x7e]
+// GFX12: v_rcp_f16_e32 v5, 0.5 ; encoding: [0xf0,0xa8,0x0a,0x7e]
 
 v_rcp_f16 v5, src_scc
-// GFX12: v_rcp_f16_e32 v5, src_scc               ; encoding: [0xfd,0xa8,0x0a,0x7e]
+// GFX12: v_rcp_f16_e32 v5, src_scc ; encoding: [0xfd,0xa8,0x0a,0x7e]
 
 v_rcp_f16 v127, 0xfe0b
-// GFX12: v_rcp_f16_e32 v127, 0xfe0b              ; encoding: [0xff,0xa8,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_rcp_f16_e32 v127, 0xfe0b ; encoding: [0xff,0xa8,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_rcp_f32 v5, v1
-// GFX12: v_rcp_f32_e32 v5, v1                    ; encoding: [0x01,0x55,0x0a,0x7e]
+// GFX12: v_rcp_f32_e32 v5, v1 ; encoding: [0x01,0x55,0x0a,0x7e]
 
 v_rcp_f32 v5, v255
-// GFX12: v_rcp_f32_e32 v5, v255                  ; encoding: [0xff,0x55,0x0a,0x7e]
+// GFX12: v_rcp_f32_e32 v5, v255 ; encoding: [0xff,0x55,0x0a,0x7e]
 
 v_rcp_f32 v5, s1
-// GFX12: v_rcp_f32_e32 v5, s1                    ; encoding: [0x01,0x54,0x0a,0x7e]
+// GFX12: v_rcp_f32_e32 v5, s1 ; encoding: [0x01,0x54,0x0a,0x7e]
 
 v_rcp_f32 v5, s105
-// GFX12: v_rcp_f32_e32 v5, s105                  ; encoding: [0x69,0x54,0x0a,0x7e]
+// GFX12: v_rcp_f32_e32 v5, s105 ; encoding: [0x69,0x54,0x0a,0x7e]
 
 v_rcp_f32 v5, vcc_lo
-// GFX12: v_rcp_f32_e32 v5, vcc_lo                ; encoding: [0x6a,0x54,0x0a,0x7e]
+// GFX12: v_rcp_f32_e32 v5, vcc_lo ; encoding: [0x6a,0x54,0x0a,0x7e]
 
 v_rcp_f32 v5, vcc_hi
-// GFX12: v_rcp_f32_e32 v5, vcc_hi                ; encoding: [0x6b,0x54,0x0a,0x7e]
+// GFX12: v_rcp_f32_e32 v5, vcc_hi ; encoding: [0x6b,0x54,0x0a,0x7e]
 
 v_rcp_f32 v5, ttmp15
-// GFX12: v_rcp_f32_e32 v5, ttmp15                ; encoding: [0x7b,0x54,0x0a,0x7e]
+// GFX12: v_rcp_f32_e32 v5, ttmp15 ; encoding: [0x7b,0x54,0x0a,0x7e]
 
 v_rcp_f32 v5, m0
-// GFX12: v_rcp_f32_e32 v5, m0                    ; encoding: [0x7d,0x54,0x0a,0x7e]
+// GFX12: v_rcp_f32_e32 v5, m0 ; encoding: [0x7d,0x54,0x0a,0x7e]
 
 v_rcp_f32 v5, exec_lo
-// GFX12: v_rcp_f32_e32 v5, exec_lo               ; encoding: [0x7e,0x54,0x0a,0x7e]
+// GFX12: v_rcp_f32_e32 v5, exec_lo ; encoding: [0x7e,0x54,0x0a,0x7e]
 
 v_rcp_f32 v5, exec_hi
-// GFX12: v_rcp_f32_e32 v5, exec_hi               ; encoding: [0x7f,0x54,0x0a,0x7e]
+// GFX12: v_rcp_f32_e32 v5, exec_hi ; encoding: [0x7f,0x54,0x0a,0x7e]
 
 v_rcp_f32 v5, null
-// GFX12: v_rcp_f32_e32 v5, null                  ; encoding: [0x7c,0x54,0x0a,0x7e]
+// GFX12: v_rcp_f32_e32 v5, null ; encoding: [0x7c,0x54,0x0a,0x7e]
 
 v_rcp_f32 v5, -1
-// GFX12: v_rcp_f32_e32 v5, -1                    ; encoding: [0xc1,0x54,0x0a,0x7e]
+// GFX12: v_rcp_f32_e32 v5, -1 ; encoding: [0xc1,0x54,0x0a,0x7e]
 
 v_rcp_f32 v5, 0.5
-// GFX12: v_rcp_f32_e32 v5, 0.5                   ; encoding: [0xf0,0x54,0x0a,0x7e]
+// GFX12: v_rcp_f32_e32 v5, 0.5 ; encoding: [0xf0,0x54,0x0a,0x7e]
 
 v_rcp_f32 v5, src_scc
-// GFX12: v_rcp_f32_e32 v5, src_scc               ; encoding: [0xfd,0x54,0x0a,0x7e]
+// GFX12: v_rcp_f32_e32 v5, src_scc ; encoding: [0xfd,0x54,0x0a,0x7e]
 
 v_rcp_f32 v255, 0xaf123456
-// GFX12: v_rcp_f32_e32 v255, 0xaf123456          ; encoding: [0xff,0x54,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_rcp_f32_e32 v255, 0xaf123456 ; encoding: [0xff,0x54,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_rcp_f64 v[5:6], v[1:2]
-// GFX12: v_rcp_f64_e32 v[5:6], v[1:2]            ; encoding: [0x01,0x5f,0x0a,0x7e]
+// GFX12: v_rcp_f64_e32 v[5:6], v[1:2] ; encoding: [0x01,0x5f,0x0a,0x7e]
 
 v_rcp_f64 v[5:6], v[254:255]
-// GFX12: v_rcp_f64_e32 v[5:6], v[254:255]        ; encoding: [0xfe,0x5f,0x0a,0x7e]
+// GFX12: v_rcp_f64_e32 v[5:6], v[254:255] ; encoding: [0xfe,0x5f,0x0a,0x7e]
 
 v_rcp_f64 v[5:6], s[2:3]
-// GFX12: v_rcp_f64_e32 v[5:6], s[2:3]            ; encoding: [0x02,0x5e,0x0a,0x7e]
+// GFX12: v_rcp_f64_e32 v[5:6], s[2:3] ; encoding: [0x02,0x5e,0x0a,0x7e]
 
 v_rcp_f64 v[5:6], s[104:105]
-// GFX12: v_rcp_f64_e32 v[5:6], s[104:105]        ; encoding: [0x68,0x5e,0x0a,0x7e]
+// GFX12: v_rcp_f64_e32 v[5:6], s[104:105] ; encoding: [0x68,0x5e,0x0a,0x7e]
 
 v_rcp_f64 v[5:6], vcc
-// GFX12: v_rcp_f64_e32 v[5:6], vcc               ; encoding: [0x6a,0x5e,0x0a,0x7e]
+// GFX12: v_rcp_f64_e32 v[5:6], vcc ; encoding: [0x6a,0x5e,0x0a,0x7e]
 
 v_rcp_f64 v[5:6], ttmp[14:15]
-// GFX12: v_rcp_f64_e32 v[5:6], ttmp[14:15]       ; encoding: [0x7a,0x5e,0x0a,0x7e]
+// GFX12: v_rcp_f64_e32 v[5:6], ttmp[14:15] ; encoding: [0x7a,0x5e,0x0a,0x7e]
 
 v_rcp_f64 v[5:6], exec
-// GFX12: v_rcp_f64_e32 v[5:6], exec              ; encoding: [0x7e,0x5e,0x0a,0x7e]
+// GFX12: v_rcp_f64_e32 v[5:6], exec ; encoding: [0x7e,0x5e,0x0a,0x7e]
 
 v_rcp_f64 v[5:6], null
-// GFX12: v_rcp_f64_e32 v[5:6], null              ; encoding: [0x7c,0x5e,0x0a,0x7e]
+// GFX12: v_rcp_f64_e32 v[5:6], null ; encoding: [0x7c,0x5e,0x0a,0x7e]
 
 v_rcp_f64 v[5:6], -1
-// GFX12: v_rcp_f64_e32 v[5:6], -1                ; encoding: [0xc1,0x5e,0x0a,0x7e]
+// GFX12: v_rcp_f64_e32 v[5:6], -1 ; encoding: [0xc1,0x5e,0x0a,0x7e]
 
 v_rcp_f64 v[5:6], 0.5
-// GFX12: v_rcp_f64_e32 v[5:6], 0.5               ; encoding: [0xf0,0x5e,0x0a,0x7e]
+// GFX12: v_rcp_f64_e32 v[5:6], 0.5 ; encoding: [0xf0,0x5e,0x0a,0x7e]
 
 v_rcp_f64 v[5:6], src_scc
-// GFX12: v_rcp_f64_e32 v[5:6], src_scc           ; encoding: [0xfd,0x5e,0x0a,0x7e]
+// GFX12: v_rcp_f64_e32 v[5:6], src_scc ; encoding: [0xfd,0x5e,0x0a,0x7e]
 
 v_rcp_f64 v[254:255], 0xaf123456
-// GFX12: v_rcp_f64_e32 v[254:255], 0xaf123456    ; encoding: [0xff,0x5e,0xfc,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_rcp_f64_e32 v[254:255], 0xaf123456 ; encoding: [0xff,0x5e,0xfc,0x7f,0x56,0x34,0x12,0xaf]
 
 v_rcp_iflag_f32 v5, v1
-// GFX12: v_rcp_iflag_f32_e32 v5, v1              ; encoding: [0x01,0x57,0x0a,0x7e]
+// GFX12: v_rcp_iflag_f32_e32 v5, v1 ; encoding: [0x01,0x57,0x0a,0x7e]
 
 v_rcp_iflag_f32 v5, v255
-// GFX12: v_rcp_iflag_f32_e32 v5, v255            ; encoding: [0xff,0x57,0x0a,0x7e]
+// GFX12: v_rcp_iflag_f32_e32 v5, v255 ; encoding: [0xff,0x57,0x0a,0x7e]
 
 v_rcp_iflag_f32 v5, s1
-// GFX12: v_rcp_iflag_f32_e32 v5, s1              ; encoding: [0x01,0x56,0x0a,0x7e]
+// GFX12: v_rcp_iflag_f32_e32 v5, s1 ; encoding: [0x01,0x56,0x0a,0x7e]
 
 v_rcp_iflag_f32 v5, s105
-// GFX12: v_rcp_iflag_f32_e32 v5, s105            ; encoding: [0x69,0x56,0x0a,0x7e]
+// GFX12: v_rcp_iflag_f32_e32 v5, s105 ; encoding: [0x69,0x56,0x0a,0x7e]
 
 v_rcp_iflag_f32 v5, vcc_lo
-// GFX12: v_rcp_iflag_f32_e32 v5, vcc_lo          ; encoding: [0x6a,0x56,0x0a,0x7e]
+// GFX12: v_rcp_iflag_f32_e32 v5, vcc_lo ; encoding: [0x6a,0x56,0x0a,0x7e]
 
 v_rcp_iflag_f32 v5, vcc_hi
-// GFX12: v_rcp_iflag_f32_e32 v5, vcc_hi          ; encoding: [0x6b,0x56,0x0a,0x7e]
+// GFX12: v_rcp_iflag_f32_e32 v5, vcc_hi ; encoding: [0x6b,0x56,0x0a,0x7e]
 
 v_rcp_iflag_f32 v5, ttmp15
-// GFX12: v_rcp_iflag_f32_e32 v5, ttmp15          ; encoding: [0x7b,0x56,0x0a,0x7e]
+// GFX12: v_rcp_iflag_f32_e32 v5, ttmp15 ; encoding: [0x7b,0x56,0x0a,0x7e]
 
 v_rcp_iflag_f32 v5, m0
-// GFX12: v_rcp_iflag_f32_e32 v5, m0              ; encoding: [0x7d,0x56,0x0a,0x7e]
+// GFX12: v_rcp_iflag_f32_e32 v5, m0 ; encoding: [0x7d,0x56,0x0a,0x7e]
 
 v_rcp_iflag_f32 v5, exec_lo
-// GFX12: v_rcp_iflag_f32_e32 v5, exec_lo         ; encoding: [0x7e,0x56,0x0a,0x7e]
+// GFX12: v_rcp_iflag_f32_e32 v5, exec_lo ; encoding: [0x7e,0x56,0x0a,0x7e]
 
 v_rcp_iflag_f32 v5, exec_hi
-// GFX12: v_rcp_iflag_f32_e32 v5, exec_hi         ; encoding: [0x7f,0x56,0x0a,0x7e]
+// GFX12: v_rcp_iflag_f32_e32 v5, exec_hi ; encoding: [0x7f,0x56,0x0a,0x7e]
 
 v_rcp_iflag_f32 v5, null
-// GFX12: v_rcp_iflag_f32_e32 v5, null            ; encoding: [0x7c,0x56,0x0a,0x7e]
+// GFX12: v_rcp_iflag_f32_e32 v5, null ; encoding: [0x7c,0x56,0x0a,0x7e]
 
 v_rcp_iflag_f32 v5, -1
-// GFX12: v_rcp_iflag_f32_e32 v5, -1              ; encoding: [0xc1,0x56,0x0a,0x7e]
+// GFX12: v_rcp_iflag_f32_e32 v5, -1 ; encoding: [0xc1,0x56,0x0a,0x7e]
 
 v_rcp_iflag_f32 v5, 0.5
-// GFX12: v_rcp_iflag_f32_e32 v5, 0.5             ; encoding: [0xf0,0x56,0x0a,0x7e]
+// GFX12: v_rcp_iflag_f32_e32 v5, 0.5 ; encoding: [0xf0,0x56,0x0a,0x7e]
 
 v_rcp_iflag_f32 v5, src_scc
-// GFX12: v_rcp_iflag_f32_e32 v5, src_scc         ; encoding: [0xfd,0x56,0x0a,0x7e]
+// GFX12: v_rcp_iflag_f32_e32 v5, src_scc ; encoding: [0xfd,0x56,0x0a,0x7e]
 
 v_rcp_iflag_f32 v255, 0xaf123456
-// GFX12: v_rcp_iflag_f32_e32 v255, 0xaf123456    ; encoding: [0xff,0x56,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_rcp_iflag_f32_e32 v255, 0xaf123456 ; encoding: [0xff,0x56,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_readfirstlane_b32 s5, v1
-// GFX12: v_readfirstlane_b32 s5, v1              ; encoding: [0x01,0x05,0x0a,0x7e]
+// GFX12: v_readfirstlane_b32 s5, v1 ; encoding: [0x01,0x05,0x0a,0x7e]
 
 v_readfirstlane_b32 s105, v1
-// GFX12: v_readfirstlane_b32 s105, v1            ; encoding: [0x01,0x05,0xd2,0x7e]
+// GFX12: v_readfirstlane_b32 s105, v1 ; encoding: [0x01,0x05,0xd2,0x7e]
 
 v_readfirstlane_b32 vcc_lo, v1
-// GFX12: v_readfirstlane_b32 vcc_lo, v1          ; encoding: [0x01,0x05,0xd4,0x7e]
+// GFX12: v_readfirstlane_b32 vcc_lo, v1 ; encoding: [0x01,0x05,0xd4,0x7e]
 
 v_readfirstlane_b32 vcc_hi, v1
-// GFX12: v_readfirstlane_b32 vcc_hi, v1          ; encoding: [0x01,0x05,0xd6,0x7e]
+// GFX12: v_readfirstlane_b32 vcc_hi, v1 ; encoding: [0x01,0x05,0xd6,0x7e]
 
 v_readfirstlane_b32 ttmp15, v1
-// GFX12: v_readfirstlane_b32 ttmp15, v1          ; encoding: [0x01,0x05,0xf6,0x7e]
+// GFX12: v_readfirstlane_b32 ttmp15, v1 ; encoding: [0x01,0x05,0xf6,0x7e]
 
 v_readfirstlane_b32 null, v255
-// GFX12: v_readfirstlane_b32 null, v255          ; encoding: [0xff,0x05,0xf8,0x7e]
+// GFX12: v_readfirstlane_b32 null, v255 ; encoding: [0xff,0x05,0xf8,0x7e]
 
 v_rndne_f16 v5, v1
-// GFX12: v_rndne_f16_e32 v5, v1                  ; encoding: [0x01,0xbd,0x0a,0x7e]
+// GFX12: v_rndne_f16_e32 v5, v1 ; encoding: [0x01,0xbd,0x0a,0x7e]
 
 v_rndne_f16 v5, v127
-// GFX12: v_rndne_f16_e32 v5, v127                ; encoding: [0x7f,0xbd,0x0a,0x7e]
+// GFX12: v_rndne_f16_e32 v5, v127 ; encoding: [0x7f,0xbd,0x0a,0x7e]
 
 v_rndne_f16 v5, s1
-// GFX12: v_rndne_f16_e32 v5, s1                  ; encoding: [0x01,0xbc,0x0a,0x7e]
+// GFX12: v_rndne_f16_e32 v5, s1 ; encoding: [0x01,0xbc,0x0a,0x7e]
 
 v_rndne_f16 v5, s105
-// GFX12: v_rndne_f16_e32 v5, s105                ; encoding: [0x69,0xbc,0x0a,0x7e]
+// GFX12: v_rndne_f16_e32 v5, s105 ; encoding: [0x69,0xbc,0x0a,0x7e]
 
 v_rndne_f16 v5, vcc_lo
-// GFX12: v_rndne_f16_e32 v5, vcc_lo              ; encoding: [0x6a,0xbc,0x0a,0x7e]
+// GFX12: v_rndne_f16_e32 v5, vcc_lo ; encoding: [0x6a,0xbc,0x0a,0x7e]
 
 v_rndne_f16 v5, vcc_hi
-// GFX12: v_rndne_f16_e32 v5, vcc_hi              ; encoding: [0x6b,0xbc,0x0a,0x7e]
+// GFX12: v_rndne_f16_e32 v5, vcc_hi ; encoding: [0x6b,0xbc,0x0a,0x7e]
 
 v_rndne_f16 v5, ttmp15
-// GFX12: v_rndne_f16_e32 v5, ttmp15              ; encoding: [0x7b,0xbc,0x0a,0x7e]
+// GFX12: v_rndne_f16_e32 v5, ttmp15 ; encoding: [0x7b,0xbc,0x0a,0x7e]
 
 v_rndne_f16 v5, m0
-// GFX12: v_rndne_f16_e32 v5, m0                  ; encoding: [0x7d,0xbc,0x0a,0x7e]
+// GFX12: v_rndne_f16_e32 v5, m0 ; encoding: [0x7d,0xbc,0x0a,0x7e]
 
 v_rndne_f16 v5, exec_lo
-// GFX12: v_rndne_f16_e32 v5, exec_lo             ; encoding: [0x7e,0xbc,0x0a,0x7e]
+// GFX12: v_rndne_f16_e32 v5, exec_lo ; encoding: [0x7e,0xbc,0x0a,0x7e]
 
 v_rndne_f16 v5, exec_hi
-// GFX12: v_rndne_f16_e32 v5, exec_hi             ; encoding: [0x7f,0xbc,0x0a,0x7e]
+// GFX12: v_rndne_f16_e32 v5, exec_hi ; encoding: [0x7f,0xbc,0x0a,0x7e]
 
 v_rndne_f16 v5, null
-// GFX12: v_rndne_f16_e32 v5, null                ; encoding: [0x7c,0xbc,0x0a,0x7e]
+// GFX12: v_rndne_f16_e32 v5, null ; encoding: [0x7c,0xbc,0x0a,0x7e]
 
 v_rndne_f16 v5, -1
-// GFX12: v_rndne_f16_e32 v5, -1                  ; encoding: [0xc1,0xbc,0x0a,0x7e]
+// GFX12: v_rndne_f16_e32 v5, -1 ; encoding: [0xc1,0xbc,0x0a,0x7e]
 
 v_rndne_f16 v5, 0.5
-// GFX12: v_rndne_f16_e32 v5, 0.5                 ; encoding: [0xf0,0xbc,0x0a,0x7e]
+// GFX12: v_rndne_f16_e32 v5, 0.5 ; encoding: [0xf0,0xbc,0x0a,0x7e]
 
 v_rndne_f16 v5, src_scc
-// GFX12: v_rndne_f16_e32 v5, src_scc             ; encoding: [0xfd,0xbc,0x0a,0x7e]
+// GFX12: v_rndne_f16_e32 v5, src_scc ; encoding: [0xfd,0xbc,0x0a,0x7e]
 
 v_rndne_f16 v127, 0xfe0b
-// GFX12: v_rndne_f16_e32 v127, 0xfe0b            ; encoding: [0xff,0xbc,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_rndne_f16_e32 v127, 0xfe0b ; encoding: [0xff,0xbc,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_rndne_f32 v5, v1
-// GFX12: v_rndne_f32_e32 v5, v1                  ; encoding: [0x01,0x47,0x0a,0x7e]
+// GFX12: v_rndne_f32_e32 v5, v1 ; encoding: [0x01,0x47,0x0a,0x7e]
 
 v_rndne_f32 v5, v255
-// GFX12: v_rndne_f32_e32 v5, v255                ; encoding: [0xff,0x47,0x0a,0x7e]
+// GFX12: v_rndne_f32_e32 v5, v255 ; encoding: [0xff,0x47,0x0a,0x7e]
 
 v_rndne_f32 v5, s1
-// GFX12: v_rndne_f32_e32 v5, s1                  ; encoding: [0x01,0x46,0x0a,0x7e]
+// GFX12: v_rndne_f32_e32 v5, s1 ; encoding: [0x01,0x46,0x0a,0x7e]
 
 v_rndne_f32 v5, s105
-// GFX12: v_rndne_f32_e32 v5, s105                ; encoding: [0x69,0x46,0x0a,0x7e]
+// GFX12: v_rndne_f32_e32 v5, s105 ; encoding: [0x69,0x46,0x0a,0x7e]
 
 v_rndne_f32 v5, vcc_lo
-// GFX12: v_rndne_f32_e32 v5, vcc_lo              ; encoding: [0x6a,0x46,0x0a,0x7e]
+// GFX12: v_rndne_f32_e32 v5, vcc_lo ; encoding: [0x6a,0x46,0x0a,0x7e]
 
 v_rndne_f32 v5, vcc_hi
-// GFX12: v_rndne_f32_e32 v5, vcc_hi              ; encoding: [0x6b,0x46,0x0a,0x7e]
+// GFX12: v_rndne_f32_e32 v5, vcc_hi ; encoding: [0x6b,0x46,0x0a,0x7e]
 
 v_rndne_f32 v5, ttmp15
-// GFX12: v_rndne_f32_e32 v5, ttmp15              ; encoding: [0x7b,0x46,0x0a,0x7e]
+// GFX12: v_rndne_f32_e32 v5, ttmp15 ; encoding: [0x7b,0x46,0x0a,0x7e]
 
 v_rndne_f32 v5, m0
-// GFX12: v_rndne_f32_e32 v5, m0                  ; encoding: [0x7d,0x46,0x0a,0x7e]
+// GFX12: v_rndne_f32_e32 v5, m0 ; encoding: [0x7d,0x46,0x0a,0x7e]
 
 v_rndne_f32 v5, exec_lo
-// GFX12: v_rndne_f32_e32 v5, exec_lo             ; encoding: [0x7e,0x46,0x0a,0x7e]
+// GFX12: v_rndne_f32_e32 v5, exec_lo ; encoding: [0x7e,0x46,0x0a,0x7e]
 
 v_rndne_f32 v5, exec_hi
-// GFX12: v_rndne_f32_e32 v5, exec_hi             ; encoding: [0x7f,0x46,0x0a,0x7e]
+// GFX12: v_rndne_f32_e32 v5, exec_hi ; encoding: [0x7f,0x46,0x0a,0x7e]
 
 v_rndne_f32 v5, null
-// GFX12: v_rndne_f32_e32 v5, null                ; encoding: [0x7c,0x46,0x0a,0x7e]
+// GFX12: v_rndne_f32_e32 v5, null ; encoding: [0x7c,0x46,0x0a,0x7e]
 
 v_rndne_f32 v5, -1
-// GFX12: v_rndne_f32_e32 v5, -1                  ; encoding: [0xc1,0x46,0x0a,0x7e]
+// GFX12: v_rndne_f32_e32 v5, -1 ; encoding: [0xc1,0x46,0x0a,0x7e]
 
 v_rndne_f32 v5, 0.5
-// GFX12: v_rndne_f32_e32 v5, 0.5                 ; encoding: [0xf0,0x46,0x0a,0x7e]
+// GFX12: v_rndne_f32_e32 v5, 0.5 ; encoding: [0xf0,0x46,0x0a,0x7e]
 
 v_rndne_f32 v5, src_scc
-// GFX12: v_rndne_f32_e32 v5, src_scc             ; encoding: [0xfd,0x46,0x0a,0x7e]
+// GFX12: v_rndne_f32_e32 v5, src_scc ; encoding: [0xfd,0x46,0x0a,0x7e]
 
 v_rndne_f32 v255, 0xaf123456
-// GFX12: v_rndne_f32_e32 v255, 0xaf123456        ; encoding: [0xff,0x46,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_rndne_f32_e32 v255, 0xaf123456 ; encoding: [0xff,0x46,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_rndne_f64 v[5:6], v[1:2]
-// GFX12: v_rndne_f64_e32 v[5:6], v[1:2]          ; encoding: [0x01,0x33,0x0a,0x7e]
+// GFX12: v_rndne_f64_e32 v[5:6], v[1:2] ; encoding: [0x01,0x33,0x0a,0x7e]
 
 v_rndne_f64 v[5:6], v[254:255]
-// GFX12: v_rndne_f64_e32 v[5:6], v[254:255]      ; encoding: [0xfe,0x33,0x0a,0x7e]
+// GFX12: v_rndne_f64_e32 v[5:6], v[254:255] ; encoding: [0xfe,0x33,0x0a,0x7e]
 
 v_rndne_f64 v[5:6], s[2:3]
-// GFX12: v_rndne_f64_e32 v[5:6], s[2:3]          ; encoding: [0x02,0x32,0x0a,0x7e]
+// GFX12: v_rndne_f64_e32 v[5:6], s[2:3] ; encoding: [0x02,0x32,0x0a,0x7e]
 
 v_rndne_f64 v[5:6], s[104:105]
-// GFX12: v_rndne_f64_e32 v[5:6], s[104:105]      ; encoding: [0x68,0x32,0x0a,0x7e]
+// GFX12: v_rndne_f64_e32 v[5:6], s[104:105] ; encoding: [0x68,0x32,0x0a,0x7e]
 
 v_rndne_f64 v[5:6], vcc
-// GFX12: v_rndne_f64_e32 v[5:6], vcc             ; encoding: [0x6a,0x32,0x0a,0x7e]
+// GFX12: v_rndne_f64_e32 v[5:6], vcc ; encoding: [0x6a,0x32,0x0a,0x7e]
 
 v_rndne_f64 v[5:6], ttmp[14:15]
-// GFX12: v_rndne_f64_e32 v[5:6], ttmp[14:15]     ; encoding: [0x7a,0x32,0x0a,0x7e]
+// GFX12: v_rndne_f64_e32 v[5:6], ttmp[14:15] ; encoding: [0x7a,0x32,0x0a,0x7e]
 
 v_rndne_f64 v[5:6], exec
-// GFX12: v_rndne_f64_e32 v[5:6], exec            ; encoding: [0x7e,0x32,0x0a,0x7e]
+// GFX12: v_rndne_f64_e32 v[5:6], exec ; encoding: [0x7e,0x32,0x0a,0x7e]
 
 v_rndne_f64 v[5:6], null
-// GFX12: v_rndne_f64_e32 v[5:6], null            ; encoding: [0x7c,0x32,0x0a,0x7e]
+// GFX12: v_rndne_f64_e32 v[5:6], null ; encoding: [0x7c,0x32,0x0a,0x7e]
 
 v_rndne_f64 v[5:6], -1
-// GFX12: v_rndne_f64_e32 v[5:6], -1              ; encoding: [0xc1,0x32,0x0a,0x7e]
+// GFX12: v_rndne_f64_e32 v[5:6], -1 ; encoding: [0xc1,0x32,0x0a,0x7e]
 
 v_rndne_f64 v[5:6], 0.5
-// GFX12: v_rndne_f64_e32 v[5:6], 0.5             ; encoding: [0xf0,0x32,0x0a,0x7e]
+// GFX12: v_rndne_f64_e32 v[5:6], 0.5 ; encoding: [0xf0,0x32,0x0a,0x7e]
 
 v_rndne_f64 v[5:6], src_scc
-// GFX12: v_rndne_f64_e32 v[5:6], src_scc         ; encoding: [0xfd,0x32,0x0a,0x7e]
+// GFX12: v_rndne_f64_e32 v[5:6], src_scc ; encoding: [0xfd,0x32,0x0a,0x7e]
 
 v_rndne_f64 v[254:255], 0xaf123456
-// GFX12: v_rndne_f64_e32 v[254:255], 0xaf123456  ; encoding: [0xff,0x32,0xfc,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_rndne_f64_e32 v[254:255], 0xaf123456 ; encoding: [0xff,0x32,0xfc,0x7f,0x56,0x34,0x12,0xaf]
 
 v_rsq_f16 v5, v1
-// GFX12: v_rsq_f16_e32 v5, v1                    ; encoding: [0x01,0xad,0x0a,0x7e]
+// GFX12: v_rsq_f16_e32 v5, v1 ; encoding: [0x01,0xad,0x0a,0x7e]
 
 v_rsq_f16 v5, v127
-// GFX12: v_rsq_f16_e32 v5, v127                  ; encoding: [0x7f,0xad,0x0a,0x7e]
+// GFX12: v_rsq_f16_e32 v5, v127 ; encoding: [0x7f,0xad,0x0a,0x7e]
 
 v_rsq_f16 v5, s1
-// GFX12: v_rsq_f16_e32 v5, s1                    ; encoding: [0x01,0xac,0x0a,0x7e]
+// GFX12: v_rsq_f16_e32 v5, s1 ; encoding: [0x01,0xac,0x0a,0x7e]
 
 v_rsq_f16 v5, s105
-// GFX12: v_rsq_f16_e32 v5, s105                  ; encoding: [0x69,0xac,0x0a,0x7e]
+// GFX12: v_rsq_f16_e32 v5, s105 ; encoding: [0x69,0xac,0x0a,0x7e]
 
 v_rsq_f16 v5, vcc_lo
-// GFX12: v_rsq_f16_e32 v5, vcc_lo                ; encoding: [0x6a,0xac,0x0a,0x7e]
+// GFX12: v_rsq_f16_e32 v5, vcc_lo ; encoding: [0x6a,0xac,0x0a,0x7e]
 
 v_rsq_f16 v5, vcc_hi
-// GFX12: v_rsq_f16_e32 v5, vcc_hi                ; encoding: [0x6b,0xac,0x0a,0x7e]
+// GFX12: v_rsq_f16_e32 v5, vcc_hi ; encoding: [0x6b,0xac,0x0a,0x7e]
 
 v_rsq_f16 v5, ttmp15
-// GFX12: v_rsq_f16_e32 v5, ttmp15                ; encoding: [0x7b,0xac,0x0a,0x7e]
+// GFX12: v_rsq_f16_e32 v5, ttmp15 ; encoding: [0x7b,0xac,0x0a,0x7e]
 
 v_rsq_f16 v5, m0
-// GFX12: v_rsq_f16_e32 v5, m0                    ; encoding: [0x7d,0xac,0x0a,0x7e]
+// GFX12: v_rsq_f16_e32 v5, m0 ; encoding: [0x7d,0xac,0x0a,0x7e]
 
 v_rsq_f16 v5, exec_lo
-// GFX12: v_rsq_f16_e32 v5, exec_lo               ; encoding: [0x7e,0xac,0x0a,0x7e]
+// GFX12: v_rsq_f16_e32 v5, exec_lo ; encoding: [0x7e,0xac,0x0a,0x7e]
 
 v_rsq_f16 v5, exec_hi
-// GFX12: v_rsq_f16_e32 v5, exec_hi               ; encoding: [0x7f,0xac,0x0a,0x7e]
+// GFX12: v_rsq_f16_e32 v5, exec_hi ; encoding: [0x7f,0xac,0x0a,0x7e]
 
 v_rsq_f16 v5, null
-// GFX12: v_rsq_f16_e32 v5, null                  ; encoding: [0x7c,0xac,0x0a,0x7e]
+// GFX12: v_rsq_f16_e32 v5, null ; encoding: [0x7c,0xac,0x0a,0x7e]
 
 v_rsq_f16 v5, -1
-// GFX12: v_rsq_f16_e32 v5, -1                    ; encoding: [0xc1,0xac,0x0a,0x7e]
+// GFX12: v_rsq_f16_e32 v5, -1 ; encoding: [0xc1,0xac,0x0a,0x7e]
 
 v_rsq_f16 v5, 0.5
-// GFX12: v_rsq_f16_e32 v5, 0.5                   ; encoding: [0xf0,0xac,0x0a,0x7e]
+// GFX12: v_rsq_f16_e32 v5, 0.5 ; encoding: [0xf0,0xac,0x0a,0x7e]
 
 v_rsq_f16 v5, src_scc
-// GFX12: v_rsq_f16_e32 v5, src_scc               ; encoding: [0xfd,0xac,0x0a,0x7e]
+// GFX12: v_rsq_f16_e32 v5, src_scc ; encoding: [0xfd,0xac,0x0a,0x7e]
 
 v_rsq_f16 v127, 0xfe0b
-// GFX12: v_rsq_f16_e32 v127, 0xfe0b              ; encoding: [0xff,0xac,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_rsq_f16_e32 v127, 0xfe0b ; encoding: [0xff,0xac,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_rsq_f32 v5, v1
-// GFX12: v_rsq_f32_e32 v5, v1                    ; encoding: [0x01,0x5d,0x0a,0x7e]
+// GFX12: v_rsq_f32_e32 v5, v1 ; encoding: [0x01,0x5d,0x0a,0x7e]
 
 v_rsq_f32 v5, v255
-// GFX12: v_rsq_f32_e32 v5, v255                  ; encoding: [0xff,0x5d,0x0a,0x7e]
+// GFX12: v_rsq_f32_e32 v5, v255 ; encoding: [0xff,0x5d,0x0a,0x7e]
 
 v_rsq_f32 v5, s1
-// GFX12: v_rsq_f32_e32 v5, s1                    ; encoding: [0x01,0x5c,0x0a,0x7e]
+// GFX12: v_rsq_f32_e32 v5, s1 ; encoding: [0x01,0x5c,0x0a,0x7e]
 
 v_rsq_f32 v5, s105
-// GFX12: v_rsq_f32_e32 v5, s105                  ; encoding: [0x69,0x5c,0x0a,0x7e]
+// GFX12: v_rsq_f32_e32 v5, s105 ; encoding: [0x69,0x5c,0x0a,0x7e]
 
 v_rsq_f32 v5, vcc_lo
-// GFX12: v_rsq_f32_e32 v5, vcc_lo                ; encoding: [0x6a,0x5c,0x0a,0x7e]
+// GFX12: v_rsq_f32_e32 v5, vcc_lo ; encoding: [0x6a,0x5c,0x0a,0x7e]
 
 v_rsq_f32 v5, vcc_hi
-// GFX12: v_rsq_f32_e32 v5, vcc_hi                ; encoding: [0x6b,0x5c,0x0a,0x7e]
+// GFX12: v_rsq_f32_e32 v5, vcc_hi ; encoding: [0x6b,0x5c,0x0a,0x7e]
 
 v_rsq_f32 v5, ttmp15
-// GFX12: v_rsq_f32_e32 v5, ttmp15                ; encoding: [0x7b,0x5c,0x0a,0x7e]
+// GFX12: v_rsq_f32_e32 v5, ttmp15 ; encoding: [0x7b,0x5c,0x0a,0x7e]
 
 v_rsq_f32 v5, m0
-// GFX12: v_rsq_f32_e32 v5, m0                    ; encoding: [0x7d,0x5c,0x0a,0x7e]
+// GFX12: v_rsq_f32_e32 v5, m0 ; encoding: [0x7d,0x5c,0x0a,0x7e]
 
 v_rsq_f32 v5, exec_lo
-// GFX12: v_rsq_f32_e32 v5, exec_lo               ; encoding: [0x7e,0x5c,0x0a,0x7e]
+// GFX12: v_rsq_f32_e32 v5, exec_lo ; encoding: [0x7e,0x5c,0x0a,0x7e]
 
 v_rsq_f32 v5, exec_hi
-// GFX12: v_rsq_f32_e32 v5, exec_hi               ; encoding: [0x7f,0x5c,0x0a,0x7e]
+// GFX12: v_rsq_f32_e32 v5, exec_hi ; encoding: [0x7f,0x5c,0x0a,0x7e]
 
 v_rsq_f32 v5, null
-// GFX12: v_rsq_f32_e32 v5, null                  ; encoding: [0x7c,0x5c,0x0a,0x7e]
+// GFX12: v_rsq_f32_e32 v5, null ; encoding: [0x7c,0x5c,0x0a,0x7e]
 
 v_rsq_f32 v5, -1
-// GFX12: v_rsq_f32_e32 v5, -1                    ; encoding: [0xc1,0x5c,0x0a,0x7e]
+// GFX12: v_rsq_f32_e32 v5, -1 ; encoding: [0xc1,0x5c,0x0a,0x7e]
 
 v_rsq_f32 v5, 0.5
-// GFX12: v_rsq_f32_e32 v5, 0.5                   ; encoding: [0xf0,0x5c,0x0a,0x7e]
+// GFX12: v_rsq_f32_e32 v5, 0.5 ; encoding: [0xf0,0x5c,0x0a,0x7e]
 
 v_rsq_f32 v5, src_scc
-// GFX12: v_rsq_f32_e32 v5, src_scc               ; encoding: [0xfd,0x5c,0x0a,0x7e]
+// GFX12: v_rsq_f32_e32 v5, src_scc ; encoding: [0xfd,0x5c,0x0a,0x7e]
 
 v_rsq_f32 v255, 0xaf123456
-// GFX12: v_rsq_f32_e32 v255, 0xaf123456          ; encoding: [0xff,0x5c,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_rsq_f32_e32 v255, 0xaf123456 ; encoding: [0xff,0x5c,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_rsq_f64 v[5:6], v[1:2]
-// GFX12: v_rsq_f64_e32 v[5:6], v[1:2]            ; encoding: [0x01,0x63,0x0a,0x7e]
+// GFX12: v_rsq_f64_e32 v[5:6], v[1:2] ; encoding: [0x01,0x63,0x0a,0x7e]
 
 v_rsq_f64 v[5:6], v[254:255]
-// GFX12: v_rsq_f64_e32 v[5:6], v[254:255]        ; encoding: [0xfe,0x63,0x0a,0x7e]
+// GFX12: v_rsq_f64_e32 v[5:6], v[254:255] ; encoding: [0xfe,0x63,0x0a,0x7e]
 
 v_rsq_f64 v[5:6], s[2:3]
-// GFX12: v_rsq_f64_e32 v[5:6], s[2:3]            ; encoding: [0x02,0x62,0x0a,0x7e]
+// GFX12: v_rsq_f64_e32 v[5:6], s[2:3] ; encoding: [0x02,0x62,0x0a,0x7e]
 
 v_rsq_f64 v[5:6], s[104:105]
-// GFX12: v_rsq_f64_e32 v[5:6], s[104:105]        ; encoding: [0x68,0x62,0x0a,0x7e]
+// GFX12: v_rsq_f64_e32 v[5:6], s[104:105] ; encoding: [0x68,0x62,0x0a,0x7e]
 
 v_rsq_f64 v[5:6], vcc
-// GFX12: v_rsq_f64_e32 v[5:6], vcc               ; encoding: [0x6a,0x62,0x0a,0x7e]
+// GFX12: v_rsq_f64_e32 v[5:6], vcc ; encoding: [0x6a,0x62,0x0a,0x7e]
 
 v_rsq_f64 v[5:6], ttmp[14:15]
-// GFX12: v_rsq_f64_e32 v[5:6], ttmp[14:15]       ; encoding: [0x7a,0x62,0x0a,0x7e]
+// GFX12: v_rsq_f64_e32 v[5:6], ttmp[14:15] ; encoding: [0x7a,0x62,0x0a,0x7e]
 
 v_rsq_f64 v[5:6], exec
-// GFX12: v_rsq_f64_e32 v[5:6], exec              ; encoding: [0x7e,0x62,0x0a,0x7e]
+// GFX12: v_rsq_f64_e32 v[5:6], exec ; encoding: [0x7e,0x62,0x0a,0x7e]
 
 v_rsq_f64 v[5:6], null
-// GFX12: v_rsq_f64_e32 v[5:6], null              ; encoding: [0x7c,0x62,0x0a,0x7e]
+// GFX12: v_rsq_f64_e32 v[5:6], null ; encoding: [0x7c,0x62,0x0a,0x7e]
 
 v_rsq_f64 v[5:6], -1
-// GFX12: v_rsq_f64_e32 v[5:6], -1                ; encoding: [0xc1,0x62,0x0a,0x7e]
+// GFX12: v_rsq_f64_e32 v[5:6], -1 ; encoding: [0xc1,0x62,0x0a,0x7e]
 
 v_rsq_f64 v[5:6], 0.5
-// GFX12: v_rsq_f64_e32 v[5:6], 0.5               ; encoding: [0xf0,0x62,0x0a,0x7e]
+// GFX12: v_rsq_f64_e32 v[5:6], 0.5 ; encoding: [0xf0,0x62,0x0a,0x7e]
 
 v_rsq_f64 v[5:6], src_scc
-// GFX12: v_rsq_f64_e32 v[5:6], src_scc           ; encoding: [0xfd,0x62,0x0a,0x7e]
+// GFX12: v_rsq_f64_e32 v[5:6], src_scc ; encoding: [0xfd,0x62,0x0a,0x7e]
 
 v_rsq_f64 v[254:255], 0xaf123456
-// GFX12: v_rsq_f64_e32 v[254:255], 0xaf123456    ; encoding: [0xff,0x62,0xfc,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_rsq_f64_e32 v[254:255], 0xaf123456 ; encoding: [0xff,0x62,0xfc,0x7f,0x56,0x34,0x12,0xaf]
 
 v_sat_pk_u8_i16 v5, v1
-// GFX12: v_sat_pk_u8_i16_e32 v5, v1              ; encoding: [0x01,0xc5,0x0a,0x7e]
+// GFX12: v_sat_pk_u8_i16_e32 v5, v1 ; encoding: [0x01,0xc5,0x0a,0x7e]
 
 v_sat_pk_u8_i16 v5, v255
-// GFX12: v_sat_pk_u8_i16_e32 v5, v255            ; encoding: [0xff,0xc5,0x0a,0x7e]
+// GFX12: v_sat_pk_u8_i16_e32 v5, v255 ; encoding: [0xff,0xc5,0x0a,0x7e]
 
 v_sat_pk_u8_i16 v5, s1
-// GFX12: v_sat_pk_u8_i16_e32 v5, s1              ; encoding: [0x01,0xc4,0x0a,0x7e]
+// GFX12: v_sat_pk_u8_i16_e32 v5, s1 ; encoding: [0x01,0xc4,0x0a,0x7e]
 
 v_sat_pk_u8_i16 v5, s105
-// GFX12: v_sat_pk_u8_i16_e32 v5, s105            ; encoding: [0x69,0xc4,0x0a,0x7e]
+// GFX12: v_sat_pk_u8_i16_e32 v5, s105 ; encoding: [0x69,0xc4,0x0a,0x7e]
 
 v_sat_pk_u8_i16 v5, vcc_lo
-// GFX12: v_sat_pk_u8_i16_e32 v5, vcc_lo          ; encoding: [0x6a,0xc4,0x0a,0x7e]
+// GFX12: v_sat_pk_u8_i16_e32 v5, vcc_lo ; encoding: [0x6a,0xc4,0x0a,0x7e]
 
 v_sat_pk_u8_i16 v5, vcc_hi
-// GFX12: v_sat_pk_u8_i16_e32 v5, vcc_hi          ; encoding: [0x6b,0xc4,0x0a,0x7e]
+// GFX12: v_sat_pk_u8_i16_e32 v5, vcc_hi ; encoding: [0x6b,0xc4,0x0a,0x7e]
 
 v_sat_pk_u8_i16 v5, ttmp15
-// GFX12: v_sat_pk_u8_i16_e32 v5, ttmp15          ; encoding: [0x7b,0xc4,0x0a,0x7e]
+// GFX12: v_sat_pk_u8_i16_e32 v5, ttmp15 ; encoding: [0x7b,0xc4,0x0a,0x7e]
 
 v_sat_pk_u8_i16 v5, m0
-// GFX12: v_sat_pk_u8_i16_e32 v5, m0              ; encoding: [0x7d,0xc4,0x0a,0x7e]
+// GFX12: v_sat_pk_u8_i16_e32 v5, m0 ; encoding: [0x7d,0xc4,0x0a,0x7e]
 
 v_sat_pk_u8_i16 v5, exec_lo
-// GFX12: v_sat_pk_u8_i16_e32 v5, exec_lo         ; encoding: [0x7e,0xc4,0x0a,0x7e]
+// GFX12: v_sat_pk_u8_i16_e32 v5, exec_lo ; encoding: [0x7e,0xc4,0x0a,0x7e]
 
 v_sat_pk_u8_i16 v5, exec_hi
-// GFX12: v_sat_pk_u8_i16_e32 v5, exec_hi         ; encoding: [0x7f,0xc4,0x0a,0x7e]
+// GFX12: v_sat_pk_u8_i16_e32 v5, exec_hi ; encoding: [0x7f,0xc4,0x0a,0x7e]
 
 v_sat_pk_u8_i16 v5, null
-// GFX12: v_sat_pk_u8_i16_e32 v5, null            ; encoding: [0x7c,0xc4,0x0a,0x7e]
+// GFX12: v_sat_pk_u8_i16_e32 v5, null ; encoding: [0x7c,0xc4,0x0a,0x7e]
 
 v_sat_pk_u8_i16 v5, -1
-// GFX12: v_sat_pk_u8_i16_e32 v5, -1              ; encoding: [0xc1,0xc4,0x0a,0x7e]
+// GFX12: v_sat_pk_u8_i16_e32 v5, -1 ; encoding: [0xc1,0xc4,0x0a,0x7e]
 
 v_sat_pk_u8_i16 v5, 0.5
-// GFX12: v_sat_pk_u8_i16_e32 v5, 0.5             ; encoding: [0xf0,0xc4,0x0a,0x7e]
+// GFX12: v_sat_pk_u8_i16_e32 v5, 0.5 ; encoding: [0xf0,0xc4,0x0a,0x7e]
 
 v_sat_pk_u8_i16 v5, src_scc
-// GFX12: v_sat_pk_u8_i16_e32 v5, src_scc         ; encoding: [0xfd,0xc4,0x0a,0x7e]
+// GFX12: v_sat_pk_u8_i16_e32 v5, src_scc ; encoding: [0xfd,0xc4,0x0a,0x7e]
 
 v_sat_pk_u8_i16 v127, 0xfe0b
-// GFX12: v_sat_pk_u8_i16_e32 v127, 0xfe0b        ; encoding: [0xff,0xc4,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_sat_pk_u8_i16_e32 v127, 0xfe0b ; encoding: [0xff,0xc4,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_sin_f16 v5, v1
-// GFX12: v_sin_f16_e32 v5, v1                    ; encoding: [0x01,0xc1,0x0a,0x7e]
+// GFX12: v_sin_f16_e32 v5, v1 ; encoding: [0x01,0xc1,0x0a,0x7e]
 
 v_sin_f16 v5, v127
-// GFX12: v_sin_f16_e32 v5, v127                  ; encoding: [0x7f,0xc1,0x0a,0x7e]
+// GFX12: v_sin_f16_e32 v5, v127 ; encoding: [0x7f,0xc1,0x0a,0x7e]
 
 v_sin_f16 v5, s1
-// GFX12: v_sin_f16_e32 v5, s1                    ; encoding: [0x01,0xc0,0x0a,0x7e]
+// GFX12: v_sin_f16_e32 v5, s1 ; encoding: [0x01,0xc0,0x0a,0x7e]
 
 v_sin_f16 v5, s105
-// GFX12: v_sin_f16_e32 v5, s105                  ; encoding: [0x69,0xc0,0x0a,0x7e]
+// GFX12: v_sin_f16_e32 v5, s105 ; encoding: [0x69,0xc0,0x0a,0x7e]
 
 v_sin_f16 v5, vcc_lo
-// GFX12: v_sin_f16_e32 v5, vcc_lo                ; encoding: [0x6a,0xc0,0x0a,0x7e]
+// GFX12: v_sin_f16_e32 v5, vcc_lo ; encoding: [0x6a,0xc0,0x0a,0x7e]
 
 v_sin_f16 v5, vcc_hi
-// GFX12: v_sin_f16_e32 v5, vcc_hi                ; encoding: [0x6b,0xc0,0x0a,0x7e]
+// GFX12: v_sin_f16_e32 v5, vcc_hi ; encoding: [0x6b,0xc0,0x0a,0x7e]
 
 v_sin_f16 v5, ttmp15
-// GFX12: v_sin_f16_e32 v5, ttmp15                ; encoding: [0x7b,0xc0,0x0a,0x7e]
+// GFX12: v_sin_f16_e32 v5, ttmp15 ; encoding: [0x7b,0xc0,0x0a,0x7e]
 
 v_sin_f16 v5, m0
-// GFX12: v_sin_f16_e32 v5, m0                    ; encoding: [0x7d,0xc0,0x0a,0x7e]
+// GFX12: v_sin_f16_e32 v5, m0 ; encoding: [0x7d,0xc0,0x0a,0x7e]
 
 v_sin_f16 v5, exec_lo
-// GFX12: v_sin_f16_e32 v5, exec_lo               ; encoding: [0x7e,0xc0,0x0a,0x7e]
+// GFX12: v_sin_f16_e32 v5, exec_lo ; encoding: [0x7e,0xc0,0x0a,0x7e]
 
 v_sin_f16 v5, exec_hi
-// GFX12: v_sin_f16_e32 v5, exec_hi               ; encoding: [0x7f,0xc0,0x0a,0x7e]
+// GFX12: v_sin_f16_e32 v5, exec_hi ; encoding: [0x7f,0xc0,0x0a,0x7e]
 
 v_sin_f16 v5, null
-// GFX12: v_sin_f16_e32 v5, null                  ; encoding: [0x7c,0xc0,0x0a,0x7e]
+// GFX12: v_sin_f16_e32 v5, null ; encoding: [0x7c,0xc0,0x0a,0x7e]
 
 v_sin_f16 v5, -1
-// GFX12: v_sin_f16_e32 v5, -1                    ; encoding: [0xc1,0xc0,0x0a,0x7e]
+// GFX12: v_sin_f16_e32 v5, -1 ; encoding: [0xc1,0xc0,0x0a,0x7e]
 
 v_sin_f16 v5, 0.5
-// GFX12: v_sin_f16_e32 v5, 0.5                   ; encoding: [0xf0,0xc0,0x0a,0x7e]
+// GFX12: v_sin_f16_e32 v5, 0.5 ; encoding: [0xf0,0xc0,0x0a,0x7e]
 
 v_sin_f16 v5, src_scc
-// GFX12: v_sin_f16_e32 v5, src_scc               ; encoding: [0xfd,0xc0,0x0a,0x7e]
+// GFX12: v_sin_f16_e32 v5, src_scc ; encoding: [0xfd,0xc0,0x0a,0x7e]
 
 v_sin_f16 v127, 0xfe0b
-// GFX12: v_sin_f16_e32 v127, 0xfe0b              ; encoding: [0xff,0xc0,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_sin_f16_e32 v127, 0xfe0b ; encoding: [0xff,0xc0,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_sin_f32 v5, v1
-// GFX12: v_sin_f32_e32 v5, v1                    ; encoding: [0x01,0x6b,0x0a,0x7e]
+// GFX12: v_sin_f32_e32 v5, v1 ; encoding: [0x01,0x6b,0x0a,0x7e]
 
 v_sin_f32 v5, v255
-// GFX12: v_sin_f32_e32 v5, v255                  ; encoding: [0xff,0x6b,0x0a,0x7e]
+// GFX12: v_sin_f32_e32 v5, v255 ; encoding: [0xff,0x6b,0x0a,0x7e]
 
 v_sin_f32 v5, s1
-// GFX12: v_sin_f32_e32 v5, s1                    ; encoding: [0x01,0x6a,0x0a,0x7e]
+// GFX12: v_sin_f32_e32 v5, s1 ; encoding: [0x01,0x6a,0x0a,0x7e]
 
 v_sin_f32 v5, s105
-// GFX12: v_sin_f32_e32 v5, s105                  ; encoding: [0x69,0x6a,0x0a,0x7e]
+// GFX12: v_sin_f32_e32 v5, s105 ; encoding: [0x69,0x6a,0x0a,0x7e]
 
 v_sin_f32 v5, vcc_lo
-// GFX12: v_sin_f32_e32 v5, vcc_lo                ; encoding: [0x6a,0x6a,0x0a,0x7e]
+// GFX12: v_sin_f32_e32 v5, vcc_lo ; encoding: [0x6a,0x6a,0x0a,0x7e]
 
 v_sin_f32 v5, vcc_hi
-// GFX12: v_sin_f32_e32 v5, vcc_hi                ; encoding: [0x6b,0x6a,0x0a,0x7e]
+// GFX12: v_sin_f32_e32 v5, vcc_hi ; encoding: [0x6b,0x6a,0x0a,0x7e]
 
 v_sin_f32 v5, ttmp15
-// GFX12: v_sin_f32_e32 v5, ttmp15                ; encoding: [0x7b,0x6a,0x0a,0x7e]
+// GFX12: v_sin_f32_e32 v5, ttmp15 ; encoding: [0x7b,0x6a,0x0a,0x7e]
 
 v_sin_f32 v5, m0
-// GFX12: v_sin_f32_e32 v5, m0                    ; encoding: [0x7d,0x6a,0x0a,0x7e]
+// GFX12: v_sin_f32_e32 v5, m0 ; encoding: [0x7d,0x6a,0x0a,0x7e]
 
 v_sin_f32 v5, exec_lo
-// GFX12: v_sin_f32_e32 v5, exec_lo               ; encoding: [0x7e,0x6a,0x0a,0x7e]
+// GFX12: v_sin_f32_e32 v5, exec_lo ; encoding: [0x7e,0x6a,0x0a,0x7e]
 
 v_sin_f32 v5, exec_hi
-// GFX12: v_sin_f32_e32 v5, exec_hi               ; encoding: [0x7f,0x6a,0x0a,0x7e]
+// GFX12: v_sin_f32_e32 v5, exec_hi ; encoding: [0x7f,0x6a,0x0a,0x7e]
 
 v_sin_f32 v5, null
-// GFX12: v_sin_f32_e32 v5, null                  ; encoding: [0x7c,0x6a,0x0a,0x7e]
+// GFX12: v_sin_f32_e32 v5, null ; encoding: [0x7c,0x6a,0x0a,0x7e]
 
 v_sin_f32 v5, -1
-// GFX12: v_sin_f32_e32 v5, -1                    ; encoding: [0xc1,0x6a,0x0a,0x7e]
+// GFX12: v_sin_f32_e32 v5, -1 ; encoding: [0xc1,0x6a,0x0a,0x7e]
 
 v_sin_f32 v5, 0.5
-// GFX12: v_sin_f32_e32 v5, 0.5                   ; encoding: [0xf0,0x6a,0x0a,0x7e]
+// GFX12: v_sin_f32_e32 v5, 0.5 ; encoding: [0xf0,0x6a,0x0a,0x7e]
 
 v_sin_f32 v5, src_scc
-// GFX12: v_sin_f32_e32 v5, src_scc               ; encoding: [0xfd,0x6a,0x0a,0x7e]
+// GFX12: v_sin_f32_e32 v5, src_scc ; encoding: [0xfd,0x6a,0x0a,0x7e]
 
 v_sin_f32 v255, 0xaf123456
-// GFX12: v_sin_f32_e32 v255, 0xaf123456          ; encoding: [0xff,0x6a,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_sin_f32_e32 v255, 0xaf123456 ; encoding: [0xff,0x6a,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_sqrt_f16 v5, v1
-// GFX12: v_sqrt_f16_e32 v5, v1                   ; encoding: [0x01,0xab,0x0a,0x7e]
+// GFX12: v_sqrt_f16_e32 v5, v1 ; encoding: [0x01,0xab,0x0a,0x7e]
 
 v_sqrt_f16 v5, v127
-// GFX12: v_sqrt_f16_e32 v5, v127                 ; encoding: [0x7f,0xab,0x0a,0x7e]
+// GFX12: v_sqrt_f16_e32 v5, v127 ; encoding: [0x7f,0xab,0x0a,0x7e]
 
 v_sqrt_f16 v5, s1
-// GFX12: v_sqrt_f16_e32 v5, s1                   ; encoding: [0x01,0xaa,0x0a,0x7e]
+// GFX12: v_sqrt_f16_e32 v5, s1 ; encoding: [0x01,0xaa,0x0a,0x7e]
 
 v_sqrt_f16 v5, s105
-// GFX12: v_sqrt_f16_e32 v5, s105                 ; encoding: [0x69,0xaa,0x0a,0x7e]
+// GFX12: v_sqrt_f16_e32 v5, s105 ; encoding: [0x69,0xaa,0x0a,0x7e]
 
 v_sqrt_f16 v5, vcc_lo
-// GFX12: v_sqrt_f16_e32 v5, vcc_lo               ; encoding: [0x6a,0xaa,0x0a,0x7e]
+// GFX12: v_sqrt_f16_e32 v5, vcc_lo ; encoding: [0x6a,0xaa,0x0a,0x7e]
 
 v_sqrt_f16 v5, vcc_hi
-// GFX12: v_sqrt_f16_e32 v5, vcc_hi               ; encoding: [0x6b,0xaa,0x0a,0x7e]
+// GFX12: v_sqrt_f16_e32 v5, vcc_hi ; encoding: [0x6b,0xaa,0x0a,0x7e]
 
 v_sqrt_f16 v5, ttmp15
-// GFX12: v_sqrt_f16_e32 v5, ttmp15               ; encoding: [0x7b,0xaa,0x0a,0x7e]
+// GFX12: v_sqrt_f16_e32 v5, ttmp15 ; encoding: [0x7b,0xaa,0x0a,0x7e]
 
 v_sqrt_f16 v5, m0
-// GFX12: v_sqrt_f16_e32 v5, m0                   ; encoding: [0x7d,0xaa,0x0a,0x7e]
+// GFX12: v_sqrt_f16_e32 v5, m0 ; encoding: [0x7d,0xaa,0x0a,0x7e]
 
 v_sqrt_f16 v5, exec_lo
-// GFX12: v_sqrt_f16_e32 v5, exec_lo              ; encoding: [0x7e,0xaa,0x0a,0x7e]
+// GFX12: v_sqrt_f16_e32 v5, exec_lo ; encoding: [0x7e,0xaa,0x0a,0x7e]
 
 v_sqrt_f16 v5, exec_hi
-// GFX12: v_sqrt_f16_e32 v5, exec_hi              ; encoding: [0x7f,0xaa,0x0a,0x7e]
+// GFX12: v_sqrt_f16_e32 v5, exec_hi ; encoding: [0x7f,0xaa,0x0a,0x7e]
 
 v_sqrt_f16 v5, null
-// GFX12: v_sqrt_f16_e32 v5, null                 ; encoding: [0x7c,0xaa,0x0a,0x7e]
+// GFX12: v_sqrt_f16_e32 v5, null ; encoding: [0x7c,0xaa,0x0a,0x7e]
 
 v_sqrt_f16 v5, -1
-// GFX12: v_sqrt_f16_e32 v5, -1                   ; encoding: [0xc1,0xaa,0x0a,0x7e]
+// GFX12: v_sqrt_f16_e32 v5, -1 ; encoding: [0xc1,0xaa,0x0a,0x7e]
 
 v_sqrt_f16 v5, 0.5
-// GFX12: v_sqrt_f16_e32 v5, 0.5                  ; encoding: [0xf0,0xaa,0x0a,0x7e]
+// GFX12: v_sqrt_f16_e32 v5, 0.5 ; encoding: [0xf0,0xaa,0x0a,0x7e]
 
 v_sqrt_f16 v5, src_scc
-// GFX12: v_sqrt_f16_e32 v5, src_scc              ; encoding: [0xfd,0xaa,0x0a,0x7e]
+// GFX12: v_sqrt_f16_e32 v5, src_scc ; encoding: [0xfd,0xaa,0x0a,0x7e]
 
 v_sqrt_f16 v127, 0xfe0b
-// GFX12: v_sqrt_f16_e32 v127, 0xfe0b             ; encoding: [0xff,0xaa,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_sqrt_f16_e32 v127, 0xfe0b ; encoding: [0xff,0xaa,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_sqrt_f32 v5, v1
-// GFX12: v_sqrt_f32_e32 v5, v1                   ; encoding: [0x01,0x67,0x0a,0x7e]
+// GFX12: v_sqrt_f32_e32 v5, v1 ; encoding: [0x01,0x67,0x0a,0x7e]
 
 v_sqrt_f32 v5, v255
-// GFX12: v_sqrt_f32_e32 v5, v255                 ; encoding: [0xff,0x67,0x0a,0x7e]
+// GFX12: v_sqrt_f32_e32 v5, v255 ; encoding: [0xff,0x67,0x0a,0x7e]
 
 v_sqrt_f32 v5, s1
-// GFX12: v_sqrt_f32_e32 v5, s1                   ; encoding: [0x01,0x66,0x0a,0x7e]
+// GFX12: v_sqrt_f32_e32 v5, s1 ; encoding: [0x01,0x66,0x0a,0x7e]
 
 v_sqrt_f32 v5, s105
-// GFX12: v_sqrt_f32_e32 v5, s105                 ; encoding: [0x69,0x66,0x0a,0x7e]
+// GFX12: v_sqrt_f32_e32 v5, s105 ; encoding: [0x69,0x66,0x0a,0x7e]
 
 v_sqrt_f32 v5, vcc_lo
-// GFX12: v_sqrt_f32_e32 v5, vcc_lo               ; encoding: [0x6a,0x66,0x0a,0x7e]
+// GFX12: v_sqrt_f32_e32 v5, vcc_lo ; encoding: [0x6a,0x66,0x0a,0x7e]
 
 v_sqrt_f32 v5, vcc_hi
-// GFX12: v_sqrt_f32_e32 v5, vcc_hi               ; encoding: [0x6b,0x66,0x0a,0x7e]
+// GFX12: v_sqrt_f32_e32 v5, vcc_hi ; encoding: [0x6b,0x66,0x0a,0x7e]
 
 v_sqrt_f32 v5, ttmp15
-// GFX12: v_sqrt_f32_e32 v5, ttmp15               ; encoding: [0x7b,0x66,0x0a,0x7e]
+// GFX12: v_sqrt_f32_e32 v5, ttmp15 ; encoding: [0x7b,0x66,0x0a,0x7e]
 
 v_sqrt_f32 v5, m0
-// GFX12: v_sqrt_f32_e32 v5, m0                   ; encoding: [0x7d,0x66,0x0a,0x7e]
+// GFX12: v_sqrt_f32_e32 v5, m0 ; encoding: [0x7d,0x66,0x0a,0x7e]
 
 v_sqrt_f32 v5, exec_lo
-// GFX12: v_sqrt_f32_e32 v5, exec_lo              ; encoding: [0x7e,0x66,0x0a,0x7e]
+// GFX12: v_sqrt_f32_e32 v5, exec_lo ; encoding: [0x7e,0x66,0x0a,0x7e]
 
 v_sqrt_f32 v5, exec_hi
-// GFX12: v_sqrt_f32_e32 v5, exec_hi              ; encoding: [0x7f,0x66,0x0a,0x7e]
+// GFX12: v_sqrt_f32_e32 v5, exec_hi ; encoding: [0x7f,0x66,0x0a,0x7e]
 
 v_sqrt_f32 v5, null
-// GFX12: v_sqrt_f32_e32 v5, null                 ; encoding: [0x7c,0x66,0x0a,0x7e]
+// GFX12: v_sqrt_f32_e32 v5, null ; encoding: [0x7c,0x66,0x0a,0x7e]
 
 v_sqrt_f32 v5, -1
-// GFX12: v_sqrt_f32_e32 v5, -1                   ; encoding: [0xc1,0x66,0x0a,0x7e]
+// GFX12: v_sqrt_f32_e32 v5, -1 ; encoding: [0xc1,0x66,0x0a,0x7e]
 
 v_sqrt_f32 v5, 0.5
-// GFX12: v_sqrt_f32_e32 v5, 0.5                  ; encoding: [0xf0,0x66,0x0a,0x7e]
+// GFX12: v_sqrt_f32_e32 v5, 0.5 ; encoding: [0xf0,0x66,0x0a,0x7e]
 
 v_sqrt_f32 v5, src_scc
-// GFX12: v_sqrt_f32_e32 v5, src_scc              ; encoding: [0xfd,0x66,0x0a,0x7e]
+// GFX12: v_sqrt_f32_e32 v5, src_scc ; encoding: [0xfd,0x66,0x0a,0x7e]
 
 v_sqrt_f32 v255, 0xaf123456
-// GFX12: v_sqrt_f32_e32 v255, 0xaf123456         ; encoding: [0xff,0x66,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_sqrt_f32_e32 v255, 0xaf123456 ; encoding: [0xff,0x66,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_sqrt_f64 v[5:6], v[1:2]
-// GFX12: v_sqrt_f64_e32 v[5:6], v[1:2]           ; encoding: [0x01,0x69,0x0a,0x7e]
+// GFX12: v_sqrt_f64_e32 v[5:6], v[1:2] ; encoding: [0x01,0x69,0x0a,0x7e]
 
 v_sqrt_f64 v[5:6], v[254:255]
-// GFX12: v_sqrt_f64_e32 v[5:6], v[254:255]       ; encoding: [0xfe,0x69,0x0a,0x7e]
+// GFX12: v_sqrt_f64_e32 v[5:6], v[254:255] ; encoding: [0xfe,0x69,0x0a,0x7e]
 
 v_sqrt_f64 v[5:6], s[2:3]
-// GFX12: v_sqrt_f64_e32 v[5:6], s[2:3]           ; encoding: [0x02,0x68,0x0a,0x7e]
+// GFX12: v_sqrt_f64_e32 v[5:6], s[2:3] ; encoding: [0x02,0x68,0x0a,0x7e]
 
 v_sqrt_f64 v[5:6], s[104:105]
-// GFX12: v_sqrt_f64_e32 v[5:6], s[104:105]       ; encoding: [0x68,0x68,0x0a,0x7e]
+// GFX12: v_sqrt_f64_e32 v[5:6], s[104:105] ; encoding: [0x68,0x68,0x0a,0x7e]
 
 v_sqrt_f64 v[5:6], vcc
-// GFX12: v_sqrt_f64_e32 v[5:6], vcc              ; encoding: [0x6a,0x68,0x0a,0x7e]
+// GFX12: v_sqrt_f64_e32 v[5:6], vcc ; encoding: [0x6a,0x68,0x0a,0x7e]
 
 v_sqrt_f64 v[5:6], ttmp[14:15]
-// GFX12: v_sqrt_f64_e32 v[5:6], ttmp[14:15]      ; encoding: [0x7a,0x68,0x0a,0x7e]
+// GFX12: v_sqrt_f64_e32 v[5:6], ttmp[14:15] ; encoding: [0x7a,0x68,0x0a,0x7e]
 
 v_sqrt_f64 v[5:6], exec
-// GFX12: v_sqrt_f64_e32 v[5:6], exec             ; encoding: [0x7e,0x68,0x0a,0x7e]
+// GFX12: v_sqrt_f64_e32 v[5:6], exec ; encoding: [0x7e,0x68,0x0a,0x7e]
 
 v_sqrt_f64 v[5:6], null
-// GFX12: v_sqrt_f64_e32 v[5:6], null             ; encoding: [0x7c,0x68,0x0a,0x7e]
+// GFX12: v_sqrt_f64_e32 v[5:6], null ; encoding: [0x7c,0x68,0x0a,0x7e]
 
 v_sqrt_f64 v[5:6], -1
-// GFX12: v_sqrt_f64_e32 v[5:6], -1               ; encoding: [0xc1,0x68,0x0a,0x7e]
+// GFX12: v_sqrt_f64_e32 v[5:6], -1 ; encoding: [0xc1,0x68,0x0a,0x7e]
 
 v_sqrt_f64 v[5:6], 0.5
-// GFX12: v_sqrt_f64_e32 v[5:6], 0.5              ; encoding: [0xf0,0x68,0x0a,0x7e]
+// GFX12: v_sqrt_f64_e32 v[5:6], 0.5 ; encoding: [0xf0,0x68,0x0a,0x7e]
 
 v_sqrt_f64 v[5:6], src_scc
-// GFX12: v_sqrt_f64_e32 v[5:6], src_scc          ; encoding: [0xfd,0x68,0x0a,0x7e]
+// GFX12: v_sqrt_f64_e32 v[5:6], src_scc ; encoding: [0xfd,0x68,0x0a,0x7e]
 
 v_sqrt_f64 v[254:255], 0xaf123456
-// GFX12: v_sqrt_f64_e32 v[254:255], 0xaf123456   ; encoding: [0xff,0x68,0xfc,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_sqrt_f64_e32 v[254:255], 0xaf123456 ; encoding: [0xff,0x68,0xfc,0x7f,0x56,0x34,0x12,0xaf]
 
 v_swap_b16 v5.l, v1.h
-// GFX12: v_swap_b16  v5.l, v1.h                  ; encoding: [0x81,0xcd,0x0a,0x7e]
+// GFX12: v_swap_b16  v5.l, v1.h ; encoding: [0x81,0xcd,0x0a,0x7e]
 
 v_swap_b16 v5.h, v1.l
-// GFX12: v_swap_b16  v5.h, v1.l                  ; encoding: [0x01,0xcd,0x0a,0x7f]
+// GFX12: v_swap_b16  v5.h, v1.l ; encoding: [0x01,0xcd,0x0a,0x7f]
 
 v_swap_b16 v127.l, v127.l
-// GFX12: v_swap_b16  v127.l, v127.l              ; encoding: [0x7f,0xcd,0xfe,0x7e]
+// GFX12: v_swap_b16  v127.l, v127.l ; encoding: [0x7f,0xcd,0xfe,0x7e]
 
 v_swap_b32 v5, v1
-// GFX12: v_swap_b32 v5, v1                       ; encoding: [0x01,0xcb,0x0a,0x7e]
+// GFX12: v_swap_b32 v5, v1 ; encoding: [0x01,0xcb,0x0a,0x7e]
 
 v_swap_b32 v255, v255
-// GFX12: v_swap_b32 v255, v255                   ; encoding: [0xff,0xcb,0xfe,0x7f]
+// GFX12: v_swap_b32 v255, v255 ; encoding: [0xff,0xcb,0xfe,0x7f]
 
 v_swaprel_b32 v5, v1
-// GFX12: v_swaprel_b32 v5, v1                    ; encoding: [0x01,0xd1,0x0a,0x7e]
+// GFX12: v_swaprel_b32 v5, v1 ; encoding: [0x01,0xd1,0x0a,0x7e]
 
 v_swaprel_b32 v255, v255
-// GFX12: v_swaprel_b32 v255, v255                ; encoding: [0xff,0xd1,0xfe,0x7f]
+// GFX12: v_swaprel_b32 v255, v255 ; encoding: [0xff,0xd1,0xfe,0x7f]
 
 v_trunc_f16 v5, v1
-// GFX12: v_trunc_f16_e32 v5, v1                  ; encoding: [0x01,0xbb,0x0a,0x7e]
+// GFX12: v_trunc_f16_e32 v5, v1 ; encoding: [0x01,0xbb,0x0a,0x7e]
 
 v_trunc_f16 v5, v127
-// GFX12: v_trunc_f16_e32 v5, v127                ; encoding: [0x7f,0xbb,0x0a,0x7e]
+// GFX12: v_trunc_f16_e32 v5, v127 ; encoding: [0x7f,0xbb,0x0a,0x7e]
 
 v_trunc_f16 v5, s1
-// GFX12: v_trunc_f16_e32 v5, s1                  ; encoding: [0x01,0xba,0x0a,0x7e]
+// GFX12: v_trunc_f16_e32 v5, s1 ; encoding: [0x01,0xba,0x0a,0x7e]
 
 v_trunc_f16 v5, s105
-// GFX12: v_trunc_f16_e32 v5, s105                ; encoding: [0x69,0xba,0x0a,0x7e]
+// GFX12: v_trunc_f16_e32 v5, s105 ; encoding: [0x69,0xba,0x0a,0x7e]
 
 v_trunc_f16 v5, vcc_lo
-// GFX12: v_trunc_f16_e32 v5, vcc_lo              ; encoding: [0x6a,0xba,0x0a,0x7e]
+// GFX12: v_trunc_f16_e32 v5, vcc_lo ; encoding: [0x6a,0xba,0x0a,0x7e]
 
 v_trunc_f16 v5, vcc_hi
-// GFX12: v_trunc_f16_e32 v5, vcc_hi              ; encoding: [0x6b,0xba,0x0a,0x7e]
+// GFX12: v_trunc_f16_e32 v5, vcc_hi ; encoding: [0x6b,0xba,0x0a,0x7e]
 
 v_trunc_f16 v5, ttmp15
-// GFX12: v_trunc_f16_e32 v5, ttmp15              ; encoding: [0x7b,0xba,0x0a,0x7e]
+// GFX12: v_trunc_f16_e32 v5, ttmp15 ; encoding: [0x7b,0xba,0x0a,0x7e]
 
 v_trunc_f16 v5, m0
-// GFX12: v_trunc_f16_e32 v5, m0                  ; encoding: [0x7d,0xba,0x0a,0x7e]
+// GFX12: v_trunc_f16_e32 v5, m0 ; encoding: [0x7d,0xba,0x0a,0x7e]
 
 v_trunc_f16 v5, exec_lo
-// GFX12: v_trunc_f16_e32 v5, exec_lo             ; encoding: [0x7e,0xba,0x0a,0x7e]
+// GFX12: v_trunc_f16_e32 v5, exec_lo ; encoding: [0x7e,0xba,0x0a,0x7e]
 
 v_trunc_f16 v5, exec_hi
-// GFX12: v_trunc_f16_e32 v5, exec_hi             ; encoding: [0x7f,0xba,0x0a,0x7e]
+// GFX12: v_trunc_f16_e32 v5, exec_hi ; encoding: [0x7f,0xba,0x0a,0x7e]
 
 v_trunc_f16 v5, null
-// GFX12: v_trunc_f16_e32 v5, null                ; encoding: [0x7c,0xba,0x0a,0x7e]
+// GFX12: v_trunc_f16_e32 v5, null ; encoding: [0x7c,0xba,0x0a,0x7e]
 
 v_trunc_f16 v5, -1
-// GFX12: v_trunc_f16_e32 v5, -1                  ; encoding: [0xc1,0xba,0x0a,0x7e]
+// GFX12: v_trunc_f16_e32 v5, -1 ; encoding: [0xc1,0xba,0x0a,0x7e]
 
 v_trunc_f16 v5, 0.5
-// GFX12: v_trunc_f16_e32 v5, 0.5                 ; encoding: [0xf0,0xba,0x0a,0x7e]
+// GFX12: v_trunc_f16_e32 v5, 0.5 ; encoding: [0xf0,0xba,0x0a,0x7e]
 
 v_trunc_f16 v5, src_scc
-// GFX12: v_trunc_f16_e32 v5, src_scc             ; encoding: [0xfd,0xba,0x0a,0x7e]
+// GFX12: v_trunc_f16_e32 v5, src_scc ; encoding: [0xfd,0xba,0x0a,0x7e]
 
 v_trunc_f16 v127, 0xfe0b
-// GFX12: v_trunc_f16_e32 v127, 0xfe0b            ; encoding: [0xff,0xba,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
+// GFX12: v_trunc_f16_e32 v127, 0xfe0b ; encoding: [0xff,0xba,0xfe,0x7e,0x0b,0xfe,0x00,0x00]
 
 v_trunc_f32 v5, v1
-// GFX12: v_trunc_f32_e32 v5, v1                  ; encoding: [0x01,0x43,0x0a,0x7e]
+// GFX12: v_trunc_f32_e32 v5, v1 ; encoding: [0x01,0x43,0x0a,0x7e]
 
 v_trunc_f32 v5, v255
-// GFX12: v_trunc_f32_e32 v5, v255                ; encoding: [0xff,0x43,0x0a,0x7e]
+// GFX12: v_trunc_f32_e32 v5, v255 ; encoding: [0xff,0x43,0x0a,0x7e]
 
 v_trunc_f32 v5, s1
-// GFX12: v_trunc_f32_e32 v5, s1                  ; encoding: [0x01,0x42,0x0a,0x7e]
+// GFX12: v_trunc_f32_e32 v5, s1 ; encoding: [0x01,0x42,0x0a,0x7e]
 
 v_trunc_f32 v5, s105
-// GFX12: v_trunc_f32_e32 v5, s105                ; encoding: [0x69,0x42,0x0a,0x7e]
+// GFX12: v_trunc_f32_e32 v5, s105 ; encoding: [0x69,0x42,0x0a,0x7e]
 
 v_trunc_f32 v5, vcc_lo
-// GFX12: v_trunc_f32_e32 v5, vcc_lo              ; encoding: [0x6a,0x42,0x0a,0x7e]
+// GFX12: v_trunc_f32_e32 v5, vcc_lo ; encoding: [0x6a,0x42,0x0a,0x7e]
 
 v_trunc_f32 v5, vcc_hi
-// GFX12: v_trunc_f32_e32 v5, vcc_hi              ; encoding: [0x6b,0x42,0x0a,0x7e]
+// GFX12: v_trunc_f32_e32 v5, vcc_hi ; encoding: [0x6b,0x42,0x0a,0x7e]
 
 v_trunc_f32 v5, ttmp15
-// GFX12: v_trunc_f32_e32 v5, ttmp15              ; encoding: [0x7b,0x42,0x0a,0x7e]
+// GFX12: v_trunc_f32_e32 v5, ttmp15 ; encoding: [0x7b,0x42,0x0a,0x7e]
 
 v_trunc_f32 v5, m0
-// GFX12: v_trunc_f32_e32 v5, m0                  ; encoding: [0x7d,0x42,0x0a,0x7e]
+// GFX12: v_trunc_f32_e32 v5, m0 ; encoding: [0x7d,0x42,0x0a,0x7e]
 
 v_trunc_f32 v5, exec_lo
-// GFX12: v_trunc_f32_e32 v5, exec_lo             ; encoding: [0x7e,0x42,0x0a,0x7e]
+// GFX12: v_trunc_f32_e32 v5, exec_lo ; encoding: [0x7e,0x42,0x0a,0x7e]
 
 v_trunc_f32 v5, exec_hi
-// GFX12: v_trunc_f32_e32 v5, exec_hi             ; encoding: [0x7f,0x42,0x0a,0x7e]
+// GFX12: v_trunc_f32_e32 v5, exec_hi ; encoding: [0x7f,0x42,0x0a,0x7e]
 
 v_trunc_f32 v5, null
-// GFX12: v_trunc_f32_e32 v5, null                ; encoding: [0x7c,0x42,0x0a,0x7e]
+// GFX12: v_trunc_f32_e32 v5, null ; encoding: [0x7c,0x42,0x0a,0x7e]
 
 v_trunc_f32 v5, -1
-// GFX12: v_trunc_f32_e32 v5, -1                  ; encoding: [0xc1,0x42,0x0a,0x7e]
+// GFX12: v_trunc_f32_e32 v5, -1 ; encoding: [0xc1,0x42,0x0a,0x7e]
 
 v_trunc_f32 v5, 0.5
-// GFX12: v_trunc_f32_e32 v5, 0.5                 ; encoding: [0xf0,0x42,0x0a,0x7e]
+// GFX12: v_trunc_f32_e32 v5, 0.5 ; encoding: [0xf0,0x42,0x0a,0x7e]
 
 v_trunc_f32 v5, src_scc
-// GFX12: v_trunc_f32_e32 v5, src_scc             ; encoding: [0xfd,0x42,0x0a,0x7e]
+// GFX12: v_trunc_f32_e32 v5, src_scc ; encoding: [0xfd,0x42,0x0a,0x7e]
 
 v_trunc_f32 v255, 0xaf123456
-// GFX12: v_trunc_f32_e32 v255, 0xaf123456        ; encoding: [0xff,0x42,0xfe,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_trunc_f32_e32 v255, 0xaf123456 ; encoding: [0xff,0x42,0xfe,0x7f,0x56,0x34,0x12,0xaf]
 
 v_trunc_f64 v[5:6], v[1:2]
-// GFX12: v_trunc_f64_e32 v[5:6], v[1:2]          ; encoding: [0x01,0x2f,0x0a,0x7e]
+// GFX12: v_trunc_f64_e32 v[5:6], v[1:2] ; encoding: [0x01,0x2f,0x0a,0x7e]
 
 v_trunc_f64 v[5:6], v[254:255]
-// GFX12: v_trunc_f64_e32 v[5:6], v[254:255]      ; encoding: [0xfe,0x2f,0x0a,0x7e]
+// GFX12: v_trunc_f64_e32 v[5:6], v[254:255] ; encoding: [0xfe,0x2f,0x0a,0x7e]
 
 v_trunc_f64 v[5:6], s[2:3]
-// GFX12: v_trunc_f64_e32 v[5:6], s[2:3]          ; encoding: [0x02,0x2e,0x0a,0x7e]
+// GFX12: v_trunc_f64_e32 v[5:6], s[2:3] ; encoding: [0x02,0x2e,0x0a,0x7e]
 
 v_trunc_f64 v[5:6], s[104:105]
-// GFX12: v_trunc_f64_e32 v[5:6], s[104:105]      ; encoding: [0x68,0x2e,0x0a,0x7e]
+// GFX12: v_trunc_f64_e32 v[5:6], s[104:105] ; encoding: [0x68,0x2e,0x0a,0x7e]
 
 v_trunc_f64 v[5:6], vcc
-// GFX12: v_trunc_f64_e32 v[5:6], vcc             ; encoding: [0x6a,0x2e,0x0a,0x7e]
+// GFX12: v_trunc_f64_e32 v[5:6], vcc ; encoding: [0x6a,0x2e,0x0a,0x7e]
 
 v_trunc_f64 v[5:6], ttmp[14:15]
-// GFX12: v_trunc_f64_e32 v[5:6], ttmp[14:15]     ; encoding: [0x7a,0x2e,0x0a,0x7e]
+// GFX12: v_trunc_f64_e32 v[5:6], ttmp[14:15] ; encoding: [0x7a,0x2e,0x0a,0x7e]
 
 v_trunc_f64 v[5:6], exec
-// GFX12: v_trunc_f64_e32 v[5:6], exec            ; encoding: [0x7e,0x2e,0x0a,0x7e]
+// GFX12: v_trunc_f64_e32 v[5:6], exec ; encoding: [0x7e,0x2e,0x0a,0x7e]
 
 v_trunc_f64 v[5:6], null
-// GFX12: v_trunc_f64_e32 v[5:6], null            ; encoding: [0x7c,0x2e,0x0a,0x7e]
+// GFX12: v_trunc_f64_e32 v[5:6], null ; encoding: [0x7c,0x2e,0x0a,0x7e]
 
 v_trunc_f64 v[5:6], -1
-// GFX12: v_trunc_f64_e32 v[5:6], -1              ; encoding: [0xc1,0x2e,0x0a,0x7e]
+// GFX12: v_trunc_f64_e32 v[5:6], -1 ; encoding: [0xc1,0x2e,0x0a,0x7e]
 
 v_trunc_f64 v[5:6], 0.5
-// GFX12: v_trunc_f64_e32 v[5:6], 0.5             ; encoding: [0xf0,0x2e,0x0a,0x7e]
+// GFX12: v_trunc_f64_e32 v[5:6], 0.5 ; encoding: [0xf0,0x2e,0x0a,0x7e]
 
 v_trunc_f64 v[5:6], src_scc
-// GFX12: v_trunc_f64_e32 v[5:6], src_scc         ; encoding: [0xfd,0x2e,0x0a,0x7e]
+// GFX12: v_trunc_f64_e32 v[5:6], src_scc ; encoding: [0xfd,0x2e,0x0a,0x7e]
 
 v_trunc_f64 v[254:255], 0xaf123456
-// GFX12: v_trunc_f64_e32 v[254:255], 0xaf123456  ; encoding: [0xff,0x2e,0xfc,0x7f,0x56,0x34,0x12,0xaf]
+// GFX12: v_trunc_f64_e32 v[254:255], 0xaf123456 ; encoding: [0xff,0x2e,0xfc,0x7f,0x56,0x34,0x12,0xaf]
diff --git a/llvm/tools/llvm-mc/llvm-mc.cpp b/llvm/tools/llvm-mc/llvm-mc.cpp
index 2b2cd906219a938..898d79b9233b9a3 100644
--- a/llvm/tools/llvm-mc/llvm-mc.cpp
+++ b/llvm/tools/llvm-mc/llvm-mc.cpp
@@ -104,6 +104,10 @@ static cl::opt<bool>
                      cl::desc("Preserve Comments in outputted assembly"),
                      cl::cat(MCCategory));
 
+static cl::opt<unsigned> CommentColumn("comment-column",
+                                       cl::desc("Asm comments indentation"),
+                                       cl::init(40));
+
 enum OutputFileType {
   OFT_Null,
   OFT_AssemblyFile,
@@ -405,6 +409,7 @@ int main(int argc, char **argv) {
     }
   }
   MAI->setPreserveAsmComments(PreserveComments);
+  MAI->setCommentColumn(CommentColumn);
 
   // Package up features to be passed to target/subtarget
   std::string FeaturesStr;

From af81b4f9f98f6ac89f876637f065d6525e374468 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Valentin=20Clement=20=28=E3=83=90=E3=83=AC=E3=83=B3?=
 =?UTF-8?q?=E3=82=BF=E3=82=A4=E3=83=B3=20=E3=82=AF=E3=83=AC=E3=83=A1?=
 =?UTF-8?q?=E3=83=B3=29?= <clementval@gmail.com>
Date: Fri, 16 Aug 2024 11:56:22 -0700
Subject: [PATCH 230/441] Reland "[flang][cuda][driver] Make sure flang does
 not switch to cc1 (#104613)"

Flang is switch to cc1 when we use `-x cuda`. Make sure we can use fc1
with cuda fortran input.

The current pipeline will fail at MLIR level for the moment.
---
 clang/lib/Driver/Types.cpp        | 3 +++
 flang/test/Driver/cuda-option.f90 | 3 ++-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/clang/lib/Driver/Types.cpp b/clang/lib/Driver/Types.cpp
index 2b9b391c19c9fd0..3de45b00b4d00fd 100644
--- a/clang/lib/Driver/Types.cpp
+++ b/clang/lib/Driver/Types.cpp
@@ -170,6 +170,9 @@ bool types::isAcceptedByFlang(ID Id) {
   case TY_LLVM_IR:
   case TY_LLVM_BC:
     return true;
+  case TY_PP_CUDA:
+  case TY_CUDA:
+    return true;
   }
 }
 
diff --git a/flang/test/Driver/cuda-option.f90 b/flang/test/Driver/cuda-option.f90
index 562f8683b0ff7d8..3fe5fd22cc984ba 100644
--- a/flang/test/Driver/cuda-option.f90
+++ b/flang/test/Driver/cuda-option.f90
@@ -1,5 +1,6 @@
 ! Test -fcuda option
 ! RUN: %flang_fc1 -cpp -x cuda -fdebug-unparse %s -o - | FileCheck %s
+! RUN: %flang -cpp -fsyntax-only -x cuda %s -o -
 ! RUN: not %flang_fc1 -cpp %s -o - 2>&1 | FileCheck %s --check-prefix=ERROR
 program main
 #if _CUDA
@@ -12,4 +13,4 @@ program main
 ! CHECK: INTEGER :: var = 1
 ! CHECK: INTEGER, DEVICE :: dvar
 
-! ERROR: cuda-option.f90:8:19: error: expected end of statement
+! ERROR: cuda-option.f90:{{.*}}:{{.*}}: error: expected end of statement

From 8c5d76ac508ece0b41cfd33247d1f0551c80a9e8 Mon Sep 17 00:00:00 2001
From: vporpo <vporpodas@google.com>
Date: Fri, 16 Aug 2024 13:16:43 -0700
Subject: [PATCH 231/441] [SandboxIR][Tracker][NFC] GenericSetterWithIdx
 (#104615)

This patch adds a generic change tracker class, similar to
GenericSetter, but for getter/setter functions that also take an index
argument. For example: `Foo:get(Idx)` and `Foo::set(Idx, Val)`. These
setter/getter patterns are common enough that using a common
implementation seems beneficial.
---
 llvm/include/llvm/SandboxIR/Tracker.h | 48 ++++++++++++---------------
 llvm/lib/SandboxIR/SandboxIR.cpp      | 23 ++++++++-----
 llvm/lib/SandboxIR/Tracker.cpp        | 41 -----------------------
 3 files changed, 37 insertions(+), 75 deletions(-)

diff --git a/llvm/include/llvm/SandboxIR/Tracker.h b/llvm/include/llvm/SandboxIR/Tracker.h
index 80efbf9e8f6e082..2b4dfafa3b0e5d0 100644
--- a/llvm/include/llvm/SandboxIR/Tracker.h
+++ b/llvm/include/llvm/SandboxIR/Tracker.h
@@ -96,25 +96,6 @@ class UseSet : public IRChangeBase {
 #endif
 };
 
-class PHISetIncoming : public IRChangeBase {
-  PHINode *PHI;
-  unsigned Idx;
-  PointerUnion<Value *, BasicBlock *> OrigValueOrBB;
-
-public:
-  enum class What {
-    Value,
-    Block,
-  };
-  PHISetIncoming(PHINode *PHI, unsigned Idx, What What);
-  void revert(Tracker &Tracker) final;
-  void accept() final {}
-#ifndef NDEBUG
-  void dump(raw_ostream &OS) const final { OS << "PHISetIncoming"; }
-  LLVM_DUMP_METHOD void dump() const final;
-#endif
-};
-
 class PHIRemoveIncoming : public IRChangeBase {
   PHINode *PHI;
   unsigned RemovedIdx;
@@ -250,18 +231,33 @@ class GenericSetter final : public IRChangeBase {
 #endif
 };
 
-class CallBrInstSetIndirectDest : public IRChangeBase {
-  CallBrInst *CallBr;
+/// Similar to GenericSetter but the setters/getters have an index as their
+/// first argument. This is commont in cases like: getOperand(unsigned Idx)
+template <auto GetterFn, auto SetterFn>
+class GenericSetterWithIdx final : public IRChangeBase {
+  /// Helper for getting the class type from the getter
+  template <typename ClassT, typename RetT>
+  static ClassT getClassTypeFromGetter(RetT (ClassT::*Fn)(unsigned) const);
+  template <typename ClassT, typename RetT>
+  static ClassT getClassTypeFromGetter(RetT (ClassT::*Fn)(unsigned));
+
+  using InstrT = decltype(getClassTypeFromGetter(GetterFn));
+  using SavedValT = std::invoke_result_t<decltype(GetterFn), InstrT, unsigned>;
+  InstrT *I;
+  SavedValT OrigVal;
   unsigned Idx;
-  BasicBlock *OrigIndirectDest;
 
 public:
-  CallBrInstSetIndirectDest(CallBrInst *CallBr, unsigned Idx);
-  void revert(Tracker &Tracker) final;
+  GenericSetterWithIdx(InstrT *I, unsigned Idx)
+      : I(I), OrigVal((I->*GetterFn)(Idx)), Idx(Idx) {}
+  void revert(Tracker &Tracker) final { (I->*SetterFn)(Idx, OrigVal); }
   void accept() final {}
 #ifndef NDEBUG
-  void dump(raw_ostream &OS) const final { OS << "CallBrInstSetIndirectDest"; }
-  LLVM_DUMP_METHOD void dump() const final;
+  void dump(raw_ostream &OS) const final { OS << "GenericSetterWithIdx"; }
+  LLVM_DUMP_METHOD void dump() const final {
+    dump(dbgs());
+    dbgs() << "\n";
+  }
 #endif
 };
 
diff --git a/llvm/lib/SandboxIR/SandboxIR.cpp b/llvm/lib/SandboxIR/SandboxIR.cpp
index 96c5af807d1001a..559d9ae7c00e7a6 100644
--- a/llvm/lib/SandboxIR/SandboxIR.cpp
+++ b/llvm/lib/SandboxIR/SandboxIR.cpp
@@ -1031,7 +1031,10 @@ void CallBrInst::setDefaultDest(BasicBlock *BB) {
   cast<llvm::CallBrInst>(Val)->setDefaultDest(cast<llvm::BasicBlock>(BB->Val));
 }
 void CallBrInst::setIndirectDest(unsigned Idx, BasicBlock *BB) {
-  Ctx.getTracker().emplaceIfTracking<CallBrInstSetIndirectDest>(this, Idx);
+  Ctx.getTracker()
+      .emplaceIfTracking<GenericSetterWithIdx<&CallBrInst::getIndirectDest,
+                                              &CallBrInst::setIndirectDest>>(
+          this, Idx);
   cast<llvm::CallBrInst>(Val)->setIndirectDest(Idx,
                                                cast<llvm::BasicBlock>(BB->Val));
 }
@@ -1102,10 +1105,10 @@ Value *PHINode::getIncomingValue(unsigned Idx) const {
   return Ctx.getValue(cast<llvm::PHINode>(Val)->getIncomingValue(Idx));
 }
 void PHINode::setIncomingValue(unsigned Idx, Value *V) {
-  auto &Tracker = Ctx.getTracker();
-  Tracker.emplaceIfTracking<PHISetIncoming>(this, Idx,
-                                            PHISetIncoming::What::Value);
-
+  Ctx.getTracker()
+      .emplaceIfTracking<GenericSetterWithIdx<&PHINode::getIncomingValue,
+                                              &PHINode::setIncomingValue>>(this,
+                                                                           Idx);
   cast<llvm::PHINode>(Val)->setIncomingValue(Idx, V->Val);
 }
 BasicBlock *PHINode::getIncomingBlock(unsigned Idx) const {
@@ -1118,9 +1121,13 @@ BasicBlock *PHINode::getIncomingBlock(const Use &U) const {
   return cast<BasicBlock>(Ctx.getValue(BB));
 }
 void PHINode::setIncomingBlock(unsigned Idx, BasicBlock *BB) {
-  auto &Tracker = Ctx.getTracker();
-  Tracker.emplaceIfTracking<PHISetIncoming>(this, Idx,
-                                            PHISetIncoming::What::Block);
+  // Helper to disambiguate PHINode::getIncomingBlock(unsigned).
+  constexpr BasicBlock *(PHINode::*GetIncomingBlockFn)(unsigned) const =
+      &PHINode::getIncomingBlock;
+  Ctx.getTracker()
+      .emplaceIfTracking<
+          GenericSetterWithIdx<GetIncomingBlockFn, &PHINode::setIncomingBlock>>(
+          this, Idx);
   cast<llvm::PHINode>(Val)->setIncomingBlock(Idx,
                                              cast<llvm::BasicBlock>(BB->Val));
 }
diff --git a/llvm/lib/SandboxIR/Tracker.cpp b/llvm/lib/SandboxIR/Tracker.cpp
index 23f4125dba026ba..baf218aa41c1cdd 100644
--- a/llvm/lib/SandboxIR/Tracker.cpp
+++ b/llvm/lib/SandboxIR/Tracker.cpp
@@ -27,32 +27,6 @@ void UseSwap::dump() const {
 }
 #endif // NDEBUG
 
-PHISetIncoming::PHISetIncoming(PHINode *PHI, unsigned Idx, What What)
-    : PHI(PHI), Idx(Idx) {
-  switch (What) {
-  case What::Value:
-    OrigValueOrBB = PHI->getIncomingValue(Idx);
-    break;
-  case What::Block:
-    OrigValueOrBB = PHI->getIncomingBlock(Idx);
-    break;
-  }
-}
-
-void PHISetIncoming::revert(Tracker &Tracker) {
-  if (auto *V = OrigValueOrBB.dyn_cast<Value *>())
-    PHI->setIncomingValue(Idx, V);
-  else
-    PHI->setIncomingBlock(Idx, OrigValueOrBB.get<BasicBlock *>());
-}
-
-#ifndef NDEBUG
-void PHISetIncoming::dump() const {
-  dump(dbgs());
-  dbgs() << "\n";
-}
-#endif // NDEBUG
-
 PHIRemoveIncoming::PHIRemoveIncoming(PHINode *PHI, unsigned RemovedIdx)
     : PHI(PHI), RemovedIdx(RemovedIdx) {
   RemovedV = PHI->getIncomingValue(RemovedIdx);
@@ -186,21 +160,6 @@ void RemoveFromParent::dump() const {
 }
 #endif
 
-CallBrInstSetIndirectDest::CallBrInstSetIndirectDest(CallBrInst *CallBr,
-                                                     unsigned Idx)
-    : CallBr(CallBr), Idx(Idx) {
-  OrigIndirectDest = CallBr->getIndirectDest(Idx);
-}
-void CallBrInstSetIndirectDest::revert(Tracker &Tracker) {
-  CallBr->setIndirectDest(Idx, OrigIndirectDest);
-}
-#ifndef NDEBUG
-void CallBrInstSetIndirectDest::dump() const {
-  dump(dbgs());
-  dbgs() << "\n";
-}
-#endif
-
 MoveInstr::MoveInstr(Instruction *MovedI) : MovedI(MovedI) {
   if (auto *NextI = MovedI->getNextNode())
     NextInstrOrBB = NextI;

From de9338fc625d8d67c18032492b2e743cd53f4b37 Mon Sep 17 00:00:00 2001
From: Valentin Clement <clementval@gmail.com>
Date: Fri, 16 Aug 2024 13:19:01 -0700
Subject: [PATCH 232/441] Revert "Reland "[flang][cuda][driver] Make sure flang
 does not switch to cc1 (#104613)""

This reverts commit af81b4f9f98f6ac89f876637f065d6525e374468.
---
 clang/lib/Driver/Types.cpp        | 3 ---
 flang/test/Driver/cuda-option.f90 | 3 +--
 2 files changed, 1 insertion(+), 5 deletions(-)

diff --git a/clang/lib/Driver/Types.cpp b/clang/lib/Driver/Types.cpp
index 3de45b00b4d00fd..2b9b391c19c9fd0 100644
--- a/clang/lib/Driver/Types.cpp
+++ b/clang/lib/Driver/Types.cpp
@@ -170,9 +170,6 @@ bool types::isAcceptedByFlang(ID Id) {
   case TY_LLVM_IR:
   case TY_LLVM_BC:
     return true;
-  case TY_PP_CUDA:
-  case TY_CUDA:
-    return true;
   }
 }
 
diff --git a/flang/test/Driver/cuda-option.f90 b/flang/test/Driver/cuda-option.f90
index 3fe5fd22cc984ba..562f8683b0ff7d8 100644
--- a/flang/test/Driver/cuda-option.f90
+++ b/flang/test/Driver/cuda-option.f90
@@ -1,6 +1,5 @@
 ! Test -fcuda option
 ! RUN: %flang_fc1 -cpp -x cuda -fdebug-unparse %s -o - | FileCheck %s
-! RUN: %flang -cpp -fsyntax-only -x cuda %s -o -
 ! RUN: not %flang_fc1 -cpp %s -o - 2>&1 | FileCheck %s --check-prefix=ERROR
 program main
 #if _CUDA
@@ -13,4 +12,4 @@ program main
 ! CHECK: INTEGER :: var = 1
 ! CHECK: INTEGER, DEVICE :: dvar
 
-! ERROR: cuda-option.f90:{{.*}}:{{.*}}: error: expected end of statement
+! ERROR: cuda-option.f90:8:19: error: expected end of statement

From 1a8817aebe12b8d4708ee849fb8f792cffb7ae85 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Valentin=20Clement=20=28=E3=83=90=E3=83=AC=E3=83=B3?=
 =?UTF-8?q?=E3=82=BF=E3=82=A4=E3=83=B3=20=E3=82=AF=E3=83=AC=E3=83=A1?=
 =?UTF-8?q?=E3=83=B3=29?= <clementval@gmail.com>
Date: Fri, 16 Aug 2024 11:56:22 -0700
Subject: [PATCH 233/441] Reland "[flang][cuda][driver] Make sure flang does
 not switch to cc1 (#104613)"

Flang is switch to cc1 when we use `-x cuda`. Make sure we can use fc1
with cuda fortran input.

The current pipeline will fail at MLIR level for the moment.
---
 clang/lib/Driver/Types.cpp        | 3 +++
 flang/test/Driver/cuda-option.f90 | 3 ++-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/clang/lib/Driver/Types.cpp b/clang/lib/Driver/Types.cpp
index 2b9b391c19c9fd0..3de45b00b4d00fd 100644
--- a/clang/lib/Driver/Types.cpp
+++ b/clang/lib/Driver/Types.cpp
@@ -170,6 +170,9 @@ bool types::isAcceptedByFlang(ID Id) {
   case TY_LLVM_IR:
   case TY_LLVM_BC:
     return true;
+  case TY_PP_CUDA:
+  case TY_CUDA:
+    return true;
   }
 }
 
diff --git a/flang/test/Driver/cuda-option.f90 b/flang/test/Driver/cuda-option.f90
index 562f8683b0ff7d8..5dda307d44eddd5 100644
--- a/flang/test/Driver/cuda-option.f90
+++ b/flang/test/Driver/cuda-option.f90
@@ -1,5 +1,6 @@
 ! Test -fcuda option
 ! RUN: %flang_fc1 -cpp -x cuda -fdebug-unparse %s -o - | FileCheck %s
+! RUN: %flang --target=x86_64-linux-gnu -cpp -fsyntax-only -x cuda %s -o -
 ! RUN: not %flang_fc1 -cpp %s -o - 2>&1 | FileCheck %s --check-prefix=ERROR
 program main
 #if _CUDA
@@ -12,4 +13,4 @@ program main
 ! CHECK: INTEGER :: var = 1
 ! CHECK: INTEGER, DEVICE :: dvar
 
-! ERROR: cuda-option.f90:8:19: error: expected end of statement
+! ERROR: cuda-option.f90:{{.*}}:{{.*}}: error: expected end of statement

From fd9aa5e40d903e8411b924d94348c0e348cf50db Mon Sep 17 00:00:00 2001
From: Valentin Clement <clementval@gmail.com>
Date: Fri, 16 Aug 2024 13:46:36 -0700
Subject: [PATCH 234/441] [flang][cuda] Remove run line

---
 flang/test/Driver/cuda-option.f90 | 1 -
 1 file changed, 1 deletion(-)

diff --git a/flang/test/Driver/cuda-option.f90 b/flang/test/Driver/cuda-option.f90
index 5dda307d44eddd5..0740ed509a07768 100644
--- a/flang/test/Driver/cuda-option.f90
+++ b/flang/test/Driver/cuda-option.f90
@@ -1,6 +1,5 @@
 ! Test -fcuda option
 ! RUN: %flang_fc1 -cpp -x cuda -fdebug-unparse %s -o - | FileCheck %s
-! RUN: %flang --target=x86_64-linux-gnu -cpp -fsyntax-only -x cuda %s -o -
 ! RUN: not %flang_fc1 -cpp %s -o - 2>&1 | FileCheck %s --check-prefix=ERROR
 program main
 #if _CUDA

From 13779ec29ed724666407bd60f8c4cc8228107ea5 Mon Sep 17 00:00:00 2001
From: Sergei Barannikov <barannikov88@gmail.com>
Date: Fri, 16 Aug 2024 23:49:01 +0300
Subject: [PATCH 235/441] [DataLayout] Refactor parsing of "p" specification
 (#104583)

Split off of #104545 to reduce patch size.
Similar to #104546, this introduces `parseSize` and `parseAlignment`,
which are improved versions of `getInt` tailored for specific needs.

I'm not a GTest guru, so the tests are not ideal.
---
 llvm/include/llvm/IR/DataLayout.h    |  12 +-
 llvm/lib/IR/DataLayout.cpp           | 181 +++++++++++++---------
 llvm/test/Assembler/getInt.ll        |   3 -
 llvm/unittests/IR/DataLayoutTest.cpp | 222 +++++++++++++++++++++++----
 4 files changed, 308 insertions(+), 110 deletions(-)
 delete mode 100644 llvm/test/Assembler/getInt.ll

diff --git a/llvm/include/llvm/IR/DataLayout.h b/llvm/include/llvm/IR/DataLayout.h
index 228b723ee663ff7..85dae10883c67cc 100644
--- a/llvm/include/llvm/IR/DataLayout.h
+++ b/llvm/include/llvm/IR/DataLayout.h
@@ -154,10 +154,9 @@ class DataLayout {
   /// Returns the default address space specification if not found.
   const PointerSpec &getPointerSpec(uint32_t AddrSpace) const;
 
-  /// Attempts to set the specification for pointer in the given address space.
-  /// Returns an error description on failure.
-  Error setPointerSpec(uint32_t AddrSpace, uint32_t BitWidth, Align ABIAlign,
-                       Align PrefAlign, uint32_t IndexBitWidth);
+  /// Sets or updates the specification for pointer in the given address space.
+  void setPointerSpec(uint32_t AddrSpace, uint32_t BitWidth, Align ABIAlign,
+                      Align PrefAlign, uint32_t IndexBitWidth);
 
   /// Internal helper to get alignment for integer of given bitwidth.
   Align getIntegerAlignment(uint32_t BitWidth, bool abi_or_pref) const;
@@ -165,8 +164,11 @@ class DataLayout {
   /// Internal helper method that returns requested alignment for type.
   Align getAlignment(Type *Ty, bool abi_or_pref) const;
 
+  /// Attempts to parse a pointer specification ('p').
+  Error parsePointerSpec(StringRef Spec);
+
   /// Attempts to parse a single specification.
-  Error parseSpecification(StringRef Specification);
+  Error parseSpecification(StringRef Spec);
 
   /// Attempts to parse a data layout string.
   Error parseLayoutString(StringRef LayoutString);
diff --git a/llvm/lib/IR/DataLayout.cpp b/llvm/lib/IR/DataLayout.cpp
index 7043fa410192a35..425a9024c897d4e 100644
--- a/llvm/lib/IR/DataLayout.cpp
+++ b/llvm/lib/IR/DataLayout.cpp
@@ -288,6 +288,54 @@ static Error parseAddrSpace(StringRef Str, unsigned &AddrSpace) {
   return Error::success();
 }
 
+/// Attempts to parse a size component of a specification.
+static Error parseSize(StringRef Str, unsigned &BitWidth,
+                       StringRef Name = "size") {
+  if (Str.empty())
+    return createStringError(Name + " component cannot be empty");
+
+  if (!to_integer(Str, BitWidth, 10) || BitWidth == 0 || !isUInt<24>(BitWidth))
+    return createStringError(Name + " must be a non-zero 24-bit integer");
+
+  return Error::success();
+}
+
+/// Attempts to parse an alignment component of a specification.
+///
+/// On success, returns the value converted to byte amount in \p Alignment.
+/// If the value is zero and \p AllowZero is true, \p Alignment is set to one.
+///
+/// Return an error in a number of cases:
+/// - \p Str is empty or contains characters other than decimal digits;
+/// - the value is zero and \p AllowZero is false;
+/// - the value is too large;
+/// - the value is not a multiple of the byte width;
+/// - the value converted to byte amount is not not a power of two.
+static Error parseAlignment(StringRef Str, Align &Alignment, StringRef Name,
+                            bool AllowZero = false) {
+  if (Str.empty())
+    return createStringError(Name + " alignment component cannot be empty");
+
+  unsigned Value;
+  if (!to_integer(Str, Value, 10) || !isUInt<16>(Value))
+    return createStringError(Name + " alignment must be a 16-bit integer");
+
+  if (Value == 0) {
+    if (!AllowZero)
+      return createStringError(Name + " alignment must be non-zero");
+    Alignment = Align(1);
+    return Error::success();
+  }
+
+  constexpr unsigned ByteWidth = 8;
+  if (Value % ByteWidth || !isPowerOf2_32(Value / ByteWidth))
+    return createStringError(
+        Name + " alignment must be a power of two times the byte width");
+
+  Alignment = Align(Value / ByteWidth);
+  return Error::success();
+}
+
 /// Checked version of split, to ensure mandatory subparts.
 static Error split(StringRef Str, char Separator,
                    std::pair<StringRef, StringRef> &Split) {
@@ -328,6 +376,56 @@ static Error getAddrSpace(StringRef R, unsigned &AddrSpace) {
   return Error::success();
 }
 
+Error DataLayout::parsePointerSpec(StringRef Spec) {
+  // p[<n>]:<size>:<abi>[:<pref>[:<idx>]]
+  SmallVector<StringRef, 5> Components;
+  assert(Spec.front() == 'p');
+  Spec.drop_front().split(Components, ':');
+
+  if (Components.size() < 3 || Components.size() > 5)
+    return createSpecFormatError("p[<n>]:<size>:<abi>[:<pref>[:<idx>]]");
+
+  // Address space. Optional, defaults to 0.
+  unsigned AddrSpace = 0;
+  if (!Components[0].empty())
+    if (Error Err = parseAddrSpace(Components[0], AddrSpace))
+      return Err;
+
+  // Size. Required, cannot be zero.
+  unsigned BitWidth;
+  if (Error Err = parseSize(Components[1], BitWidth, "pointer size"))
+    return Err;
+
+  // ABI alignment. Required, cannot be zero.
+  Align ABIAlign;
+  if (Error Err = parseAlignment(Components[2], ABIAlign, "ABI"))
+    return Err;
+
+  // Preferred alignment. Optional, defaults to the ABI alignment.
+  // Cannot be zero.
+  Align PrefAlign = ABIAlign;
+  if (Components.size() > 3)
+    if (Error Err = parseAlignment(Components[3], PrefAlign, "preferred"))
+      return Err;
+
+  if (PrefAlign < ABIAlign)
+    return createStringError(
+        "preferred alignment cannot be less than the ABI alignment");
+
+  // Index size. Optional, defaults to pointer size. Cannot be zero.
+  unsigned IndexBitWidth = BitWidth;
+  if (Components.size() > 4)
+    if (Error Err = parseSize(Components[4], IndexBitWidth, "index size"))
+      return Err;
+
+  if (IndexBitWidth > BitWidth)
+    return createStringError(
+        "index size cannot be larger than the pointer size");
+
+  setPointerSpec(AddrSpace, BitWidth, ABIAlign, PrefAlign, IndexBitWidth);
+  return Error::success();
+}
+
 Error DataLayout::parseSpecification(StringRef Spec) {
   // The "ni" specifier is the only two-character specifier. Handle it first.
   if (Spec.starts_with("ni")) {
@@ -349,6 +447,13 @@ Error DataLayout::parseSpecification(StringRef Spec) {
     return Error::success();
   }
 
+  // The rest of the specifiers are single-character.
+  assert(!Spec.empty() && "Empty specification is handled by the caller");
+  char Specifier = Spec.front();
+
+  if (Specifier == 'p')
+    return parsePointerSpec(Spec);
+
   // Split at ':'.
   std::pair<StringRef, StringRef> Split;
   if (Error Err = ::split(Spec, ':', Split))
@@ -372,69 +477,6 @@ Error DataLayout::parseSpecification(StringRef Spec) {
   case 'e':
     BigEndian = false;
     break;
-  case 'p': {
-    // Address space.
-    unsigned AddrSpace = 0;
-    if (!Tok.empty())
-      if (Error Err = getInt(Tok, AddrSpace))
-        return Err;
-    if (!isUInt<24>(AddrSpace))
-      return reportError("Invalid address space, must be a 24-bit integer");
-
-    // Size.
-    if (Rest.empty())
-      return reportError(
-          "Missing size specification for pointer in datalayout string");
-    if (Error Err = ::split(Rest, ':', Split))
-      return Err;
-    unsigned PointerMemSize;
-    if (Error Err = getInt(Tok, PointerMemSize))
-      return Err;
-    if (!PointerMemSize)
-      return reportError("Invalid pointer size of 0 bytes");
-
-    // ABI alignment.
-    if (Rest.empty())
-      return reportError(
-          "Missing alignment specification for pointer in datalayout string");
-    if (Error Err = ::split(Rest, ':', Split))
-      return Err;
-    unsigned PointerABIAlign;
-    if (Error Err = getIntInBytes(Tok, PointerABIAlign))
-      return Err;
-    if (!isPowerOf2_64(PointerABIAlign))
-      return reportError("Pointer ABI alignment must be a power of 2");
-
-    // Size of index used in GEP for address calculation.
-    // The parameter is optional. By default it is equal to size of pointer.
-    unsigned IndexSize = PointerMemSize;
-
-    // Preferred alignment.
-    unsigned PointerPrefAlign = PointerABIAlign;
-    if (!Rest.empty()) {
-      if (Error Err = ::split(Rest, ':', Split))
-        return Err;
-      if (Error Err = getIntInBytes(Tok, PointerPrefAlign))
-        return Err;
-      if (!isPowerOf2_64(PointerPrefAlign))
-        return reportError("Pointer preferred alignment must be a power of 2");
-
-      // Now read the index. It is the second optional parameter here.
-      if (!Rest.empty()) {
-        if (Error Err = ::split(Rest, ':', Split))
-          return Err;
-        if (Error Err = getInt(Tok, IndexSize))
-          return Err;
-        if (!IndexSize)
-          return reportError("Invalid index size of 0 bytes");
-      }
-    }
-    if (Error Err = setPointerSpec(AddrSpace, PointerMemSize,
-                                   assumeAligned(PointerABIAlign),
-                                   assumeAligned(PointerPrefAlign), IndexSize))
-      return Err;
-    break;
-  }
   case 'i':
   case 'v':
   case 'f':
@@ -680,15 +722,9 @@ DataLayout::getPointerSpec(uint32_t AddrSpace) const {
   return PointerSpecs[0];
 }
 
-Error DataLayout::setPointerSpec(uint32_t AddrSpace, uint32_t BitWidth,
-                                 Align ABIAlign, Align PrefAlign,
-                                 uint32_t IndexBitWidth) {
-  if (PrefAlign < ABIAlign)
-    return reportError(
-        "Preferred alignment cannot be less than the ABI alignment");
-  if (IndexBitWidth > BitWidth)
-    return reportError("Index width cannot be larger than pointer width");
-
+void DataLayout::setPointerSpec(uint32_t AddrSpace, uint32_t BitWidth,
+                                Align ABIAlign, Align PrefAlign,
+                                uint32_t IndexBitWidth) {
   auto I = lower_bound(PointerSpecs, AddrSpace, LessPointerAddrSpace());
   if (I == PointerSpecs.end() || I->AddrSpace != AddrSpace) {
     PointerSpecs.insert(I, PointerSpec{AddrSpace, BitWidth, ABIAlign, PrefAlign,
@@ -699,7 +735,6 @@ Error DataLayout::setPointerSpec(uint32_t AddrSpace, uint32_t BitWidth,
     I->PrefAlign = PrefAlign;
     I->IndexBitWidth = IndexBitWidth;
   }
-  return Error::success();
 }
 
 Align DataLayout::getIntegerAlignment(uint32_t BitWidth,
diff --git a/llvm/test/Assembler/getInt.ll b/llvm/test/Assembler/getInt.ll
deleted file mode 100644
index 8e2537ae6cf1d8e..000000000000000
--- a/llvm/test/Assembler/getInt.ll
+++ /dev/null
@@ -1,3 +0,0 @@
-; RUN: not opt < %s 2>&1 | grep 'not a number, or does not fit in an unsigned int'
-
-target datalayout = "p:4294967296:64:64"
diff --git a/llvm/unittests/IR/DataLayoutTest.cpp b/llvm/unittests/IR/DataLayoutTest.cpp
index a40ac5b83a83e40..4b236b2d165f9c3 100644
--- a/llvm/unittests/IR/DataLayoutTest.cpp
+++ b/llvm/unittests/IR/DataLayoutTest.cpp
@@ -30,26 +30,12 @@ TEST(DataLayoutTest, ParseErrors) {
   EXPECT_THAT_EXPECTED(
       DataLayout::parse("n0"),
       FailedWithMessage("Zero width native integer type in datalayout string"));
-  EXPECT_THAT_EXPECTED(
-      DataLayout::parse("p16777216:64:64:64"),
-      FailedWithMessage("Invalid address space, must be a 24-bit integer"));
   EXPECT_THAT_EXPECTED(
       DataLayout::parse("a1:64"),
       FailedWithMessage("Sized aggregate specification in datalayout string"));
   EXPECT_THAT_EXPECTED(
       DataLayout::parse("a:"),
       FailedWithMessage("Trailing separator in datalayout string"));
-  EXPECT_THAT_EXPECTED(
-      DataLayout::parse("p:48:52"),
-      FailedWithMessage("number of bits must be a byte width multiple"));
-  EXPECT_THAT_EXPECTED(
-      DataLayout::parse("e-p"),
-      FailedWithMessage(
-          "Missing size specification for pointer in datalayout string"));
-  EXPECT_THAT_EXPECTED(
-      DataLayout::parse("e-p:64"),
-      FailedWithMessage(
-          "Missing alignment specification for pointer in datalayout string"));
   EXPECT_THAT_EXPECTED(
       DataLayout::parse("m"),
       FailedWithMessage("Expected mangling specifier in datalayout string"));
@@ -79,21 +65,6 @@ TEST(DataLayoutTest, ParseErrors) {
   EXPECT_THAT_EXPECTED(
       DataLayout::parse("i16777216:16:16"),
       FailedWithMessage("Invalid bit width, must be a 24-bit integer"));
-  EXPECT_THAT_EXPECTED(
-      DataLayout::parse("p:32:32:16"),
-      FailedWithMessage(
-          "Preferred alignment cannot be less than the ABI alignment"));
-  EXPECT_THAT_EXPECTED(DataLayout::parse("p:0:32:32"),
-                       FailedWithMessage("Invalid pointer size of 0 bytes"));
-  EXPECT_THAT_EXPECTED(
-      DataLayout::parse("p:64:24:64"),
-      FailedWithMessage("Pointer ABI alignment must be a power of 2"));
-  EXPECT_THAT_EXPECTED(
-      DataLayout::parse("p:64:64:24"),
-      FailedWithMessage("Pointer preferred alignment must be a power of 2"));
-  EXPECT_THAT_EXPECTED(
-      DataLayout::parse("p:64:64:64:128"),
-      FailedWithMessage("Index width cannot be larger than pointer width"));
   EXPECT_THAT_EXPECTED(
       DataLayout::parse("v128:0:128"),
       FailedWithMessage(
@@ -134,6 +105,103 @@ TEST(DataLayout, LayoutStringFormat) {
         FailedWithMessage("empty specification is not allowed"));
 }
 
+TEST(DataLayout, ParsePointerSpec) {
+  for (StringRef Str :
+       {"p:16:8", "p:16:16:64", "p:32:64:64:32", "p0:32:64", "p42:64:32:32",
+        "p16777215:32:32:64:8", "p16777215:16777215:32768:32768:16777215"})
+    EXPECT_THAT_EXPECTED(DataLayout::parse(Str), Succeeded());
+
+  for (StringRef Str :
+       {"p", "p0", "p:32", "p0:32", "p:32:32:32:32:32", "p0:32:32:32:32:32"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(Str),
+        FailedWithMessage("malformed specification, must be of the form "
+                          "\"p[<n>]:<size>:<abi>[:<pref>[:<idx>]]\""));
+
+  // address space
+  for (StringRef Str : {"p0x0:32:32", "px:32:32:32", "p16777216:32:32:32:32"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(Str),
+        FailedWithMessage("address space must be a 24-bit integer"));
+
+  // pointer size
+  for (StringRef Str : {"p::32", "p0::32"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(Str),
+        FailedWithMessage("pointer size component cannot be empty"));
+
+  for (StringRef Str : {"p:0:32", "p0:0x1:32:32", "p42:16777216:32:32:32"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(Str),
+        FailedWithMessage("pointer size must be a non-zero 24-bit integer"));
+
+  // ABI alignment
+  for (StringRef Str : {"p:32:", "p0:32::32", "p42:32::32:32"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(Str),
+        FailedWithMessage("ABI alignment component cannot be empty"));
+
+  for (StringRef Str : {"p:32:x", "p0:32:0x20:32", "p42:32:65536:32:32"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(Str),
+        FailedWithMessage("ABI alignment must be a 16-bit integer"));
+
+  for (StringRef Str : {"p:32:0", "p0:32:0:32", "p42:32:0:32:32"})
+    EXPECT_THAT_EXPECTED(DataLayout::parse(Str),
+                         FailedWithMessage("ABI alignment must be non-zero"));
+
+  for (StringRef Str : {"p:32:4", "p42:32:24:32", "p0:32:65535:32:32"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(Str),
+        FailedWithMessage(
+            "ABI alignment must be a power of two times the byte width"));
+
+  // preferred alignment
+  for (StringRef Str : {"p:32:32:", "p0:32:32:", "p42:32:32::32"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(Str),
+        FailedWithMessage("preferred alignment component cannot be empty"));
+
+  for (StringRef Str : {"p:32:32:x", "p0:32:32:0x20", "p42:32:32:65536:32"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(Str),
+        FailedWithMessage("preferred alignment must be a 16-bit integer"));
+
+  for (StringRef Str : {"p:32:32:0", "p0:32:32:0", "p42:32:32:0:32"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(Str),
+        FailedWithMessage("preferred alignment must be non-zero"));
+
+  for (StringRef Str : {"p:32:32:4", "p0:32:32:24", "p42:32:32:65535:32"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(Str),
+        FailedWithMessage(
+            "preferred alignment must be a power of two times the byte width"));
+
+  for (StringRef Str : {"p:64:64:32", "p0:16:32:16:16"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(Str),
+        FailedWithMessage(
+            "preferred alignment cannot be less than the ABI alignment"));
+
+  // index size
+  for (StringRef Str : {"p:32:32:32:", "p0:32:32:32:"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(Str),
+        FailedWithMessage("index size component cannot be empty"));
+
+  for (StringRef Str :
+       {"p:32:32:32:0", "p0:32:32:32:0x20", "p42:32:32:32:16777216"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(Str),
+        FailedWithMessage("index size must be a non-zero 24-bit integer"));
+
+  for (StringRef Str : {"p:16:16:16:17", "p0:32:64:64:64", "p42:16:64:64:32"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(Str),
+        FailedWithMessage("index size cannot be larger than the pointer size"));
+}
+
 TEST(DataLayout, ParseNonIntegralAddrSpace) {
   for (StringRef Str : {"ni:1", "ni:16777215", "ni:1:16777215"})
     EXPECT_THAT_EXPECTED(DataLayout::parse(Str), Succeeded());
@@ -160,6 +228,102 @@ TEST(DataLayout, ParseNonIntegralAddrSpace) {
         FailedWithMessage("address space 0 cannot be non-integral"));
 }
 
+TEST(DataLayout, GetPointerSizeInBits) {
+  std::tuple<StringRef, unsigned, unsigned, unsigned> Cases[] = {
+      {"", 64, 64, 64},
+      {"p:16:32", 16, 16, 16},
+      {"p0:32:64", 32, 32, 32},
+      {"p1:16:32", 64, 16, 64},
+      {"p1:31:32-p2:15:16:16:14", 64, 31, 15},
+  };
+  for (auto [Layout, V0, V1, V2] : Cases) {
+    DataLayout DL = cantFail(DataLayout::parse(Layout));
+    EXPECT_EQ(DL.getPointerSizeInBits(0), V0) << Layout;
+    EXPECT_EQ(DL.getPointerSizeInBits(1), V1) << Layout;
+    EXPECT_EQ(DL.getPointerSizeInBits(2), V2) << Layout;
+  }
+}
+
+TEST(DataLayout, GetPointerSize) {
+  std::tuple<StringRef, unsigned, unsigned, unsigned> Cases[] = {
+      {"", 8, 8, 8},
+      {"p:16:32", 2, 2, 2},
+      {"p0:32:64", 4, 4, 4},
+      {"p1:17:32", 8, 3, 8},
+      {"p1:31:64-p2:23:8:16:9", 8, 4, 3},
+  };
+  for (auto [Layout, V0, V1, V2] : Cases) {
+    DataLayout DL = cantFail(DataLayout::parse(Layout));
+    EXPECT_EQ(DL.getPointerSize(0), V0) << Layout;
+    EXPECT_EQ(DL.getPointerSize(1), V1) << Layout;
+    EXPECT_EQ(DL.getPointerSize(2), V2) << Layout;
+  }
+}
+
+TEST(DataLayout, GetIndexSizeInBits) {
+  std::tuple<StringRef, unsigned, unsigned, unsigned> Cases[] = {
+      {"", 64, 64, 64},
+      {"p:16:32", 16, 16, 16},
+      {"p0:32:64", 32, 32, 32},
+      {"p1:16:32:32:10", 64, 10, 64},
+      {"p1:31:32:64:20-p2:17:16:16:15", 64, 20, 15},
+  };
+  for (auto [Layout, V0, V1, V2] : Cases) {
+    DataLayout DL = cantFail(DataLayout::parse(Layout));
+    EXPECT_EQ(DL.getIndexSizeInBits(0), V0) << Layout;
+    EXPECT_EQ(DL.getIndexSizeInBits(1), V1) << Layout;
+    EXPECT_EQ(DL.getIndexSizeInBits(2), V2) << Layout;
+  }
+}
+
+TEST(DataLayout, GetIndexSize) {
+  std::tuple<StringRef, unsigned, unsigned, unsigned> Cases[] = {
+      {"", 8, 8, 8},
+      {"p:16:32", 2, 2, 2},
+      {"p0:27:64", 4, 4, 4},
+      {"p1:19:32:64:5", 8, 1, 8},
+      {"p1:33:32:64:23-p2:21:8:16:13", 8, 3, 2},
+  };
+  for (auto [Layout, V0, V1, V2] : Cases) {
+    DataLayout DL = cantFail(DataLayout::parse(Layout));
+    EXPECT_EQ(DL.getIndexSize(0), V0) << Layout;
+    EXPECT_EQ(DL.getIndexSize(1), V1) << Layout;
+    EXPECT_EQ(DL.getIndexSize(2), V2) << Layout;
+  }
+}
+
+TEST(DataLayout, GetPointerABIAlignment) {
+  std::tuple<StringRef, unsigned, unsigned, unsigned> Cases[] = {
+      {"", 8, 8, 8},
+      {"p:16:32", 4, 4, 4},
+      {"p0:16:32:64", 4, 4, 4},
+      {"p1:32:16:64", 8, 2, 8},
+      {"p1:33:16:32:15-p2:23:8:16:9", 8, 2, 1},
+  };
+  for (auto [Layout, V0, V1, V2] : Cases) {
+    DataLayout DL = cantFail(DataLayout::parse(Layout));
+    EXPECT_EQ(DL.getPointerABIAlignment(0).value(), V0) << Layout;
+    EXPECT_EQ(DL.getPointerABIAlignment(1).value(), V1) << Layout;
+    EXPECT_EQ(DL.getPointerABIAlignment(2).value(), V2) << Layout;
+  }
+}
+
+TEST(DataLayout, GetPointerPrefAlignment) {
+  std::tuple<StringRef, unsigned, unsigned, unsigned> Cases[] = {
+      {"", 8, 8, 8},
+      {"p:16:32", 4, 4, 4},
+      {"p0:8:16:32", 4, 4, 4},
+      {"p1:32:8:16", 8, 2, 8},
+      {"p1:33:8:16:31-p2:23:8:32:17", 8, 2, 4},
+  };
+  for (auto [Layout, V0, V1, V2] : Cases) {
+    DataLayout DL = cantFail(DataLayout::parse(Layout));
+    EXPECT_EQ(DL.getPointerPrefAlignment(0).value(), V0) << Layout;
+    EXPECT_EQ(DL.getPointerPrefAlignment(1).value(), V1) << Layout;
+    EXPECT_EQ(DL.getPointerPrefAlignment(2).value(), V2) << Layout;
+  }
+}
+
 TEST(DataLayout, IsNonIntegralAddressSpace) {
   DataLayout Default;
   EXPECT_THAT(Default.getNonIntegralAddressSpaces(), ::testing::SizeIs(0));

From 4a0bbbcbcf6ebc87e794e7b86b9f4651bffcd806 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev@outlook.com>
Date: Fri, 16 Aug 2024 13:24:03 -0700
Subject: [PATCH 236/441] [SLP]Fix PR104637: do not create new nodes for fully
 overlapped non-schedulable nodes

If the scalars do not require scheduling and were already vectorized,
but in the different order, compiler still tries to create the new node.
It may cause the compiler crash for the gathered operands. Instead need
to consider such nodes as full overlap and just reshuffle vectorized
node.

Fixes https://github.com/llvm/llvm-project/issues/104637
---
 .../Transforms/Vectorize/SLPVectorizer.cpp    | 15 +++++++
 .../X86/full-non-schedulable-overlap.ll       | 42 +++++++++++++++++++
 2 files changed, 57 insertions(+)
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/full-non-schedulable-overlap.ll

diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 667d0df781aa4cc..9ecd8160a978916 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -7144,6 +7144,21 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
                        ReuseShuffleIndices);
         return;
       }
+      SmallPtrSet<const TreeEntry *, 4> Nodes;
+      Nodes.insert(getTreeEntry(S.OpValue));
+      for (const TreeEntry *E : MultiNodeScalars.lookup(S.OpValue))
+        Nodes.insert(E);
+      SmallPtrSet<Value *, 8> Values(VL.begin(), VL.end());
+      if (any_of(Nodes, [&](const TreeEntry *E) {
+            return all_of(E->Scalars,
+                          [&](Value *V) { return Values.contains(V); });
+          })) {
+        LLVM_DEBUG(dbgs() << "SLP: Gathering due to full overlap.\n");
+        if (TryToFindDuplicates(S))
+          newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx,
+                       ReuseShuffleIndices);
+        return;
+      }
     } else {
       // Record the reuse of the tree node.  FIXME, currently this is only used
       // to properly draw the graph rather than for the actual vectorization.
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/full-non-schedulable-overlap.ll b/llvm/test/Transforms/SLPVectorizer/X86/full-non-schedulable-overlap.ll
new file mode 100644
index 000000000000000..231a9512c9ee99d
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/full-non-schedulable-overlap.ll
@@ -0,0 +1,42 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+
+define void @test(double %v) {
+; CHECK-LABEL: define void @test(
+; CHECK-SAME: double [[V:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x double> <double 0.000000e+00, double poison>, double [[V]], i32 1
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul <2 x double> zeroinitializer, [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> poison, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[T50_02:%.*]] = phi double [ 0.000000e+00, %[[ENTRY]] ], [ [[TMP8:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = phi <2 x double> [ zeroinitializer, %[[ENTRY]] ], [ [[TMP10:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[TMP4:%.*]] = fmul <2 x double> [[TMP3]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x double> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x double> [[TMP4]], i32 1
+; CHECK-NEXT:    [[TMP7:%.*]] = fadd double [[TMP6]], [[TMP5]]
+; CHECK-NEXT:    [[TMP8]] = fadd double [[TMP7]], [[V]]
+; CHECK-NEXT:    [[TMP9:%.*]] = fmul <2 x double> zeroinitializer, [[TMP1]]
+; CHECK-NEXT:    [[TMP10]] = fadd <2 x double> zeroinitializer, [[TMP9]]
+; CHECK-NEXT:    br label %[[LOOP]]
+;
+entry:
+  %mul.3 = fmul double 0.000000e+00, %v
+  %mul.4 = fmul double 0.000000e+00, 0.000000e+00
+  br label %loop
+
+loop:
+  %t48.0 = phi double [ 0.000000e+00, %entry ], [ %5, %loop ]
+  %t50.02 = phi double [ 0.000000e+00, %entry ], [ %3, %loop ]
+  %t52.0 = phi double [ 0.000000e+00, %entry ], [ %7, %loop ]
+  %0 = fmul double %t52.0, %mul.3
+  %1 = fmul double %t48.0, %mul.4
+  %2 = fadd double %1, %0
+  %3 = fadd double %2, %v
+  %4 = fmul double 0.000000e+00, %mul.3
+  %5 = fadd double 0.000000e+00, %4
+  %6 = fmul double 0.000000e+00, %mul.4
+  %7 = fadd double 0.000000e+00, %6
+  br label %loop
+}

From f33d519cd471b1eec41c8b26f892ddb66bf8144f Mon Sep 17 00:00:00 2001
From: vporpo <vporpodas@google.com>
Date: Fri, 16 Aug 2024 13:59:58 -0700
Subject: [PATCH 237/441] [SandboxIR] Implement ConstantInt (#104639)

This patch implements a very basic version of sandboxir::ConstantInt. It
is missing most of the factory functions present in llvm::ConstantInt,
but these will be added later.
---
 llvm/include/llvm/SandboxIR/SandboxIR.h       | 50 ++++++++++++++++---
 .../llvm/SandboxIR/SandboxIRValues.def        |  1 +
 llvm/lib/SandboxIR/SandboxIR.cpp              | 18 ++++---
 llvm/unittests/SandboxIR/SandboxIRTest.cpp    | 43 ++++++++++++----
 4 files changed, 88 insertions(+), 24 deletions(-)

diff --git a/llvm/include/llvm/SandboxIR/SandboxIR.h b/llvm/include/llvm/SandboxIR/SandboxIR.h
index 75c9d5017054b4a..3c568f4956857f3 100644
--- a/llvm/include/llvm/SandboxIR/SandboxIR.h
+++ b/llvm/include/llvm/SandboxIR/SandboxIR.h
@@ -107,6 +107,7 @@ namespace llvm {
 namespace sandboxir {
 
 class BasicBlock;
+class ConstantInt;
 class Context;
 class Function;
 class Instruction;
@@ -489,22 +490,22 @@ class Constant : public sandboxir::User {
       : sandboxir::User(ClassID::Constant, C, SBCtx) {}
   Constant(ClassID ID, llvm::Constant *C, sandboxir::Context &SBCtx)
       : sandboxir::User(ID, C, SBCtx) {}
-  friend class Function; // For constructor
-  friend class Context;  // For constructor.
-  Use getOperandUseInternal(unsigned OpIdx, bool Verify) const final {
+  friend class ConstantInt; // For constructor.
+  friend class Function;    // For constructor
+  friend class Context;     // For constructor.
+  Use getOperandUseInternal(unsigned OpIdx, bool Verify) const override {
     return getOperandUseDefault(OpIdx, Verify);
   }
 
 public:
-  static Constant *createInt(Type *Ty, uint64_t V, Context &Ctx,
-                             bool IsSigned = false);
   /// For isa/dyn_cast.
   static bool classof(const sandboxir::Value *From) {
     return From->getSubclassID() == ClassID::Constant ||
+           From->getSubclassID() == ClassID::ConstantInt ||
            From->getSubclassID() == ClassID::Function;
   }
   sandboxir::Context &getParent() const { return getContext(); }
-  unsigned getUseOperandNo(const Use &Use) const final {
+  unsigned getUseOperandNo(const Use &Use) const override {
     return getUseOperandNoDefault(Use);
   }
 #ifndef NDEBUG
@@ -515,6 +516,41 @@ class Constant : public sandboxir::User {
 #endif
 };
 
+class ConstantInt : public Constant {
+  ConstantInt(llvm::ConstantInt *C, sandboxir::Context &Ctx)
+      : Constant(ClassID::ConstantInt, C, Ctx) {}
+  friend class Context; // For constructor.
+
+  Use getOperandUseInternal(unsigned OpIdx, bool Verify) const final {
+    llvm_unreachable("ConstantInt has no operands!");
+  }
+
+public:
+  /// If Ty is a vector type, return a Constant with a splat of the given
+  /// value. Otherwise return a ConstantInt for the given value.
+  static ConstantInt *get(Type *Ty, uint64_t V, Context &Ctx,
+                          bool IsSigned = false);
+
+  // TODO: Implement missing functions.
+
+  /// For isa/dyn_cast.
+  static bool classof(const sandboxir::Value *From) {
+    return From->getSubclassID() == ClassID::ConstantInt;
+  }
+  unsigned getUseOperandNo(const Use &Use) const override {
+    llvm_unreachable("ConstantInt has no operands!");
+  }
+#ifndef NDEBUG
+  void verify() const override {
+    assert(isa<llvm::ConstantInt>(Val) && "Expected a ConstantInst!");
+  }
+  void dumpOS(raw_ostream &OS) const override {
+    dumpCommonPrefix(OS);
+    dumpCommonSuffix(OS);
+  }
+#endif
+};
+
 /// Iterator for `Instruction`s in a `BasicBlock.
 /// \Returns an sandboxir::Instruction & when derereferenced.
 class BBIterator {
@@ -2045,7 +2081,7 @@ class Context {
   Constant *getOrCreateConstant(llvm::Constant *LLVMC) {
     return cast<Constant>(getOrCreateValueInternal(LLVMC, 0));
   }
-  friend class Constant; // For getOrCreateConstant().
+  friend class ConstantInt; // For getOrCreateConstant().
   /// Create a sandboxir::BasicBlock for an existing LLVM IR \p BB. This will
   /// also create all contents of the block.
   BasicBlock *createBasicBlock(llvm::BasicBlock *BB);
diff --git a/llvm/include/llvm/SandboxIR/SandboxIRValues.def b/llvm/include/llvm/SandboxIR/SandboxIRValues.def
index 81a916be21e4a3b..cba5b69ebf1217c 100644
--- a/llvm/include/llvm/SandboxIR/SandboxIRValues.def
+++ b/llvm/include/llvm/SandboxIR/SandboxIRValues.def
@@ -19,6 +19,7 @@ DEF_VALUE(Argument, Argument)
 DEF_USER(User, User)
 DEF_VALUE(Block, BasicBlock)
 DEF_USER(Constant, Constant)
+DEF_USER(ConstantInt, ConstantInt)
 
 #ifndef DEF_INSTR
 #define DEF_INSTR(ID, OPCODE, CLASS)
diff --git a/llvm/lib/SandboxIR/SandboxIR.cpp b/llvm/lib/SandboxIR/SandboxIR.cpp
index 559d9ae7c00e7a6..6e8c83528bded92 100644
--- a/llvm/lib/SandboxIR/SandboxIR.cpp
+++ b/llvm/lib/SandboxIR/SandboxIR.cpp
@@ -1695,18 +1695,20 @@ Value *ExtractElementInst::create(Value *Vec, Value *Idx,
   return Ctx.getOrCreateConstant(cast<llvm::Constant>(NewV));
 }
 
-Constant *Constant::createInt(Type *Ty, uint64_t V, Context &Ctx,
-                              bool IsSigned) {
-  llvm::Constant *LLVMC = llvm::ConstantInt::get(Ty, V, IsSigned);
-  return Ctx.getOrCreateConstant(LLVMC);
-}
-
 #ifndef NDEBUG
 void Constant::dumpOS(raw_ostream &OS) const {
   dumpCommonPrefix(OS);
   dumpCommonSuffix(OS);
 }
+#endif // NDEBUG
 
+ConstantInt *ConstantInt::get(Type *Ty, uint64_t V, Context &Ctx,
+                              bool IsSigned) {
+  auto *LLVMC = llvm::ConstantInt::get(Ty, V, IsSigned);
+  return cast<ConstantInt>(Ctx.getOrCreateConstant(LLVMC));
+}
+
+#ifndef NDEBUG
 void Function::dumpNameAndArgs(raw_ostream &OS) const {
   auto *F = cast<llvm::Function>(Val);
   OS << *F->getReturnType() << " @" << F->getName() << "(";
@@ -1788,6 +1790,10 @@ Value *Context::getOrCreateValueInternal(llvm::Value *LLVMV, llvm::User *U) {
     return It->second.get();
 
   if (auto *C = dyn_cast<llvm::Constant>(LLVMV)) {
+    if (auto *CI = dyn_cast<llvm::ConstantInt>(C)) {
+      It->second = std::unique_ptr<ConstantInt>(new ConstantInt(CI, *this));
+      return It->second.get();
+    }
     if (auto *F = dyn_cast<llvm::Function>(LLVMV))
       It->second = std::unique_ptr<Function>(new Function(F, *this));
     else
diff --git a/llvm/unittests/SandboxIR/SandboxIRTest.cpp b/llvm/unittests/SandboxIR/SandboxIRTest.cpp
index e4563ab8f07ba6a..6c0f1b19243c289 100644
--- a/llvm/unittests/SandboxIR/SandboxIRTest.cpp
+++ b/llvm/unittests/SandboxIR/SandboxIRTest.cpp
@@ -103,6 +103,30 @@ define void @foo(i32 %v1) {
 #endif
 }
 
+TEST_F(SandboxIRTest, ConstantInt) {
+  parseIR(C, R"IR(
+define void @foo(i32 %v0) {
+  %add0 = add i32 %v0, 42
+  ret void
+}
+)IR");
+  Function &LLVMF = *M->getFunction("foo");
+  sandboxir::Context Ctx(C);
+
+  auto &F = *Ctx.createFunction(&LLVMF);
+  auto &BB = *F.begin();
+  auto It = BB.begin();
+  auto *Add0 = cast<sandboxir::BinaryOperator>(&*It++);
+  auto *FortyTwo = cast<sandboxir::ConstantInt>(Add0->getOperand(1));
+
+  // Check that creating an identical constant gives us the same object.
+  auto *NewCI = sandboxir::ConstantInt::get(Type::getInt32Ty(C), 42, Ctx);
+  EXPECT_EQ(NewCI, FortyTwo);
+  // Check new constant.
+  auto *FortyThree = sandboxir::ConstantInt::get(Type::getInt32Ty(C), 43, Ctx);
+  EXPECT_NE(FortyThree, FortyTwo);
+}
+
 TEST_F(SandboxIRTest, Use) {
   parseIR(C, R"IR(
 define i32 @foo(i32 %v0, i32 %v1) {
@@ -610,12 +634,11 @@ define void @foo(i1 %c0, i8 %v0, i8 %v1, i1 %c1) {
   }
   {
     // Check SelectInst::create() Folded.
-    auto *False =
-        sandboxir::Constant::createInt(llvm::Type::getInt1Ty(C), 0, Ctx,
-                                       /*IsSigned=*/false);
+    auto *False = sandboxir::ConstantInt::get(llvm::Type::getInt1Ty(C), 0, Ctx,
+                                              /*IsSigned=*/false);
     auto *FortyTwo =
-        sandboxir::Constant::createInt(llvm::Type::getInt1Ty(C), 42, Ctx,
-                                       /*IsSigned=*/false);
+        sandboxir::ConstantInt::get(llvm::Type::getInt1Ty(C), 42, Ctx,
+                                    /*IsSigned=*/false);
     auto *NewSel =
         sandboxir::SelectInst::create(False, FortyTwo, FortyTwo, Ret, Ctx);
     EXPECT_TRUE(isa<sandboxir::Constant>(NewSel));
@@ -706,7 +729,7 @@ define void @foo(i8 %v0, i8 %v1, <2 x i8> %vec) {
 
   auto *LLVMArg0 = LLVMF.getArg(0);
   auto *LLVMArgVec = LLVMF.getArg(2);
-  auto *Zero = sandboxir::Constant::createInt(Type::getInt8Ty(C), 0, Ctx);
+  auto *Zero = sandboxir::ConstantInt::get(Type::getInt8Ty(C), 0, Ctx);
   auto *LLVMZero = llvm::ConstantInt::get(Type::getInt8Ty(C), 0);
   EXPECT_EQ(
       sandboxir::InsertElementInst::isValidOperands(ArgVec, Arg0, Zero),
@@ -1866,8 +1889,7 @@ define void @foo(i8 %arg0, i8 %arg1, float %farg0, float %farg1) {
   }
   {
     // Check create() when it gets folded.
-    auto *FortyTwo =
-        sandboxir::Constant::createInt(Type::getInt32Ty(C), 42, Ctx);
+    auto *FortyTwo = sandboxir::ConstantInt::get(Type::getInt32Ty(C), 42, Ctx);
     auto *NewV = sandboxir::BinaryOperator::create(
         sandboxir::Instruction::Opcode::Add, FortyTwo, FortyTwo,
         /*InsertBefore=*/Ret, Ctx, "Folded");
@@ -1923,8 +1945,7 @@ define void @foo(i8 %arg0, i8 %arg1, float %farg0, float %farg1) {
   }
   {
     // Check createWithCopiedFlags() when it gets folded.
-    auto *FortyTwo =
-        sandboxir::Constant::createInt(Type::getInt32Ty(C), 42, Ctx);
+    auto *FortyTwo = sandboxir::ConstantInt::get(Type::getInt32Ty(C), 42, Ctx);
     auto *NewV = sandboxir::BinaryOperator::createWithCopiedFlags(
         sandboxir::Instruction::Opcode::Add, FortyTwo, FortyTwo, CopyFrom,
         /*InsertBefore=*/Ret, Ctx, "Folded");
@@ -2378,7 +2399,7 @@ define void @foo() {
   auto *Ty = Type::getInt32Ty(C);
   unsigned AddrSpace = 42;
   auto *PtrTy = PointerType::get(C, AddrSpace);
-  auto *ArraySize = sandboxir::Constant::createInt(Ty, 43, Ctx);
+  auto *ArraySize = sandboxir::ConstantInt::get(Ty, 43, Ctx);
   {
     // Check create() WhereIt, WhereBB.
     auto *NewI = cast<sandboxir::AllocaInst>(sandboxir::AllocaInst::create(

From 70995a1a3379ed3c21b1c5da6723f04166cb0ae6 Mon Sep 17 00:00:00 2001
From: Krzysztof Drewniak <Krzysztof.Drewniak@amd.com>
Date: Fri, 16 Aug 2024 16:24:25 -0500
Subject: [PATCH 238/441] [ScalarizeMaskedMemIntr] Optimize splat non-constant
 masks (#104537)

In cases (like the ones added in the tests) where the condition of a
masked load or store is a splat but not a constant (that is, a masked
operation is being used to implement patterns like "load if the current
lane is in-bounds, otherwise return 0"), optimize the 'scalarized' code
to perform an aligned vector load/store if the splat constant is true.

Additionally, take a few steps to preserve aliasing information and
names when nothing is scalarized while I'm here.

As motivation, some LLVM IR users will genatate masked load/store in
cases that map to this kind of predicated operation (where either the
vector is loaded/stored or it isn't) in order to take advantage of
hardware primitives, but on AMDGPU, where we don't have a masked load or
store, this pass would scalarize a load or store that was intended to be
- and can be - vectorized while also introducing expensive branches.

Fixes #104520

Pre-commit tests at #104527
---
 .../Scalar/ScalarizeMaskedMemIntrin.cpp       |  64 +-
 llvm/test/CodeGen/X86/bfloat.ll               | 586 +-----------------
 llvm/test/CodeGen/X86/shuffle-half.ll         | 298 +--------
 .../X86/expand-masked-load.ll                 |  34 +-
 .../X86/expand-masked-store.ll                |  25 +-
 5 files changed, 90 insertions(+), 917 deletions(-)

diff --git a/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp b/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp
index 8eadf8900020d9e..9cb7bad94c20bcd 100644
--- a/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp
+++ b/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp
@@ -17,6 +17,7 @@
 #include "llvm/ADT/Twine.h"
 #include "llvm/Analysis/DomTreeUpdater.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
+#include "llvm/Analysis/VectorUtils.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/Constant.h"
 #include "llvm/IR/Constants.h"
@@ -161,7 +162,9 @@ static void scalarizeMaskedLoad(const DataLayout &DL, CallInst *CI,
 
   // Short-cut if the mask is all-true.
   if (isa<Constant>(Mask) && cast<Constant>(Mask)->isAllOnesValue()) {
-    Value *NewI = Builder.CreateAlignedLoad(VecType, Ptr, AlignVal);
+    LoadInst *NewI = Builder.CreateAlignedLoad(VecType, Ptr, AlignVal);
+    NewI->copyMetadata(*CI);
+    NewI->takeName(CI);
     CI->replaceAllUsesWith(NewI);
     CI->eraseFromParent();
     return;
@@ -188,8 +191,39 @@ static void scalarizeMaskedLoad(const DataLayout &DL, CallInst *CI,
     return;
   }
 
+  // Optimize the case where the "masked load" is a predicated load - that is,
+  // where the mask is the splat of a non-constant scalar boolean. In that case,
+  // use that splated value as the guard on a conditional vector load.
+  if (isSplatValue(Mask, /*Index=*/0)) {
+    Value *Predicate = Builder.CreateExtractElement(Mask, uint64_t(0ull),
+                                                    Mask->getName() + ".first");
+    Instruction *ThenTerm =
+        SplitBlockAndInsertIfThen(Predicate, InsertPt, /*Unreachable=*/false,
+                                  /*BranchWeights=*/nullptr, DTU);
+
+    BasicBlock *CondBlock = ThenTerm->getParent();
+    CondBlock->setName("cond.load");
+    Builder.SetInsertPoint(CondBlock->getTerminator());
+    LoadInst *Load = Builder.CreateAlignedLoad(VecType, Ptr, AlignVal,
+                                               CI->getName() + ".cond.load");
+    Load->copyMetadata(*CI);
+
+    BasicBlock *PostLoad = ThenTerm->getSuccessor(0);
+    Builder.SetInsertPoint(PostLoad, PostLoad->begin());
+    PHINode *Phi = Builder.CreatePHI(VecType, /*NumReservedValues=*/2);
+    Phi->addIncoming(Load, CondBlock);
+    Phi->addIncoming(Src0, IfBlock);
+    Phi->takeName(CI);
+
+    CI->replaceAllUsesWith(Phi);
+    CI->eraseFromParent();
+    ModifiedDT = true;
+    return;
+  }
   // If the mask is not v1i1, use scalar bit test operations. This generates
   // better results on X86 at least.
+  // Note: this produces worse code on AMDGPU, where the "i1" is implicitly SIMD
+  // - what's a good way to detect this?
   Value *SclrMask;
   if (VectorWidth != 1) {
     Type *SclrMaskTy = Builder.getIntNTy(VectorWidth);
@@ -297,7 +331,9 @@ static void scalarizeMaskedStore(const DataLayout &DL, CallInst *CI,
 
   // Short-cut if the mask is all-true.
   if (isa<Constant>(Mask) && cast<Constant>(Mask)->isAllOnesValue()) {
-    Builder.CreateAlignedStore(Src, Ptr, AlignVal);
+    StoreInst *Store = Builder.CreateAlignedStore(Src, Ptr, AlignVal);
+    Store->takeName(CI);
+    Store->copyMetadata(*CI);
     CI->eraseFromParent();
     return;
   }
@@ -319,8 +355,31 @@ static void scalarizeMaskedStore(const DataLayout &DL, CallInst *CI,
     return;
   }
 
+  // Optimize the case where the "masked store" is a predicated store - that is,
+  // when the mask is the splat of a non-constant scalar boolean. In that case,
+  // optimize to a conditional store.
+  if (isSplatValue(Mask, /*Index=*/0)) {
+    Value *Predicate = Builder.CreateExtractElement(Mask, uint64_t(0ull),
+                                                    Mask->getName() + ".first");
+    Instruction *ThenTerm =
+        SplitBlockAndInsertIfThen(Predicate, InsertPt, /*Unreachable=*/false,
+                                  /*BranchWeights=*/nullptr, DTU);
+    BasicBlock *CondBlock = ThenTerm->getParent();
+    CondBlock->setName("cond.store");
+    Builder.SetInsertPoint(CondBlock->getTerminator());
+
+    StoreInst *Store = Builder.CreateAlignedStore(Src, Ptr, AlignVal);
+    Store->takeName(CI);
+    Store->copyMetadata(*CI);
+
+    CI->eraseFromParent();
+    ModifiedDT = true;
+    return;
+  }
+
   // If the mask is not v1i1, use scalar bit test operations. This generates
   // better results on X86 at least.
+
   Value *SclrMask;
   if (VectorWidth != 1) {
     Type *SclrMaskTy = Builder.getIntNTy(VectorWidth);
@@ -997,7 +1056,6 @@ static bool optimizeCallInst(CallInst *CI, bool &ModifiedDT,
         any_of(II->args(),
                [](Value *V) { return isa<ScalableVectorType>(V->getType()); }))
       return false;
-
     switch (II->getIntrinsicID()) {
     default:
       break;
diff --git a/llvm/test/CodeGen/X86/bfloat.ll b/llvm/test/CodeGen/X86/bfloat.ll
index ec76e8b05678b0b..3759909a2ccc8e3 100644
--- a/llvm/test/CodeGen/X86/bfloat.ll
+++ b/llvm/test/CodeGen/X86/bfloat.ll
@@ -759,347 +759,21 @@ define <32 x bfloat> @pr63017_2() nounwind {
 ;
 ; SSE2-LABEL: pr63017_2:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pushq %r14
-; SSE2-NEXT:    pushq %rbx
-; SSE2-NEXT:    subq $200, %rsp
 ; SSE2-NEXT:    xorl %eax, %eax
 ; SSE2-NEXT:    testb %al, %al
 ; SSE2-NEXT:    jne .LBB12_1
 ; SSE2-NEXT:  # %bb.2: # %cond.load
 ; SSE2-NEXT:    movzwl (%rax), %eax
 ; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; SSE2-NEXT:    movd {{.*#+}} xmm0 = [-1.0E+0,0.0E+0,0.0E+0,0.0E+0]
-; SSE2-NEXT:    movd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movdqa %xmm0, %xmm15
-; SSE2-NEXT:    movd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movdqa %xmm0, %xmm13
-; SSE2-NEXT:    movdqa %xmm0, %xmm14
-; SSE2-NEXT:    movdqa %xmm0, %xmm11
-; SSE2-NEXT:    movdqa %xmm0, %xmm12
-; SSE2-NEXT:    movdqa %xmm0, %xmm9
-; SSE2-NEXT:    movdqa %xmm0, %xmm10
-; SSE2-NEXT:    movdqa %xmm0, %xmm7
-; SSE2-NEXT:    movdqa %xmm0, %xmm8
-; SSE2-NEXT:    movdqa %xmm0, %xmm5
-; SSE2-NEXT:    movdqa %xmm0, %xmm6
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    movdqa %xmm0, %xmm4
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    movd %eax, %xmm0
 ; SSE2-NEXT:    jmp .LBB12_3
 ; SSE2-NEXT:  .LBB12_1:
-; SSE2-NEXT:    movd {{.*#+}} xmm2 = [-1.0E+0,0.0E+0,0.0E+0,0.0E+0]
-; SSE2-NEXT:    movdqa %xmm2, %xmm0
-; SSE2-NEXT:    movd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movdqa %xmm2, %xmm15
-; SSE2-NEXT:    movd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movdqa %xmm2, %xmm13
-; SSE2-NEXT:    movdqa %xmm2, %xmm14
-; SSE2-NEXT:    movdqa %xmm2, %xmm11
-; SSE2-NEXT:    movdqa %xmm2, %xmm12
-; SSE2-NEXT:    movdqa %xmm2, %xmm9
-; SSE2-NEXT:    movdqa %xmm2, %xmm10
-; SSE2-NEXT:    movdqa %xmm2, %xmm7
-; SSE2-NEXT:    movdqa %xmm2, %xmm8
-; SSE2-NEXT:    movdqa %xmm2, %xmm5
-; SSE2-NEXT:    movdqa %xmm2, %xmm6
-; SSE2-NEXT:    movdqa %xmm2, %xmm3
-; SSE2-NEXT:    movdqa %xmm2, %xmm4
-; SSE2-NEXT:    movdqa %xmm2, %xmm1
-; SSE2-NEXT:    movd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:  .LBB12_3: # %else
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    jne .LBB12_5
-; SSE2-NEXT:  # %bb.4: # %cond.load1
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movd %eax, %xmm0
-; SSE2-NEXT:  .LBB12_5: # %else2
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    jne .LBB12_7
-; SSE2-NEXT:  # %bb.6: # %cond.load4
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; SSE2-NEXT:  .LBB12_7: # %else5
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    jne .LBB12_9
-; SSE2-NEXT:  # %bb.8: # %cond.load7
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; SSE2-NEXT:  .LBB12_9: # %else8
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    jne .LBB12_11
-; SSE2-NEXT:  # %bb.10: # %cond.load10
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; SSE2-NEXT:  .LBB12_11: # %else11
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    jne .LBB12_13
-; SSE2-NEXT:  # %bb.12: # %cond.load13
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; SSE2-NEXT:  .LBB12_13: # %else14
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    jne .LBB12_15
-; SSE2-NEXT:  # %bb.14: # %cond.load16
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; SSE2-NEXT:  .LBB12_15: # %else17
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    jne .LBB12_17
-; SSE2-NEXT:  # %bb.16: # %cond.load19
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; SSE2-NEXT:  .LBB12_17: # %else20
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    jne .LBB12_19
-; SSE2-NEXT:  # %bb.18: # %cond.load22
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; SSE2-NEXT:  .LBB12_19: # %else23
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    jne .LBB12_21
-; SSE2-NEXT:  # %bb.20: # %cond.load25
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; SSE2-NEXT:  .LBB12_21: # %else26
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    jne .LBB12_23
-; SSE2-NEXT:  # %bb.22: # %cond.load28
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; SSE2-NEXT:  .LBB12_23: # %else29
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    jne .LBB12_25
-; SSE2-NEXT:  # %bb.24: # %cond.load31
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; SSE2-NEXT:  .LBB12_25: # %else32
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    jne .LBB12_27
-; SSE2-NEXT:  # %bb.26: # %cond.load34
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; SSE2-NEXT:  .LBB12_27: # %else35
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    jne .LBB12_29
-; SSE2-NEXT:  # %bb.28: # %cond.load37
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; SSE2-NEXT:  .LBB12_29: # %else38
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    jne .LBB12_31
-; SSE2-NEXT:  # %bb.30: # %cond.load40
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; SSE2-NEXT:  .LBB12_31: # %else41
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    jne .LBB12_33
-; SSE2-NEXT:  # %bb.32: # %cond.load43
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; SSE2-NEXT:  .LBB12_33: # %else44
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    jne .LBB12_35
-; SSE2-NEXT:  # %bb.34: # %cond.load46
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movd %eax, %xmm15
-; SSE2-NEXT:  .LBB12_35: # %else47
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    jne .LBB12_37
-; SSE2-NEXT:  # %bb.36: # %cond.load49
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; SSE2-NEXT:  .LBB12_37: # %else50
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    jne .LBB12_39
-; SSE2-NEXT:  # %bb.38: # %cond.load52
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movd %eax, %xmm13
-; SSE2-NEXT:  .LBB12_39: # %else53
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    jne .LBB12_41
-; SSE2-NEXT:  # %bb.40: # %cond.load55
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movd %eax, %xmm14
-; SSE2-NEXT:  .LBB12_41: # %else56
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    jne .LBB12_43
-; SSE2-NEXT:  # %bb.42: # %cond.load58
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movd %eax, %xmm11
-; SSE2-NEXT:  .LBB12_43: # %else59
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    jne .LBB12_45
-; SSE2-NEXT:  # %bb.44: # %cond.load61
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movd %eax, %xmm12
-; SSE2-NEXT:  .LBB12_45: # %else62
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    jne .LBB12_47
-; SSE2-NEXT:  # %bb.46: # %cond.load64
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movd %eax, %xmm9
-; SSE2-NEXT:  .LBB12_47: # %else65
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    jne .LBB12_49
-; SSE2-NEXT:  # %bb.48: # %cond.load67
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movd %eax, %xmm10
-; SSE2-NEXT:  .LBB12_49: # %else68
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    jne .LBB12_51
-; SSE2-NEXT:  # %bb.50: # %cond.load70
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movd %eax, %xmm7
-; SSE2-NEXT:  .LBB12_51: # %else71
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    jne .LBB12_53
-; SSE2-NEXT:  # %bb.52: # %cond.load73
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movd %eax, %xmm8
-; SSE2-NEXT:  .LBB12_53: # %else74
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    jne .LBB12_55
-; SSE2-NEXT:  # %bb.54: # %cond.load76
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movd %eax, %xmm5
-; SSE2-NEXT:  .LBB12_55: # %else77
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    jne .LBB12_57
-; SSE2-NEXT:  # %bb.56: # %cond.load79
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movd %eax, %xmm6
-; SSE2-NEXT:  .LBB12_57: # %else80
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    jne .LBB12_59
-; SSE2-NEXT:  # %bb.58: # %cond.load82
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movd %eax, %xmm3
-; SSE2-NEXT:  .LBB12_59: # %else83
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    jne .LBB12_61
-; SSE2-NEXT:  # %bb.60: # %cond.load85
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movd %eax, %xmm4
-; SSE2-NEXT:  .LBB12_61: # %else86
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    jne .LBB12_63
-; SSE2-NEXT:  # %bb.62: # %cond.load88
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movd %eax, %xmm1
-; SSE2-NEXT:  .LBB12_63: # %else89
-; SSE2-NEXT:    xorl %eax, %eax
-; SSE2-NEXT:    testb %al, %al
-; SSE2-NEXT:    movd %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    movd %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:    jne .LBB12_64
-; SSE2-NEXT:  # %bb.65: # %cond.load91
-; SSE2-NEXT:    movzwl (%rax), %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
-; SSE2-NEXT:    jmp .LBB12_66
-; SSE2-NEXT:  .LBB12_64:
-; SSE2-NEXT:    movd %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
-; SSE2-NEXT:  .LBB12_66: # %else92
+; SSE2-NEXT:    movd {{.*#+}} xmm0 = [-1.0E+0,0.0E+0,0.0E+0,0.0E+0]
+; SSE2-NEXT:  .LBB12_3:
+; SSE2-NEXT:    pushq %r14
+; SSE2-NEXT:    pushq %rbx
+; SSE2-NEXT:    subq $88, %rsp
+; SSE2-NEXT:    movd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
 ; SSE2-NEXT:    callq __truncsfbf2@PLT
 ; SSE2-NEXT:    pextrw $0, %xmm0, %ebx
 ; SSE2-NEXT:    shll $16, %ebx
@@ -1316,7 +990,7 @@ define <32 x bfloat> @pr63017_2() nounwind {
 ; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
 ; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
 ; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
-; SSE2-NEXT:    addq $200, %rsp
+; SSE2-NEXT:    addq $88, %rsp
 ; SSE2-NEXT:    popq %rbx
 ; SSE2-NEXT:    popq %r14
 ; SSE2-NEXT:    retq
@@ -1329,250 +1003,14 @@ define <32 x bfloat> @pr63017_2() nounwind {
 ;
 ; AVXNC-LABEL: pr63017_2:
 ; AVXNC:       # %bb.0:
-; AVXNC-NEXT:    vpbroadcastw {{.*#+}} ymm0 = [49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024]
+; AVXNC-NEXT:    vbroadcastss {{.*#+}} ymm0 = [49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024]
 ; AVXNC-NEXT:    xorl %eax, %eax
 ; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    vmovdqa %ymm0, %ymm1
 ; AVXNC-NEXT:    jne .LBB12_2
 ; AVXNC-NEXT:  # %bb.1: # %cond.load
-; AVXNC-NEXT:    vpbroadcastw {{.*#+}} ymm1 = [49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024]
-; AVXNC-NEXT:    vpbroadcastw {{.*#+}} ymm0 = [49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024,49024]
-; AVXNC-NEXT:    vpinsrw $0, (%rax), %xmm0, %xmm2
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_2: # %else
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_4
-; AVXNC-NEXT:  # %bb.3: # %cond.load1
-; AVXNC-NEXT:    vpinsrw $1, (%rax), %xmm0, %xmm2
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_4: # %else2
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_6
-; AVXNC-NEXT:  # %bb.5: # %cond.load4
-; AVXNC-NEXT:    vpinsrw $2, (%rax), %xmm0, %xmm2
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_6: # %else5
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_8
-; AVXNC-NEXT:  # %bb.7: # %cond.load7
-; AVXNC-NEXT:    vpinsrw $3, (%rax), %xmm0, %xmm2
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_8: # %else8
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_10
-; AVXNC-NEXT:  # %bb.9: # %cond.load10
-; AVXNC-NEXT:    vpinsrw $4, (%rax), %xmm0, %xmm2
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_10: # %else11
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_12
-; AVXNC-NEXT:  # %bb.11: # %cond.load13
-; AVXNC-NEXT:    vpinsrw $5, (%rax), %xmm0, %xmm2
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_12: # %else14
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_14
-; AVXNC-NEXT:  # %bb.13: # %cond.load16
-; AVXNC-NEXT:    vpinsrw $6, (%rax), %xmm0, %xmm2
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_14: # %else17
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_16
-; AVXNC-NEXT:  # %bb.15: # %cond.load19
-; AVXNC-NEXT:    vpinsrw $7, (%rax), %xmm0, %xmm2
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_16: # %else20
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_18
-; AVXNC-NEXT:  # %bb.17: # %cond.load22
-; AVXNC-NEXT:    vpbroadcastw (%rax), %ymm2
-; AVXNC-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0],ymm0[1,2,3,4,5,6,7],ymm2[8],ymm0[9,10,11,12,13,14,15]
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_18: # %else23
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_20
-; AVXNC-NEXT:  # %bb.19: # %cond.load25
-; AVXNC-NEXT:    vpbroadcastw (%rax), %ymm2
-; AVXNC-NEXT:    vpblendw {{.*#+}} ymm2 = ymm0[0],ymm2[1],ymm0[2,3,4,5,6,7,8],ymm2[9],ymm0[10,11,12,13,14,15]
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_20: # %else26
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_22
-; AVXNC-NEXT:  # %bb.21: # %cond.load28
-; AVXNC-NEXT:    vpbroadcastw (%rax), %ymm2
-; AVXNC-NEXT:    vpblendw {{.*#+}} ymm2 = ymm0[0,1],ymm2[2],ymm0[3,4,5,6,7,8,9],ymm2[10],ymm0[11,12,13,14,15]
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_22: # %else29
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_24
-; AVXNC-NEXT:  # %bb.23: # %cond.load31
-; AVXNC-NEXT:    vpbroadcastw (%rax), %ymm2
-; AVXNC-NEXT:    vpblendw {{.*#+}} ymm2 = ymm0[0,1,2],ymm2[3],ymm0[4,5,6,7,8,9,10],ymm2[11],ymm0[12,13,14,15]
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_24: # %else32
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_26
-; AVXNC-NEXT:  # %bb.25: # %cond.load34
-; AVXNC-NEXT:    vpbroadcastw (%rax), %ymm2
-; AVXNC-NEXT:    vpblendw {{.*#+}} ymm2 = ymm0[0,1,2,3],ymm2[4],ymm0[5,6,7,8,9,10,11],ymm2[12],ymm0[13,14,15]
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_26: # %else35
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_28
-; AVXNC-NEXT:  # %bb.27: # %cond.load37
-; AVXNC-NEXT:    vpbroadcastw (%rax), %ymm2
-; AVXNC-NEXT:    vpblendw {{.*#+}} ymm2 = ymm0[0,1,2,3,4],ymm2[5],ymm0[6,7,8,9,10,11,12],ymm2[13],ymm0[14,15]
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_28: # %else38
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_30
-; AVXNC-NEXT:  # %bb.29: # %cond.load40
-; AVXNC-NEXT:    vpbroadcastw (%rax), %ymm2
-; AVXNC-NEXT:    vpblendw {{.*#+}} ymm2 = ymm0[0,1,2,3,4,5],ymm2[6],ymm0[7,8,9,10,11,12,13],ymm2[14],ymm0[15]
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_30: # %else41
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_32
-; AVXNC-NEXT:  # %bb.31: # %cond.load43
-; AVXNC-NEXT:    vpbroadcastw (%rax), %ymm2
-; AVXNC-NEXT:    vpblendw {{.*#+}} ymm2 = ymm0[0,1,2,3,4,5,6],ymm2[7],ymm0[8,9,10,11,12,13,14],ymm2[15]
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_32: # %else44
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_34
-; AVXNC-NEXT:  # %bb.33: # %cond.load46
-; AVXNC-NEXT:    vpinsrw $0, (%rax), %xmm1, %xmm2
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_34: # %else47
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_36
-; AVXNC-NEXT:  # %bb.35: # %cond.load49
-; AVXNC-NEXT:    vpinsrw $1, (%rax), %xmm1, %xmm2
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_36: # %else50
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_38
-; AVXNC-NEXT:  # %bb.37: # %cond.load52
-; AVXNC-NEXT:    vpinsrw $2, (%rax), %xmm1, %xmm2
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_38: # %else53
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_40
-; AVXNC-NEXT:  # %bb.39: # %cond.load55
-; AVXNC-NEXT:    vpinsrw $3, (%rax), %xmm1, %xmm2
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_40: # %else56
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_42
-; AVXNC-NEXT:  # %bb.41: # %cond.load58
-; AVXNC-NEXT:    vpinsrw $4, (%rax), %xmm1, %xmm2
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_42: # %else59
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_44
-; AVXNC-NEXT:  # %bb.43: # %cond.load61
-; AVXNC-NEXT:    vpinsrw $5, (%rax), %xmm1, %xmm2
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_44: # %else62
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_46
-; AVXNC-NEXT:  # %bb.45: # %cond.load64
-; AVXNC-NEXT:    vpinsrw $6, (%rax), %xmm1, %xmm2
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_46: # %else65
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_48
-; AVXNC-NEXT:  # %bb.47: # %cond.load67
-; AVXNC-NEXT:    vpinsrw $7, (%rax), %xmm1, %xmm2
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_48: # %else68
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_50
-; AVXNC-NEXT:  # %bb.49: # %cond.load70
-; AVXNC-NEXT:    vpbroadcastw (%rax), %ymm2
-; AVXNC-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0],ymm1[1,2,3,4,5,6,7],ymm2[8],ymm1[9,10,11,12,13,14,15]
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_50: # %else71
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_52
-; AVXNC-NEXT:  # %bb.51: # %cond.load73
-; AVXNC-NEXT:    vpbroadcastw (%rax), %ymm2
-; AVXNC-NEXT:    vpblendw {{.*#+}} ymm2 = ymm1[0],ymm2[1],ymm1[2,3,4,5,6,7,8],ymm2[9],ymm1[10,11,12,13,14,15]
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_52: # %else74
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_54
-; AVXNC-NEXT:  # %bb.53: # %cond.load76
-; AVXNC-NEXT:    vpbroadcastw (%rax), %ymm2
-; AVXNC-NEXT:    vpblendw {{.*#+}} ymm2 = ymm1[0,1],ymm2[2],ymm1[3,4,5,6,7,8,9],ymm2[10],ymm1[11,12,13,14,15]
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_54: # %else77
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_56
-; AVXNC-NEXT:  # %bb.55: # %cond.load79
-; AVXNC-NEXT:    vpbroadcastw (%rax), %ymm2
-; AVXNC-NEXT:    vpblendw {{.*#+}} ymm2 = ymm1[0,1,2],ymm2[3],ymm1[4,5,6,7,8,9,10],ymm2[11],ymm1[12,13,14,15]
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_56: # %else80
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_58
-; AVXNC-NEXT:  # %bb.57: # %cond.load82
-; AVXNC-NEXT:    vpbroadcastw (%rax), %ymm2
-; AVXNC-NEXT:    vpblendw {{.*#+}} ymm2 = ymm1[0,1,2,3],ymm2[4],ymm1[5,6,7,8,9,10,11],ymm2[12],ymm1[13,14,15]
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_58: # %else83
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_60
-; AVXNC-NEXT:  # %bb.59: # %cond.load85
-; AVXNC-NEXT:    vpbroadcastw (%rax), %ymm2
-; AVXNC-NEXT:    vpblendw {{.*#+}} ymm2 = ymm1[0,1,2,3,4],ymm2[5],ymm1[6,7,8,9,10,11,12],ymm2[13],ymm1[14,15]
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_60: # %else86
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_62
-; AVXNC-NEXT:  # %bb.61: # %cond.load88
-; AVXNC-NEXT:    vpbroadcastw (%rax), %ymm2
-; AVXNC-NEXT:    vpblendw {{.*#+}} ymm2 = ymm1[0,1,2,3,4,5],ymm2[6],ymm1[7,8,9,10,11,12,13],ymm2[14],ymm1[15]
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_62: # %else89
-; AVXNC-NEXT:    xorl %eax, %eax
-; AVXNC-NEXT:    testb %al, %al
-; AVXNC-NEXT:    jne .LBB12_64
-; AVXNC-NEXT:  # %bb.63: # %cond.load91
-; AVXNC-NEXT:    vpbroadcastw (%rax), %ymm2
-; AVXNC-NEXT:    vpblendw {{.*#+}} ymm2 = ymm1[0,1,2,3,4,5,6],ymm2[7],ymm1[8,9,10,11,12,13,14],ymm2[15]
-; AVXNC-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
-; AVXNC-NEXT:  .LBB12_64: # %else92
+; AVXNC-NEXT:    vmovups (%rax), %ymm0
+; AVXNC-NEXT:  .LBB12_2:
+; AVXNC-NEXT:    vmovaps %ymm0, %ymm1
 ; AVXNC-NEXT:    retq
   %1 = call <32 x bfloat> @llvm.masked.load.v32bf16.p0(ptr poison, i32 2, <32 x i1> poison, <32 x bfloat> <bfloat 0xRBF80, bfloat 0xRBF80, bfloat 0xRBF80, bfloat 0xRBF80, bfloat 0xRBF80, bfloat 0xRBF80, bfloat 0xRBF80, bfloat 0xRBF80, bfloat 0xRBF80, bfloat 0xRBF80, bfloat 0xRBF80, bfloat 0xRBF80, bfloat 0xRBF80, bfloat 0xRBF80, bfloat 0xRBF80, bfloat 0xRBF80, bfloat 0xRBF80, bfloat 0xRBF80, bfloat 0xRBF80, bfloat 0xRBF80, bfloat 0xRBF80, bfloat 0xRBF80, bfloat 0xRBF80, bfloat 0xRBF80, bfloat 0xRBF80, bfloat 0xRBF80, bfloat 0xRBF80, bfloat 0xRBF80, bfloat 0xRBF80, bfloat 0xRBF80, bfloat 0xRBF80, bfloat 0xRBF80>)
   ret <32 x bfloat> %1
diff --git a/llvm/test/CodeGen/X86/shuffle-half.ll b/llvm/test/CodeGen/X86/shuffle-half.ll
index 291fe841043ed42..001db2c7cecae1f 100644
--- a/llvm/test/CodeGen/X86/shuffle-half.ll
+++ b/llvm/test/CodeGen/X86/shuffle-half.ll
@@ -4,305 +4,13 @@
 define <32 x half> @dump_vec() {
 ; CHECK-LABEL: dump_vec:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; CHECK-NEXT:    vxorps %xmm0, %xmm0, %xmm0
 ; CHECK-NEXT:    xorl %eax, %eax
 ; CHECK-NEXT:    testb %al, %al
 ; CHECK-NEXT:    jne .LBB0_2
 ; CHECK-NEXT:  # %bb.1: # %cond.load
-; CHECK-NEXT:    vpinsrw $0, (%rax), %xmm0, %xmm0
-; CHECK-NEXT:    vmovd {{.*#+}} xmm1 = [65535,0,0,0]
-; CHECK-NEXT:    vpand %ymm0, %ymm1, %ymm0
-; CHECK-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; CHECK-NEXT:    vinserti64x4 $0, %ymm0, %zmm1, %zmm0
-; CHECK-NEXT:  .LBB0_2: # %else
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_4
-; CHECK-NEXT:  # %bb.3: # %cond.load1
-; CHECK-NEXT:    vpbroadcastw (%rax), %xmm1
-; CHECK-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1],xmm0[2,3,4,5,6,7]
-; CHECK-NEXT:    vinserti32x4 $0, %xmm1, %zmm0, %zmm0
-; CHECK-NEXT:  .LBB0_4: # %else2
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_6
-; CHECK-NEXT:  # %bb.5: # %cond.load4
-; CHECK-NEXT:    vpbroadcastw (%rax), %xmm1
-; CHECK-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2],xmm0[3,4,5,6,7]
-; CHECK-NEXT:    vinserti32x4 $0, %xmm1, %zmm0, %zmm0
-; CHECK-NEXT:  .LBB0_6: # %else5
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_8
-; CHECK-NEXT:  # %bb.7: # %cond.load7
-; CHECK-NEXT:    vpbroadcastw (%rax), %xmm1
-; CHECK-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0,1,2],xmm1[3],xmm0[4,5,6,7]
-; CHECK-NEXT:    vinserti32x4 $0, %xmm1, %zmm0, %zmm0
-; CHECK-NEXT:  .LBB0_8: # %else8
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_10
-; CHECK-NEXT:  # %bb.9: # %cond.load10
-; CHECK-NEXT:    vpbroadcastw (%rax), %xmm1
-; CHECK-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0,1,2,3],xmm1[4],xmm0[5,6,7]
-; CHECK-NEXT:    vinserti32x4 $0, %xmm1, %zmm0, %zmm0
-; CHECK-NEXT:  .LBB0_10: # %else11
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_12
-; CHECK-NEXT:  # %bb.11: # %cond.load13
-; CHECK-NEXT:    vpbroadcastw (%rax), %xmm1
-; CHECK-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0,1,2,3,4],xmm1[5],xmm0[6,7]
-; CHECK-NEXT:    vinserti32x4 $0, %xmm1, %zmm0, %zmm0
-; CHECK-NEXT:  .LBB0_12: # %else14
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_14
-; CHECK-NEXT:  # %bb.13: # %cond.load16
-; CHECK-NEXT:    vpbroadcastw (%rax), %xmm1
-; CHECK-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0,1,2,3,4,5],xmm1[6],xmm0[7]
-; CHECK-NEXT:    vinserti32x4 $0, %xmm1, %zmm0, %zmm0
-; CHECK-NEXT:  .LBB0_14: # %else17
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_16
-; CHECK-NEXT:  # %bb.15: # %cond.load19
-; CHECK-NEXT:    vpbroadcastw (%rax), %xmm1
-; CHECK-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0,1,2,3,4,5,6],xmm1[7]
-; CHECK-NEXT:    vinserti32x4 $0, %xmm1, %zmm0, %zmm0
-; CHECK-NEXT:  .LBB0_16: # %else20
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_18
-; CHECK-NEXT:  # %bb.17: # %cond.load22
-; CHECK-NEXT:    vpbroadcastw (%rax), %ymm1
-; CHECK-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm0[1,2,3,4,5,6,7],ymm1[8],ymm0[9,10,11,12,13,14,15]
-; CHECK-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; CHECK-NEXT:    vshuff64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[4,5,6,7]
-; CHECK-NEXT:  .LBB0_18: # %else23
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_20
-; CHECK-NEXT:  # %bb.19: # %cond.load25
-; CHECK-NEXT:    vpbroadcastw (%rax), %ymm1
-; CHECK-NEXT:    vpblendw {{.*#+}} ymm1 = ymm0[0],ymm1[1],ymm0[2,3,4,5,6,7,8],ymm1[9],ymm0[10,11,12,13,14,15]
-; CHECK-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; CHECK-NEXT:    vshuff64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[4,5,6,7]
-; CHECK-NEXT:  .LBB0_20: # %else26
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_22
-; CHECK-NEXT:  # %bb.21: # %cond.load28
-; CHECK-NEXT:    vpbroadcastw (%rax), %ymm1
-; CHECK-NEXT:    vpblendw {{.*#+}} ymm1 = ymm0[0,1],ymm1[2],ymm0[3,4,5,6,7,8,9],ymm1[10],ymm0[11,12,13,14,15]
-; CHECK-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; CHECK-NEXT:    vshuff64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[4,5,6,7]
-; CHECK-NEXT:  .LBB0_22: # %else29
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_24
-; CHECK-NEXT:  # %bb.23: # %cond.load31
-; CHECK-NEXT:    vpbroadcastw (%rax), %ymm1
-; CHECK-NEXT:    vpblendw {{.*#+}} ymm1 = ymm0[0,1,2],ymm1[3],ymm0[4,5,6,7,8,9,10],ymm1[11],ymm0[12,13,14,15]
-; CHECK-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; CHECK-NEXT:    vshuff64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[4,5,6,7]
-; CHECK-NEXT:  .LBB0_24: # %else32
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_26
-; CHECK-NEXT:  # %bb.25: # %cond.load34
-; CHECK-NEXT:    vpbroadcastw (%rax), %ymm1
-; CHECK-NEXT:    vpblendw {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4],ymm0[5,6,7,8,9,10,11],ymm1[12],ymm0[13,14,15]
-; CHECK-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; CHECK-NEXT:    vshuff64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[4,5,6,7]
-; CHECK-NEXT:  .LBB0_26: # %else35
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_28
-; CHECK-NEXT:  # %bb.27: # %cond.load37
-; CHECK-NEXT:    vpbroadcastw (%rax), %ymm1
-; CHECK-NEXT:    vpblendw {{.*#+}} ymm1 = ymm0[0,1,2,3,4],ymm1[5],ymm0[6,7,8,9,10,11,12],ymm1[13],ymm0[14,15]
-; CHECK-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; CHECK-NEXT:    vshuff64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[4,5,6,7]
-; CHECK-NEXT:  .LBB0_28: # %else38
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_30
-; CHECK-NEXT:  # %bb.29: # %cond.load40
-; CHECK-NEXT:    vpbroadcastw (%rax), %ymm1
-; CHECK-NEXT:    vpblendw {{.*#+}} ymm1 = ymm0[0,1,2,3,4,5],ymm1[6],ymm0[7,8,9,10,11,12,13],ymm1[14],ymm0[15]
-; CHECK-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; CHECK-NEXT:    vshuff64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[4,5,6,7]
-; CHECK-NEXT:  .LBB0_30: # %else41
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_32
-; CHECK-NEXT:  # %bb.31: # %cond.load43
-; CHECK-NEXT:    vpbroadcastw (%rax), %ymm1
-; CHECK-NEXT:    vpblendw {{.*#+}} ymm1 = ymm0[0,1,2,3,4,5,6],ymm1[7],ymm0[8,9,10,11,12,13,14],ymm1[15]
-; CHECK-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; CHECK-NEXT:    vshuff64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[4,5,6,7]
-; CHECK-NEXT:  .LBB0_32: # %else44
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_34
-; CHECK-NEXT:  # %bb.33: # %cond.load46
-; CHECK-NEXT:    vpbroadcastw (%rax), %xmm1
-; CHECK-NEXT:    vextractf64x4 $1, %zmm0, %ymm2
-; CHECK-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3,4,5,6,7]
-; CHECK-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
-; CHECK-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
-; CHECK-NEXT:  .LBB0_34: # %else47
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_36
-; CHECK-NEXT:  # %bb.35: # %cond.load49
-; CHECK-NEXT:    vpbroadcastw (%rax), %xmm1
-; CHECK-NEXT:    vextractf64x4 $1, %zmm0, %ymm2
-; CHECK-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2,3,4,5,6,7]
-; CHECK-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
-; CHECK-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
-; CHECK-NEXT:  .LBB0_36: # %else50
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_38
-; CHECK-NEXT:  # %bb.37: # %cond.load52
-; CHECK-NEXT:    vpbroadcastw (%rax), %xmm1
-; CHECK-NEXT:    vextractf64x4 $1, %zmm0, %ymm2
-; CHECK-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1],xmm1[2],xmm2[3,4,5,6,7]
-; CHECK-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
-; CHECK-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
-; CHECK-NEXT:  .LBB0_38: # %else53
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_40
-; CHECK-NEXT:  # %bb.39: # %cond.load55
-; CHECK-NEXT:    vpbroadcastw (%rax), %xmm1
-; CHECK-NEXT:    vextractf64x4 $1, %zmm0, %ymm2
-; CHECK-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1,2],xmm1[3],xmm2[4,5,6,7]
-; CHECK-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
-; CHECK-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
-; CHECK-NEXT:  .LBB0_40: # %else56
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_42
-; CHECK-NEXT:  # %bb.41: # %cond.load58
-; CHECK-NEXT:    vpbroadcastw (%rax), %xmm1
-; CHECK-NEXT:    vextractf64x4 $1, %zmm0, %ymm2
-; CHECK-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1,2,3],xmm1[4],xmm2[5,6,7]
-; CHECK-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
-; CHECK-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
-; CHECK-NEXT:  .LBB0_42: # %else59
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_44
-; CHECK-NEXT:  # %bb.43: # %cond.load61
-; CHECK-NEXT:    vpbroadcastw (%rax), %xmm1
-; CHECK-NEXT:    vextractf64x4 $1, %zmm0, %ymm2
-; CHECK-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1,2,3,4],xmm1[5],xmm2[6,7]
-; CHECK-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
-; CHECK-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
-; CHECK-NEXT:  .LBB0_44: # %else62
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_46
-; CHECK-NEXT:  # %bb.45: # %cond.load64
-; CHECK-NEXT:    vpbroadcastw (%rax), %xmm1
-; CHECK-NEXT:    vextractf64x4 $1, %zmm0, %ymm2
-; CHECK-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1,2,3,4,5],xmm1[6],xmm2[7]
-; CHECK-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
-; CHECK-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
-; CHECK-NEXT:  .LBB0_46: # %else65
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_48
-; CHECK-NEXT:  # %bb.47: # %cond.load67
-; CHECK-NEXT:    vpbroadcastw (%rax), %xmm1
-; CHECK-NEXT:    vextractf64x4 $1, %zmm0, %ymm2
-; CHECK-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1,2,3,4,5,6],xmm1[7]
-; CHECK-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
-; CHECK-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
-; CHECK-NEXT:  .LBB0_48: # %else68
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_50
-; CHECK-NEXT:  # %bb.49: # %cond.load70
-; CHECK-NEXT:    vpbroadcastw (%rax), %ymm1
-; CHECK-NEXT:    vextractf64x4 $1, %zmm0, %ymm2
-; CHECK-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm2[1,2,3,4,5,6,7],ymm1[8],ymm2[9,10,11,12,13,14,15]
-; CHECK-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; CHECK-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
-; CHECK-NEXT:  .LBB0_50: # %else71
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_52
-; CHECK-NEXT:  # %bb.51: # %cond.load73
-; CHECK-NEXT:    vpbroadcastw (%rax), %ymm1
-; CHECK-NEXT:    vextractf64x4 $1, %zmm0, %ymm2
-; CHECK-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0],ymm1[1],ymm2[2,3,4,5,6,7,8],ymm1[9],ymm2[10,11,12,13,14,15]
-; CHECK-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; CHECK-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
-; CHECK-NEXT:  .LBB0_52: # %else74
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_54
-; CHECK-NEXT:  # %bb.53: # %cond.load76
-; CHECK-NEXT:    vpbroadcastw (%rax), %ymm1
-; CHECK-NEXT:    vextractf64x4 $1, %zmm0, %ymm2
-; CHECK-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0,1],ymm1[2],ymm2[3,4,5,6,7,8,9],ymm1[10],ymm2[11,12,13,14,15]
-; CHECK-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; CHECK-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
-; CHECK-NEXT:  .LBB0_54: # %else77
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_56
-; CHECK-NEXT:  # %bb.55: # %cond.load79
-; CHECK-NEXT:    vpbroadcastw (%rax), %ymm1
-; CHECK-NEXT:    vextractf64x4 $1, %zmm0, %ymm2
-; CHECK-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3],ymm2[4,5,6,7,8,9,10],ymm1[11],ymm2[12,13,14,15]
-; CHECK-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; CHECK-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
-; CHECK-NEXT:  .LBB0_56: # %else80
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_58
-; CHECK-NEXT:  # %bb.57: # %cond.load82
-; CHECK-NEXT:    vpbroadcastw (%rax), %ymm1
-; CHECK-NEXT:    vextractf64x4 $1, %zmm0, %ymm2
-; CHECK-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4],ymm2[5,6,7,8,9,10,11],ymm1[12],ymm2[13,14,15]
-; CHECK-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; CHECK-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
-; CHECK-NEXT:  .LBB0_58: # %else83
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_60
-; CHECK-NEXT:  # %bb.59: # %cond.load85
-; CHECK-NEXT:    vpbroadcastw (%rax), %ymm1
-; CHECK-NEXT:    vextractf64x4 $1, %zmm0, %ymm2
-; CHECK-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0,1,2,3,4],ymm1[5],ymm2[6,7,8,9,10,11,12],ymm1[13],ymm2[14,15]
-; CHECK-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; CHECK-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
-; CHECK-NEXT:  .LBB0_60: # %else86
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_62
-; CHECK-NEXT:  # %bb.61: # %cond.load88
-; CHECK-NEXT:    vpbroadcastw (%rax), %ymm1
-; CHECK-NEXT:    vextractf64x4 $1, %zmm0, %ymm2
-; CHECK-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6],ymm2[7,8,9,10,11,12,13],ymm1[14],ymm2[15]
-; CHECK-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; CHECK-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
-; CHECK-NEXT:  .LBB0_62: # %else89
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    testb %al, %al
-; CHECK-NEXT:    jne .LBB0_64
-; CHECK-NEXT:  # %bb.63: # %cond.load91
-; CHECK-NEXT:    vpbroadcastw (%rax), %ymm1
-; CHECK-NEXT:    vextractf64x4 $1, %zmm0, %ymm2
-; CHECK-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5,6],ymm1[7],ymm2[8,9,10,11,12,13,14],ymm1[15]
-; CHECK-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; CHECK-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
-; CHECK-NEXT:  .LBB0_64: # %else92
+; CHECK-NEXT:    vmovups (%rax), %zmm0
+; CHECK-NEXT:  .LBB0_2:
 ; CHECK-NEXT:    retq
   %1 = call <32 x half> @llvm.masked.load.v32f16.p0(ptr poison, i32 2, <32 x i1> poison, <32 x half> <half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0>)
   ret <32 x half> %1
diff --git a/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-load.ll b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-load.ll
index 9b1c59829b9ffb7..fffb5f021e52d4c 100644
--- a/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-load.ll
+++ b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-load.ll
@@ -32,8 +32,8 @@ define <2 x i64> @scalarize_v2i64(ptr %p, <2 x i1> %mask, <2 x i64> %passthru) {
 
 define <2 x i64> @scalarize_v2i64_ones_mask(ptr %p, <2 x i64> %passthru) {
 ; CHECK-LABEL: @scalarize_v2i64_ones_mask(
-; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr [[P:%.*]], align 8
-; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+; CHECK-NEXT:    [[RET:%.*]] = load <2 x i64>, ptr [[P:%.*]], align 8
+; CHECK-NEXT:    ret <2 x i64> [[RET]]
 ;
   %ret = call <2 x i64> @llvm.masked.load.v2i64.p0(ptr %p, i32 8, <2 x i1> <i1 true, i1 true>, <2 x i64> %passthru)
   ret <2 x i64> %ret
@@ -58,34 +58,18 @@ define <2 x i64> @scalarize_v2i64_const_mask(ptr %p, <2 x i64> %passthru) {
   ret <2 x i64> %ret
 }
 
-; To be fixed: If the mask is the splat/broadcast of a non-constant value, use a
-; vector load
 define <2 x i64> @scalarize_v2i64_splat_mask(ptr %p, i1 %mask, <2 x i64> %passthrough) {
 ; CHECK-LABEL: @scalarize_v2i64_splat_mask(
 ; CHECK-NEXT:    [[MASK_VEC:%.*]] = insertelement <2 x i1> poison, i1 [[MASK:%.*]], i32 0
 ; CHECK-NEXT:    [[MASK_SPLAT:%.*]] = shufflevector <2 x i1> [[MASK_VEC]], <2 x i1> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK_SPLAT]] to i2
-; CHECK-NEXT:    [[TMP1:%.*]] = and i2 [[SCALAR_MASK]], 1
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i2 [[TMP1]], 0
-; CHECK-NEXT:    br i1 [[TMP2]], label [[COND_LOAD:%.*]], label [[ELSE:%.*]]
+; CHECK-NEXT:    [[MASK_SPLAT_FIRST:%.*]] = extractelement <2 x i1> [[MASK_SPLAT]], i64 0
+; CHECK-NEXT:    br i1 [[MASK_SPLAT_FIRST]], label [[COND_LOAD:%.*]], label [[TMP1:%.*]]
 ; CHECK:       cond.load:
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[P:%.*]], i32 0
-; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr [[TMP3]], align 8
-; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x i64> [[PASSTHROUGH:%.*]], i64 [[TMP4]], i64 0
-; CHECK-NEXT:    br label [[ELSE]]
-; CHECK:       else:
-; CHECK-NEXT:    [[RES_PHI_ELSE:%.*]] = phi <2 x i64> [ [[TMP5]], [[COND_LOAD]] ], [ [[PASSTHROUGH]], [[TMP0:%.*]] ]
-; CHECK-NEXT:    [[TMP6:%.*]] = and i2 [[SCALAR_MASK]], -2
-; CHECK-NEXT:    [[TMP7:%.*]] = icmp ne i2 [[TMP6]], 0
-; CHECK-NEXT:    br i1 [[TMP7]], label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]
-; CHECK:       cond.load1:
-; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[P]], i32 1
-; CHECK-NEXT:    [[TMP9:%.*]] = load i64, ptr [[TMP8]], align 8
-; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <2 x i64> [[RES_PHI_ELSE]], i64 [[TMP9]], i64 1
-; CHECK-NEXT:    br label [[ELSE2]]
-; CHECK:       else2:
-; CHECK-NEXT:    [[RES_PHI_ELSE3:%.*]] = phi <2 x i64> [ [[TMP10]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ]
-; CHECK-NEXT:    ret <2 x i64> [[RES_PHI_ELSE3]]
+; CHECK-NEXT:    [[RET_COND_LOAD:%.*]] = load <2 x i64>, ptr [[P:%.*]], align 8
+; CHECK-NEXT:    br label [[TMP1]]
+; CHECK:       1:
+; CHECK-NEXT:    [[RET:%.*]] = phi <2 x i64> [ [[RET_COND_LOAD]], [[COND_LOAD]] ], [ [[PASSTHROUGH:%.*]], [[TMP0:%.*]] ]
+; CHECK-NEXT:    ret <2 x i64> [[RET]]
 ;
   %mask.vec = insertelement <2 x i1> poison, i1 %mask, i32 0
   %mask.splat = shufflevector <2 x i1> %mask.vec, <2 x i1> poison, <2 x i32> zeroinitializer
diff --git a/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-store.ll b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-store.ll
index cd2815e67e67205..4e3679dc5da99e1 100644
--- a/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-store.ll
+++ b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-store.ll
@@ -56,31 +56,16 @@ define void @scalarize_v2i64_const_mask(ptr %p, <2 x i64> %data) {
   ret void
 }
 
-; To be fixed: If the mask is the splat/broadcast of a non-constant value, use a
-; vector store
 define void @scalarize_v2i64_splat_mask(ptr %p, <2 x i64> %data, i1 %mask) {
 ; CHECK-LABEL: @scalarize_v2i64_splat_mask(
 ; CHECK-NEXT:    [[MASK_VEC:%.*]] = insertelement <2 x i1> poison, i1 [[MASK:%.*]], i32 0
 ; CHECK-NEXT:    [[MASK_SPLAT:%.*]] = shufflevector <2 x i1> [[MASK_VEC]], <2 x i1> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK_SPLAT]] to i2
-; CHECK-NEXT:    [[TMP1:%.*]] = and i2 [[SCALAR_MASK]], 1
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i2 [[TMP1]], 0
-; CHECK-NEXT:    br i1 [[TMP2]], label [[COND_STORE:%.*]], label [[ELSE:%.*]]
+; CHECK-NEXT:    [[MASK_SPLAT_FIRST:%.*]] = extractelement <2 x i1> [[MASK_SPLAT]], i64 0
+; CHECK-NEXT:    br i1 [[MASK_SPLAT_FIRST]], label [[COND_STORE:%.*]], label [[TMP1:%.*]]
 ; CHECK:       cond.store:
-; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x i64> [[DATA:%.*]], i64 0
-; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[P:%.*]], i32 0
-; CHECK-NEXT:    store i64 [[TMP3]], ptr [[TMP4]], align 8
-; CHECK-NEXT:    br label [[ELSE]]
-; CHECK:       else:
-; CHECK-NEXT:    [[TMP5:%.*]] = and i2 [[SCALAR_MASK]], -2
-; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne i2 [[TMP5]], 0
-; CHECK-NEXT:    br i1 [[TMP6]], label [[COND_STORE1:%.*]], label [[ELSE2:%.*]]
-; CHECK:       cond.store1:
-; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i64> [[DATA]], i64 1
-; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[P]], i32 1
-; CHECK-NEXT:    store i64 [[TMP7]], ptr [[TMP8]], align 8
-; CHECK-NEXT:    br label [[ELSE2]]
-; CHECK:       else2:
+; CHECK-NEXT:    store <2 x i64> [[DATA:%.*]], ptr [[P:%.*]], align 8
+; CHECK-NEXT:    br label [[TMP1]]
+; CHECK:       1:
 ; CHECK-NEXT:    ret void
 ;
   %mask.vec = insertelement <2 x i1> poison, i1 %mask, i32 0

From bd9f2c2ba095fa8345bba9f74b279ff20d5ddaab Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Fri, 16 Aug 2024 16:27:03 -0500
Subject: [PATCH 239/441] [libc] Add missing math definitions for round and
 scal for GPU (#104636)

Summary:
These can be enabled
---
 libc/config/gpu/entrypoints.txt | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/libc/config/gpu/entrypoints.txt b/libc/config/gpu/entrypoints.txt
index 9ead5aa92d5a92b..bbae3298fae615e 100644
--- a/libc/config/gpu/entrypoints.txt
+++ b/libc/config/gpu/entrypoints.txt
@@ -425,13 +425,15 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.rint
     libc.src.math.rintf
     libc.src.math.rintl
-    libc.src.math.round
     libc.src.math.roundeven
     libc.src.math.roundevenf
     libc.src.math.roundevenl
+    libc.src.math.round
     libc.src.math.roundf
+    libc.src.math.roundl
     libc.src.math.scalbln
     libc.src.math.scalblnf
+    libc.src.math.scalblnl
     libc.src.math.scalbn
     libc.src.math.scalbnf
     libc.src.math.scalbnl
@@ -459,6 +461,7 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.tgammaf
     libc.src.math.totalorder
     libc.src.math.totalorderf
+    libc.src.math.totalorderl
     libc.src.math.totalordermag
     libc.src.math.totalordermagf
     libc.src.math.totalordermagl

From ebe7265b142f370f0a563fece5db22f57383ba2d Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Fri, 16 Aug 2024 14:54:51 -0700
Subject: [PATCH 240/441] [Mips] Fix fast isel for i16 bswap. (#103398)

We need to mask the SRL result to 8 bits before ORing in the SLL. This
is needed in case bits 23:16 of the input aren't zero. They will have
been shifted into bits 15:8.

We don't need to AND the result with 0xffff. It's ok if the upper 16
bits of the register are garbage.

Fixes #103035.
---
 llvm/lib/Target/Mips/MipsFastISel.cpp      | 4 ++--
 llvm/test/CodeGen/Mips/Fast-ISel/bswap1.ll | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/Target/Mips/MipsFastISel.cpp b/llvm/lib/Target/Mips/MipsFastISel.cpp
index 3485825b77627ee..7d8278c8ca3cf55 100644
--- a/llvm/lib/Target/Mips/MipsFastISel.cpp
+++ b/llvm/lib/Target/Mips/MipsFastISel.cpp
@@ -1608,8 +1608,8 @@ bool MipsFastISel::fastLowerIntrinsicCall(const IntrinsicInst *II) {
         }
         emitInst(Mips::SLL, TempReg[0]).addReg(SrcReg).addImm(8);
         emitInst(Mips::SRL, TempReg[1]).addReg(SrcReg).addImm(8);
-        emitInst(Mips::OR, TempReg[2]).addReg(TempReg[0]).addReg(TempReg[1]);
-        emitInst(Mips::ANDi, DestReg).addReg(TempReg[2]).addImm(0xFFFF);
+        emitInst(Mips::ANDi, TempReg[2]).addReg(TempReg[1]).addImm(0xFF);
+        emitInst(Mips::OR, DestReg).addReg(TempReg[0]).addReg(TempReg[2]);
         updateValueMap(II, DestReg);
         return true;
       }
diff --git a/llvm/test/CodeGen/Mips/Fast-ISel/bswap1.ll b/llvm/test/CodeGen/Mips/Fast-ISel/bswap1.ll
index bd762a0e1d741f0..ce664c78e86c2ae 100644
--- a/llvm/test/CodeGen/Mips/Fast-ISel/bswap1.ll
+++ b/llvm/test/CodeGen/Mips/Fast-ISel/bswap1.ll
@@ -21,8 +21,8 @@ define void @b16() {
 
   ; 32R1:           sll   $[[TMP1:[0-9]+]], $[[A_VAL]], 8
   ; 32R1:           srl   $[[TMP2:[0-9]+]], $[[A_VAL]], 8
-  ; 32R1:           or    $[[TMP3:[0-9]+]], $[[TMP1]], $[[TMP2]]
-  ; 32R1:           andi  $[[TMP4:[0-9]+]], $[[TMP3]], 65535
+  ; 32R1:           andi  $[[TMP3:[0-9]+]], $[[TMP2]], 255
+  ; 32R1:           or    $[[RESULT:[0-9]+]], $[[TMP1]], $[[TMP3]]
 
   ; 32R2:           wsbh  $[[RESULT:[0-9]+]], $[[A_VAL]]
 

From e4f3735d5f600b17b8f86956162d41ce82096685 Mon Sep 17 00:00:00 2001
From: "Oleksandr T." <oleksandr.tarasiuk@outlook.com>
Date: Sat, 17 Aug 2024 01:01:33 +0300
Subject: [PATCH 241/441] [Clang] fix crash by avoiding invalidation of extern
 main declaration during strictness checks (#104594)

Fixes #104570
---
 clang/lib/Sema/SemaDecl.cpp                   |  5 +---
 .../basic/basic.start/basic.start.main/p3.cpp | 28 ++++++++++++++-----
 2 files changed, 22 insertions(+), 11 deletions(-)

diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index d19a16cf2ba150f..1d85341bc9a8b65 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -12230,12 +12230,9 @@ void Sema::CheckMain(FunctionDecl *FD, const DeclSpec &DS) {
   //    The main function shall not be declared with a linkage-specification.
   if (FD->isExternCContext() ||
       (FD->isExternCXXContext() &&
-       FD->getDeclContext()->getRedeclContext()->isTranslationUnit())) {
+       FD->getDeclContext()->getRedeclContext()->isTranslationUnit()))
     Diag(FD->getLocation(), diag::ext_main_invalid_linkage_specification)
         << FD->getLanguageLinkage();
-    FD->setInvalidDecl();
-    return;
-  }
 
   // C++11 [basic.start.main]p3:
   //   A program that [...] declares main to be inline, static or
diff --git a/clang/test/CXX/basic/basic.start/basic.start.main/p3.cpp b/clang/test/CXX/basic/basic.start/basic.start.main/p3.cpp
index d23d00ccdeebc1b..497a9328c115618 100644
--- a/clang/test/CXX/basic/basic.start/basic.start.main/p3.cpp
+++ b/clang/test/CXX/basic/basic.start/basic.start.main/p3.cpp
@@ -11,6 +11,8 @@
 // RUN: %clang_cc1 -fsyntax-only -verify -pedantic %s -DTEST11
 // RUN: %clang_cc1 -fsyntax-only -verify -pedantic %s -DTEST12
 // RUN: %clang_cc1 -fsyntax-only -verify -pedantic %s -DTEST13
+// RUN: %clang_cc1 -fsyntax-only -verify -pedantic %s -DTEST14
+// RUN: %clang_cc1 -triple x86_64-linux -emit-llvm-only -verify -pedantic %s -DTEST15
 
 #if TEST1
 int main; // expected-error{{main cannot be declared as a variable in the global scope}}
@@ -78,12 +80,14 @@ namespace ns {
 extern "C" struct A { int main(); }; // ok
 
 namespace c {
-  extern "C" void main(); // expected-warning {{'main' should not be 'extern "C"'}}
+  extern "C" void main(); // expected-error {{'main' must return 'int'}} \
+                          // expected-warning {{'main' should not be 'extern "C"'}}
 }
 
 extern "C" {
   namespace Z {
-    void main(); // expected-warning {{'main' should not be 'extern "C"'}}
+    void main(); // expected-error {{'main' must return 'int'}} \
+                 // expected-warning {{'main' should not be 'extern "C"'}}
   }
 }
 
@@ -102,11 +106,6 @@ extern "C++" {
   int main(); // expected-warning {{'main' should not be 'extern "C++"'}}
 }
 
-extern "C" {
-  int main(); // expected-warning {{'main' should not be 'extern "C"'}}
-}
-
-extern "C" int main(); // expected-warning {{'main' should not be 'extern "C"'}}
 extern "C++" int main(); // expected-warning {{'main' should not be 'extern "C++"'}}
 
 namespace ns1 {
@@ -122,6 +121,21 @@ namespace ns2 {
   extern "C++" void main() {} // ok
 }
 
+#elif TEST14
+extern "C" {
+  int main(); // expected-warning {{'main' should not be 'extern "C"'}}
+}
+
+extern "C" int main(); // expected-warning {{'main' should not be 'extern "C"'}}
+
+#elif TEST15
+extern "C" __attribute__((visibility("default"))) __attribute__((weak))
+int main(); // expected-warning {{'main' should not be 'extern "C"'}}
+
+unsigned long g() {
+  return reinterpret_cast<unsigned long>(&main); // expected-warning {{referring to 'main' within an expression is a Clang extension}}
+}
+
 #else
 #error Unknown Test
 #endif

From bfdeceada36c0f14a7bb2e264f0b801e442368a7 Mon Sep 17 00:00:00 2001
From: Dmitry Chestnykh <dm.chestnykh@gmail.com>
Date: Sat, 17 Aug 2024 01:08:56 +0300
Subject: [PATCH 242/441] [NFC][cxxabi] Apply `cp-to-llvm.sh` (#101970)

---
 llvm/include/llvm/Demangle/ItaniumDemangle.h | 38 ++++++++++----------
 1 file changed, 19 insertions(+), 19 deletions(-)

diff --git a/llvm/include/llvm/Demangle/ItaniumDemangle.h b/llvm/include/llvm/Demangle/ItaniumDemangle.h
index b3976d7fbf41ff3..fb2060fa07f7243 100644
--- a/llvm/include/llvm/Demangle/ItaniumDemangle.h
+++ b/llvm/include/llvm/Demangle/ItaniumDemangle.h
@@ -30,7 +30,7 @@
 #include <type_traits>
 #include <utility>
 
-#ifdef _LIBCXXABI_COMPILER_CLANG
+#if defined(__clang__)
 #pragma clang diagnostic push
 #pragma clang diagnostic ignored "-Wunused-template"
 #endif
@@ -199,8 +199,7 @@ class Node {
 
   Prec Precedence : 6;
 
-  // FIXME: Make these protected.
-public:
+protected:
   /// Tracks if this node has a component on its right side, in which case we
   /// need to call printRight.
   Cache RHSComponentCache : 2;
@@ -254,6 +253,9 @@ class Node {
   Kind getKind() const { return K; }
 
   Prec getPrecedence() const { return Precedence; }
+  Cache getRHSComponentCache() const { return RHSComponentCache; }
+  Cache getArrayCache() const { return ArrayCache; }
+  Cache getFunctionCache() const { return FunctionCache; }
 
   virtual bool hasRHSComponentSlow(OutputBuffer &) const { return false; }
   virtual bool hasArraySlow(OutputBuffer &) const { return false; }
@@ -423,8 +425,8 @@ class QualType final : public Node {
 
 public:
   QualType(const Node *Child_, Qualifiers Quals_)
-      : Node(KQualType, Child_->RHSComponentCache,
-             Child_->ArrayCache, Child_->FunctionCache),
+      : Node(KQualType, Child_->getRHSComponentCache(), Child_->getArrayCache(),
+             Child_->getFunctionCache()),
         Quals(Quals_), Child(Child_) {}
 
   Qualifiers getQuals() const { return Quals; }
@@ -553,8 +555,8 @@ struct AbiTagAttr : Node {
   std::string_view Tag;
 
   AbiTagAttr(Node *Base_, std::string_view Tag_)
-      : Node(KAbiTagAttr, Base_->RHSComponentCache, Base_->ArrayCache,
-             Base_->FunctionCache),
+      : Node(KAbiTagAttr, Base_->getRHSComponentCache(), Base_->getArrayCache(),
+             Base_->getFunctionCache()),
         Base(Base_), Tag(Tag_) {}
 
   template<typename Fn> void match(Fn F) const { F(Base, Tag); }
@@ -614,7 +616,7 @@ class PointerType final : public Node {
 
 public:
   PointerType(const Node *Pointee_)
-      : Node(KPointerType, Pointee_->RHSComponentCache),
+      : Node(KPointerType, Pointee_->getRHSComponentCache()),
         Pointee(Pointee_) {}
 
   const Node *getPointee() const { return Pointee; }
@@ -698,7 +700,7 @@ class ReferenceType : public Node {
 
 public:
   ReferenceType(const Node *Pointee_, ReferenceKind RK_)
-      : Node(KReferenceType, Pointee_->RHSComponentCache),
+      : Node(KReferenceType, Pointee_->getRHSComponentCache()),
         Pointee(Pointee_), RK(RK_) {}
 
   template<typename Fn> void match(Fn F) const { F(Pointee, RK); }
@@ -741,7 +743,7 @@ class PointerToMemberType final : public Node {
 
 public:
   PointerToMemberType(const Node *ClassType_, const Node *MemberType_)
-      : Node(KPointerToMemberType, MemberType_->RHSComponentCache),
+      : Node(KPointerToMemberType, MemberType_->getRHSComponentCache()),
         ClassType(ClassType_), MemberType(MemberType_) {}
 
   template<typename Fn> void match(Fn F) const { F(ClassType, MemberType); }
@@ -1382,16 +1384,14 @@ class ParameterPack final : public Node {
 public:
   ParameterPack(NodeArray Data_) : Node(KParameterPack), Data(Data_) {
     ArrayCache = FunctionCache = RHSComponentCache = Cache::Unknown;
-    if (std::all_of(Data.begin(), Data.end(), [](Node* P) {
-          return P->ArrayCache == Cache::No;
-        }))
+    if (std::all_of(Data.begin(), Data.end(),
+                    [](Node *P) { return P->getArrayCache() == Cache::No; }))
       ArrayCache = Cache::No;
-    if (std::all_of(Data.begin(), Data.end(), [](Node* P) {
-          return P->FunctionCache == Cache::No;
-        }))
+    if (std::all_of(Data.begin(), Data.end(),
+                    [](Node *P) { return P->getFunctionCache() == Cache::No; }))
       FunctionCache = Cache::No;
-    if (std::all_of(Data.begin(), Data.end(), [](Node* P) {
-          return P->RHSComponentCache == Cache::No;
+    if (std::all_of(Data.begin(), Data.end(), [](Node *P) {
+          return P->getRHSComponentCache() == Cache::No;
         }))
       RHSComponentCache = Cache::No;
   }
@@ -5947,7 +5947,7 @@ struct ManglingParser : AbstractManglingParser<ManglingParser<Alloc>, Alloc> {
 
 DEMANGLE_NAMESPACE_END
 
-#ifdef _LIBCXXABI_COMPILER_CLANG
+#if defined(__clang__)
 #pragma clang diagnostic pop
 #endif
 

From 324b676a3d0449add8d4e24047680125f9b9a716 Mon Sep 17 00:00:00 2001
From: Thurston Dang <thurston@google.com>
Date: Fri, 16 Aug 2024 22:32:12 +0000
Subject: [PATCH 243/441] Revert "[AArch64] Fold more load.x into load.i with
 large offset"

This reverts commit 43ffe2eed0d9f73789dbe213023733d164999306.

Reason: buildbot breakage starting at https://lab.llvm.org/buildbot/#/builders/85/builds/1102

I manually bisected and found that clang crashed with 43ffe2eed0d9f73789dbe213023733d164999306 but not the immediately preceding commit (33190490c667aaf8b08d5af8b8ce84524f856e80)
---
 llvm/lib/Target/AArch64/AArch64InstrInfo.cpp  | 13 ---
 .../AArch64/AArch64LoadStoreOptimizer.cpp     | 51 +----------
 llvm/test/CodeGen/AArch64/arm64-addrmode.ll   | 85 +++++++++++--------
 3 files changed, 53 insertions(+), 96 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index 3ea2ba235ca39c9..697ae510a95655c 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -4521,20 +4521,7 @@ AArch64InstrInfo::getLdStAmountOp(const MachineInstr &MI) {
   switch (MI.getOpcode()) {
   default:
     llvm_unreachable("Unexpected opcode");
-  case AArch64::LDRBroX:
   case AArch64::LDRBBroX:
-  case AArch64::LDRSBXroX:
-  case AArch64::LDRSBWroX:
-  case AArch64::LDRHroX:
-  case AArch64::LDRHHroX:
-  case AArch64::LDRSHXroX:
-  case AArch64::LDRSHWroX:
-  case AArch64::LDRWroX:
-  case AArch64::LDRSroX:
-  case AArch64::LDRSWroX:
-  case AArch64::LDRDroX:
-  case AArch64::LDRXroX:
-  case AArch64::LDRQroX:
     return MI.getOperand(4);
   }
 }
diff --git a/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp b/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
index de1727aa6ec70f7..8de3f8db84ae2b7 100644
--- a/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
+++ b/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
@@ -509,38 +509,12 @@ static unsigned getPreIndexedOpcode(unsigned Opc) {
 }
 
 static unsigned getBaseAddressOpcode(unsigned Opc) {
-  // TODO: Add more index address stores.
+  // TODO: Add more index address loads/stores.
   switch (Opc) {
   default:
     llvm_unreachable("Opcode has no base address equivalent!");
-  case AArch64::LDRBroX:
-    return AArch64::LDRBui;
   case AArch64::LDRBBroX:
     return AArch64::LDRBBui;
-  case AArch64::LDRSBXroX:
-    return AArch64::LDRSBXui;
-  case AArch64::LDRSBWroX:
-    return AArch64::LDRSBWui;
-  case AArch64::LDRHroX:
-    return AArch64::LDRHui;
-  case AArch64::LDRHHroX:
-    return AArch64::LDRHHui;
-  case AArch64::LDRSHXroX:
-    return AArch64::LDRSHXui;
-  case AArch64::LDRSHWroX:
-    return AArch64::LDRSHWui;
-  case AArch64::LDRWroX:
-    return AArch64::LDRWui;
-  case AArch64::LDRSroX:
-    return AArch64::LDRSui;
-  case AArch64::LDRSWroX:
-    return AArch64::LDRSWui;
-  case AArch64::LDRDroX:
-    return AArch64::LDRDui;
-  case AArch64::LDRXroX:
-    return AArch64::LDRXui;
-  case AArch64::LDRQroX:
-    return AArch64::LDRQui;
   }
 }
 
@@ -792,31 +766,10 @@ static bool isMergeableIndexLdSt(MachineInstr &MI, int &Scale) {
   default:
     return false;
   // Scaled instructions.
-  // TODO: Add more index address stores.
-  case AArch64::LDRBroX:
+  // TODO: Add more index address loads/stores.
   case AArch64::LDRBBroX:
-  case AArch64::LDRSBXroX:
-  case AArch64::LDRSBWroX:
     Scale = 1;
     return true;
-  case AArch64::LDRHroX:
-  case AArch64::LDRHHroX:
-  case AArch64::LDRSHXroX:
-  case AArch64::LDRSHWroX:
-    Scale = 2;
-    return true;
-  case AArch64::LDRWroX:
-  case AArch64::LDRSroX:
-  case AArch64::LDRSWroX:
-    Scale = 4;
-    return true;
-  case AArch64::LDRDroX:
-  case AArch64::LDRXroX:
-    Scale = 8;
-    return true;
-  case AArch64::LDRQroX:
-    Scale = 16;
-    return true;
   }
 }
 
diff --git a/llvm/test/CodeGen/AArch64/arm64-addrmode.ll b/llvm/test/CodeGen/AArch64/arm64-addrmode.ll
index bfef61abd8c129f..2181eaaee7db686 100644
--- a/llvm/test/CodeGen/AArch64/arm64-addrmode.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-addrmode.ll
@@ -239,8 +239,9 @@ define i32 @LdOffset_i8_zext32(ptr %a)  {
 define i32 @LdOffset_i8_sext32(ptr %a)  {
 ; CHECK-LABEL: LdOffset_i8_sext32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    add x8, x0, #253, lsl #12 // =1036288
-; CHECK-NEXT:    ldrsb w0, [x8, #3704]
+; CHECK-NEXT:    mov w8, #56952 // =0xde78
+; CHECK-NEXT:    movk w8, #15, lsl #16
+; CHECK-NEXT:    ldrsb w0, [x0, x8]
 ; CHECK-NEXT:    ret
   %arrayidx = getelementptr inbounds i8, ptr %a, i64 1039992
   %val = load i8, ptr %arrayidx, align 1
@@ -265,8 +266,9 @@ define i64 @LdOffset_i8_zext64(ptr %a)  {
 define i64 @LdOffset_i8_sext64(ptr %a)  {
 ; CHECK-LABEL: LdOffset_i8_sext64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    add x8, x0, #253, lsl #12 // =1036288
-; CHECK-NEXT:    ldrsb x0, [x8, #3704]
+; CHECK-NEXT:    mov w8, #56952 // =0xde78
+; CHECK-NEXT:    movk w8, #15, lsl #16
+; CHECK-NEXT:    ldrsb x0, [x0, x8]
 ; CHECK-NEXT:    ret
   %arrayidx = getelementptr inbounds i8, ptr %a, i64 1039992
   %val = load i8, ptr %arrayidx, align 1
@@ -278,8 +280,9 @@ define i64 @LdOffset_i8_sext64(ptr %a)  {
 define i16 @LdOffset_i16(ptr %a)  {
 ; CHECK-LABEL: LdOffset_i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    add x8, x0, #506, lsl #12 // =2072576
-; CHECK-NEXT:    ldrh w0, [x8, #7408]
+; CHECK-NEXT:    mov w8, #48368 // =0xbcf0
+; CHECK-NEXT:    movk w8, #31, lsl #16
+; CHECK-NEXT:    ldrh w0, [x0, x8]
 ; CHECK-NEXT:    ret
   %arrayidx = getelementptr inbounds i16, ptr %a, i64 1039992
   %val = load i16, ptr %arrayidx, align 2
@@ -290,8 +293,9 @@ define i16 @LdOffset_i16(ptr %a)  {
 define i32 @LdOffset_i16_zext32(ptr %a)  {
 ; CHECK-LABEL: LdOffset_i16_zext32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    add x8, x0, #506, lsl #12 // =2072576
-; CHECK-NEXT:    ldrh w0, [x8, #7408]
+; CHECK-NEXT:    mov w8, #48368 // =0xbcf0
+; CHECK-NEXT:    movk w8, #31, lsl #16
+; CHECK-NEXT:    ldrh w0, [x0, x8]
 ; CHECK-NEXT:    ret
   %arrayidx = getelementptr inbounds i16, ptr %a, i64 1039992
   %val = load i16, ptr %arrayidx, align 2
@@ -303,8 +307,9 @@ define i32 @LdOffset_i16_zext32(ptr %a)  {
 define i32 @LdOffset_i16_sext32(ptr %a)  {
 ; CHECK-LABEL: LdOffset_i16_sext32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    add x8, x0, #506, lsl #12 // =2072576
-; CHECK-NEXT:    ldrsh w0, [x8, #7408]
+; CHECK-NEXT:    mov w8, #48368 // =0xbcf0
+; CHECK-NEXT:    movk w8, #31, lsl #16
+; CHECK-NEXT:    ldrsh w0, [x0, x8]
 ; CHECK-NEXT:    ret
   %arrayidx = getelementptr inbounds i16, ptr %a, i64 1039992
   %val = load i16, ptr %arrayidx, align 2
@@ -316,8 +321,9 @@ define i32 @LdOffset_i16_sext32(ptr %a)  {
 define i64 @LdOffset_i16_zext64(ptr %a)  {
 ; CHECK-LABEL: LdOffset_i16_zext64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    add x8, x0, #506, lsl #12 // =2072576
-; CHECK-NEXT:    ldrh w0, [x8, #7408]
+; CHECK-NEXT:    mov w8, #48368 // =0xbcf0
+; CHECK-NEXT:    movk w8, #31, lsl #16
+; CHECK-NEXT:    ldrh w0, [x0, x8]
 ; CHECK-NEXT:    ret
   %arrayidx = getelementptr inbounds i16, ptr %a, i64 1039992
   %val = load i16, ptr %arrayidx, align 2
@@ -329,8 +335,9 @@ define i64 @LdOffset_i16_zext64(ptr %a)  {
 define i64 @LdOffset_i16_sext64(ptr %a)  {
 ; CHECK-LABEL: LdOffset_i16_sext64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    add x8, x0, #506, lsl #12 // =2072576
-; CHECK-NEXT:    ldrsh x0, [x8, #7408]
+; CHECK-NEXT:    mov w8, #48368 // =0xbcf0
+; CHECK-NEXT:    movk w8, #31, lsl #16
+; CHECK-NEXT:    ldrsh x0, [x0, x8]
 ; CHECK-NEXT:    ret
   %arrayidx = getelementptr inbounds i16, ptr %a, i64 1039992
   %val = load i16, ptr %arrayidx, align 2
@@ -342,8 +349,9 @@ define i64 @LdOffset_i16_sext64(ptr %a)  {
 define i32 @LdOffset_i32(ptr %a)  {
 ; CHECK-LABEL: LdOffset_i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    add x8, x0, #1012, lsl #12 // =4145152
-; CHECK-NEXT:    ldr w0, [x8, #14816]
+; CHECK-NEXT:    mov w8, #31200 // =0x79e0
+; CHECK-NEXT:    movk w8, #63, lsl #16
+; CHECK-NEXT:    ldr w0, [x0, x8]
 ; CHECK-NEXT:    ret
   %arrayidx = getelementptr inbounds i32, ptr %a, i64 1039992
   %val = load i32, ptr %arrayidx, align 4
@@ -354,8 +362,9 @@ define i32 @LdOffset_i32(ptr %a)  {
 define i64 @LdOffset_i32_zext64(ptr %a)  {
 ; CHECK-LABEL: LdOffset_i32_zext64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    add x8, x0, #1012, lsl #12 // =4145152
-; CHECK-NEXT:    ldr w0, [x8, #14816]
+; CHECK-NEXT:    mov w8, #31200 // =0x79e0
+; CHECK-NEXT:    movk w8, #63, lsl #16
+; CHECK-NEXT:    ldr w0, [x0, x8]
 ; CHECK-NEXT:    ret
   %arrayidx = getelementptr inbounds i32, ptr %a, i64 1039992
   %val = load i32, ptr %arrayidx, align 2
@@ -367,8 +376,9 @@ define i64 @LdOffset_i32_zext64(ptr %a)  {
 define i64 @LdOffset_i32_sext64(ptr %a)  {
 ; CHECK-LABEL: LdOffset_i32_sext64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    add x8, x0, #1012, lsl #12 // =4145152
-; CHECK-NEXT:    ldrsw x0, [x8, #14816]
+; CHECK-NEXT:    mov w8, #31200 // =0x79e0
+; CHECK-NEXT:    movk w8, #63, lsl #16
+; CHECK-NEXT:    ldrsw x0, [x0, x8]
 ; CHECK-NEXT:    ret
   %arrayidx = getelementptr inbounds i32, ptr %a, i64 1039992
   %val = load i32, ptr %arrayidx, align 2
@@ -380,8 +390,9 @@ define i64 @LdOffset_i32_sext64(ptr %a)  {
 define i64 @LdOffset_i64(ptr %a)  {
 ; CHECK-LABEL: LdOffset_i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    add x8, x0, #2024, lsl #12 // =8290304
-; CHECK-NEXT:    ldr x0, [x8, #29632]
+; CHECK-NEXT:    mov w8, #62400 // =0xf3c0
+; CHECK-NEXT:    movk w8, #126, lsl #16
+; CHECK-NEXT:    ldr x0, [x0, x8]
 ; CHECK-NEXT:    ret
   %arrayidx = getelementptr inbounds i64, ptr %a, i64 1039992
   %val = load i64, ptr %arrayidx, align 4
@@ -392,8 +403,9 @@ define i64 @LdOffset_i64(ptr %a)  {
 define <2 x i32> @LdOffset_v2i32(ptr %a)  {
 ; CHECK-LABEL: LdOffset_v2i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    add x8, x0, #2024, lsl #12 // =8290304
-; CHECK-NEXT:    ldr d0, [x8, #29632]
+; CHECK-NEXT:    mov w8, #62400 // =0xf3c0
+; CHECK-NEXT:    movk w8, #126, lsl #16
+; CHECK-NEXT:    ldr d0, [x0, x8]
 ; CHECK-NEXT:    ret
   %arrayidx = getelementptr inbounds <2 x i32>, ptr %a, i64 1039992
   %val = load <2 x i32>, ptr %arrayidx, align 4
@@ -404,8 +416,9 @@ define <2 x i32> @LdOffset_v2i32(ptr %a)  {
 define <2 x i64> @LdOffset_v2i64(ptr %a)  {
 ; CHECK-LABEL: LdOffset_v2i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    add x8, x0, #4048, lsl #12 // =16580608
-; CHECK-NEXT:    ldr q0, [x8, #59264]
+; CHECK-NEXT:    mov w8, #59264 // =0xe780
+; CHECK-NEXT:    movk w8, #253, lsl #16
+; CHECK-NEXT:    ldr q0, [x0, x8]
 ; CHECK-NEXT:    ret
   %arrayidx = getelementptr inbounds <2 x i64>, ptr %a, i64 1039992
   %val = load <2 x i64>, ptr %arrayidx, align 4
@@ -416,8 +429,9 @@ define <2 x i64> @LdOffset_v2i64(ptr %a)  {
 define double @LdOffset_i8_f64(ptr %a)  {
 ; CHECK-LABEL: LdOffset_i8_f64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    add x8, x0, #253, lsl #12 // =1036288
-; CHECK-NEXT:    ldrsb w8, [x8, #3704]
+; CHECK-NEXT:    mov w8, #56952 // =0xde78
+; CHECK-NEXT:    movk w8, #15, lsl #16
+; CHECK-NEXT:    ldrsb w8, [x0, x8]
 ; CHECK-NEXT:    scvtf d0, w8
 ; CHECK-NEXT:    ret
   %arrayidx = getelementptr inbounds i8, ptr %a, i64 1039992
@@ -430,8 +444,9 @@ define double @LdOffset_i8_f64(ptr %a)  {
 define double @LdOffset_i16_f64(ptr %a)  {
 ; CHECK-LABEL: LdOffset_i16_f64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    add x8, x0, #506, lsl #12 // =2072576
-; CHECK-NEXT:    ldrsh w8, [x8, #7408]
+; CHECK-NEXT:    mov w8, #48368 // =0xbcf0
+; CHECK-NEXT:    movk w8, #31, lsl #16
+; CHECK-NEXT:    ldrsh w8, [x0, x8]
 ; CHECK-NEXT:    scvtf d0, w8
 ; CHECK-NEXT:    ret
   %arrayidx = getelementptr inbounds i16, ptr %a, i64 1039992
@@ -444,8 +459,9 @@ define double @LdOffset_i16_f64(ptr %a)  {
 define double @LdOffset_i32_f64(ptr %a)  {
 ; CHECK-LABEL: LdOffset_i32_f64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    add x8, x0, #1012, lsl #12 // =4145152
-; CHECK-NEXT:    ldr s0, [x8, #14816]
+; CHECK-NEXT:    mov w8, #31200 // =0x79e0
+; CHECK-NEXT:    movk w8, #63, lsl #16
+; CHECK-NEXT:    ldr s0, [x0, x8]
 ; CHECK-NEXT:    ucvtf d0, d0
 ; CHECK-NEXT:    ret
   %arrayidx = getelementptr inbounds i32, ptr %a, i64 1039992
@@ -458,8 +474,9 @@ define double @LdOffset_i32_f64(ptr %a)  {
 define double @LdOffset_i64_f64(ptr %a)  {
 ; CHECK-LABEL: LdOffset_i64_f64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    add x8, x0, #2024, lsl #12 // =8290304
-; CHECK-NEXT:    ldr d0, [x8, #29632]
+; CHECK-NEXT:    mov w8, #62400 // =0xf3c0
+; CHECK-NEXT:    movk w8, #126, lsl #16
+; CHECK-NEXT:    ldr d0, [x0, x8]
 ; CHECK-NEXT:    scvtf d0, d0
 ; CHECK-NEXT:    ret
   %arrayidx = getelementptr inbounds i64, ptr %a, i64 1039992

From 5ef2456a438578b0783241a2744efc62d47e5ab6 Mon Sep 17 00:00:00 2001
From: PeterChou1 <peter.chou@mail.utoronto.ca>
Date: Fri, 16 Aug 2024 18:37:17 -0400
Subject: [PATCH 244/441] [clang-doc] add support for enums comments in html
 generation (#101282)

Part of https://github.com/llvm/llvm-project/issues/101129

This patch adds support for attaching comments to enums for HTML in
clang-doc. It changes the enum generation to table tags where as
perviously we're using lists which is more in line with what other doc
generators are doing. It also gives clang-doc the ability to show user
specified enum values
---
 clang-tools-extra/clang-doc/BitcodeReader.cpp |   4 +
 clang-tools-extra/clang-doc/BitcodeWriter.cpp |   2 +
 clang-tools-extra/clang-doc/HTMLGenerator.cpp |  87 +++++++++++---
 .../clang-doc/Representation.cpp              |   2 +-
 clang-tools-extra/clang-doc/Representation.h  |   2 +
 clang-tools-extra/clang-doc/Serialize.cpp     |  16 ++-
 clang-tools-extra/test/clang-doc/enum.cpp     | 113 +++++++++++++-----
 .../unittests/clang-doc/HTMLGeneratorTest.cpp |  33 ++++-
 8 files changed, 201 insertions(+), 58 deletions(-)

diff --git a/clang-tools-extra/clang-doc/BitcodeReader.cpp b/clang-tools-extra/clang-doc/BitcodeReader.cpp
index bfb04e7407b3804..1f2fb0a8b2b855d 100644
--- a/clang-tools-extra/clang-doc/BitcodeReader.cpp
+++ b/clang-tools-extra/clang-doc/BitcodeReader.cpp
@@ -415,6 +415,10 @@ template <> llvm::Expected<CommentInfo *> getCommentInfo(TypedefInfo *I) {
   return &I->Description.emplace_back();
 }
 
+template <> llvm::Expected<CommentInfo *> getCommentInfo(EnumValueInfo *I) {
+  return &I->Description.emplace_back();
+}
+
 template <> llvm::Expected<CommentInfo *> getCommentInfo(CommentInfo *I) {
   I->Children.emplace_back(std::make_unique<CommentInfo>());
   return I->Children.back().get();
diff --git a/clang-tools-extra/clang-doc/BitcodeWriter.cpp b/clang-tools-extra/clang-doc/BitcodeWriter.cpp
index 7e5a11783d303a6..06f30f76e33d8ce 100644
--- a/clang-tools-extra/clang-doc/BitcodeWriter.cpp
+++ b/clang-tools-extra/clang-doc/BitcodeWriter.cpp
@@ -536,6 +536,8 @@ void ClangDocBitcodeWriter::emitBlock(const EnumValueInfo &I) {
   emitRecord(I.Name, ENUM_VALUE_NAME);
   emitRecord(I.Value, ENUM_VALUE_VALUE);
   emitRecord(I.ValueExpr, ENUM_VALUE_EXPR);
+  for (const auto &CI : I.Description)
+    emitBlock(CI);
 }
 
 void ClangDocBitcodeWriter::emitBlock(const RecordInfo &I) {
diff --git a/clang-tools-extra/clang-doc/HTMLGenerator.cpp b/clang-tools-extra/clang-doc/HTMLGenerator.cpp
index 21ee417da6028a8..068eec6599ead86 100644
--- a/clang-tools-extra/clang-doc/HTMLGenerator.cpp
+++ b/clang-tools-extra/clang-doc/HTMLGenerator.cpp
@@ -48,6 +48,12 @@ class HTMLTag {
     TAG_SPAN,
     TAG_TITLE,
     TAG_UL,
+    TAG_TABLE,
+    TAG_THEAD,
+    TAG_TBODY,
+    TAG_TR,
+    TAG_TD,
+    TAG_TH
   };
 
   HTMLTag() = default;
@@ -133,6 +139,12 @@ bool HTMLTag::isSelfClosing() const {
   case HTMLTag::TAG_SPAN:
   case HTMLTag::TAG_TITLE:
   case HTMLTag::TAG_UL:
+  case HTMLTag::TAG_TABLE:
+  case HTMLTag::TAG_THEAD:
+  case HTMLTag::TAG_TBODY:
+  case HTMLTag::TAG_TR:
+  case HTMLTag::TAG_TD:
+  case HTMLTag::TAG_TH:
     return false;
   }
   llvm_unreachable("Unhandled HTMLTag::TagType");
@@ -174,6 +186,18 @@ StringRef HTMLTag::toString() const {
     return "title";
   case HTMLTag::TAG_UL:
     return "ul";
+  case HTMLTag::TAG_TABLE:
+    return "table";
+  case HTMLTag::TAG_THEAD:
+    return "thead";
+  case HTMLTag::TAG_TBODY:
+    return "tbody";
+  case HTMLTag::TAG_TR:
+    return "tr";
+  case HTMLTag::TAG_TD:
+    return "td";
+  case HTMLTag::TAG_TH:
+    return "th";
   }
   llvm_unreachable("Unhandled HTMLTag::TagType");
 }
@@ -377,10 +401,27 @@ genEnumMembersBlock(const llvm::SmallVector<EnumValueInfo, 4> &Members) {
   if (Members.empty())
     return nullptr;
 
-  auto List = std::make_unique<TagNode>(HTMLTag::TAG_UL);
-  for (const auto &M : Members)
-    List->Children.emplace_back(
-        std::make_unique<TagNode>(HTMLTag::TAG_LI, M.Name));
+  auto List = std::make_unique<TagNode>(HTMLTag::TAG_TBODY);
+
+  for (const auto &M : Members) {
+    auto TRNode = std::make_unique<TagNode>(HTMLTag::TAG_TR);
+    TRNode->Children.emplace_back(
+        std::make_unique<TagNode>(HTMLTag::TAG_TD, M.Name));
+    // Use user supplied value if it exists, otherwise use the value
+    if (!M.ValueExpr.empty()) {
+      TRNode->Children.emplace_back(
+          std::make_unique<TagNode>(HTMLTag::TAG_TD, M.ValueExpr));
+    } else {
+      TRNode->Children.emplace_back(
+          std::make_unique<TagNode>(HTMLTag::TAG_TD, M.Value));
+    }
+    if (!M.Description.empty()) {
+      auto TD = std::make_unique<TagNode>(HTMLTag::TAG_TD);
+      TD->Children.emplace_back(genHTML(M.Description));
+      TRNode->Children.emplace_back(std::move(TD));
+    }
+    List->Children.emplace_back(std::move(TRNode));
+  }
   return List;
 }
 
@@ -624,7 +665,7 @@ static std::unique_ptr<HTMLNode> genHTML(const CommentInfo &I) {
     }
     return std::move(FullComment);
   }
- 
+
   if (I.Kind == "ParagraphComment") {
     auto ParagraphComment = std::make_unique<TagNode>(HTMLTag::TAG_P);
     for (const auto &Child : I.Children) {
@@ -658,22 +699,36 @@ static std::vector<std::unique_ptr<TagNode>>
 genHTML(const EnumInfo &I, const ClangDocContext &CDCtx) {
   std::vector<std::unique_ptr<TagNode>> Out;
   std::string EnumType = I.Scoped ? "enum class " : "enum ";
-
-  Out.emplace_back(
-      std::make_unique<TagNode>(HTMLTag::TAG_H3, EnumType + I.Name));
-  Out.back()->Attributes.emplace_back("id",
-                                      llvm::toHex(llvm::toStringRef(I.USR)));
-
-  std::unique_ptr<TagNode> Node = genEnumMembersBlock(I.Members);
-  if (Node)
-    Out.emplace_back(std::move(Node));
+  // Determine if enum members have comments attached
+  bool HasComments = std::any_of(
+      I.Members.begin(), I.Members.end(),
+      [](const EnumValueInfo &M) { return !M.Description.empty(); });
+  std::unique_ptr<TagNode> Table =
+      std::make_unique<TagNode>(HTMLTag::TAG_TABLE);
+  std::unique_ptr<TagNode> THead =
+      std::make_unique<TagNode>(HTMLTag::TAG_THEAD);
+  std::unique_ptr<TagNode> TRow = std::make_unique<TagNode>(HTMLTag::TAG_TR);
+  std::unique_ptr<TagNode> TD =
+      std::make_unique<TagNode>(HTMLTag::TAG_TH, EnumType + I.Name);
+  // Span 3 columns if enum has comments
+  TD->Attributes.emplace_back("colspan", HasComments ? "3" : "2");
+
+  Table->Attributes.emplace_back("id", llvm::toHex(llvm::toStringRef(I.USR)));
+  TRow->Children.emplace_back(std::move(TD));
+  THead->Children.emplace_back(std::move(TRow));
+  Table->Children.emplace_back(std::move(THead));
+
+  if (std::unique_ptr<TagNode> Node = genEnumMembersBlock(I.Members))
+    Table->Children.emplace_back(std::move(Node));
+
+  Out.emplace_back(std::move(Table));
 
   if (I.DefLoc) {
     if (!CDCtx.RepositoryUrl)
       Out.emplace_back(writeFileDefinition(*I.DefLoc));
     else
-      Out.emplace_back(writeFileDefinition(
-          *I.DefLoc, StringRef{*CDCtx.RepositoryUrl}));
+      Out.emplace_back(
+          writeFileDefinition(*I.DefLoc, StringRef{*CDCtx.RepositoryUrl}));
   }
 
   std::string Description;
diff --git a/clang-tools-extra/clang-doc/Representation.cpp b/clang-tools-extra/clang-doc/Representation.cpp
index 6f140910b321fb6..4da93b24c131fdf 100644
--- a/clang-tools-extra/clang-doc/Representation.cpp
+++ b/clang-tools-extra/clang-doc/Representation.cpp
@@ -221,7 +221,7 @@ void SymbolInfo::merge(SymbolInfo &&Other) {
 }
 
 NamespaceInfo::NamespaceInfo(SymbolID USR, StringRef Name, StringRef Path)
-      : Info(InfoType::IT_namespace, USR, Name, Path) {}
+    : Info(InfoType::IT_namespace, USR, Name, Path) {}
 
 void NamespaceInfo::merge(NamespaceInfo &&Other) {
   assert(mergeable(Other));
diff --git a/clang-tools-extra/clang-doc/Representation.h b/clang-tools-extra/clang-doc/Representation.h
index 0695197cf3ac959..bb0c534af7b740a 100644
--- a/clang-tools-extra/clang-doc/Representation.h
+++ b/clang-tools-extra/clang-doc/Representation.h
@@ -455,6 +455,8 @@ struct EnumValueInfo {
   // Stores the user-supplied initialization expression for this enumeration
   // constant. This will be empty for implicit enumeration values.
   SmallString<16> ValueExpr;
+
+  std::vector<CommentInfo> Description; /// Comment description of this field.
 };
 
 // TODO: Expand to allow for documenting templating.
diff --git a/clang-tools-extra/clang-doc/Serialize.cpp b/clang-tools-extra/clang-doc/Serialize.cpp
index 3b074d849e8a9cf..b9db78cf7d688fb 100644
--- a/clang-tools-extra/clang-doc/Serialize.cpp
+++ b/clang-tools-extra/clang-doc/Serialize.cpp
@@ -394,10 +394,20 @@ static void parseEnumerators(EnumInfo &I, const EnumDecl *D) {
     std::string ValueExpr;
     if (const Expr *InitExpr = E->getInitExpr())
       ValueExpr = getSourceCode(D, InitExpr->getSourceRange());
-
     SmallString<16> ValueStr;
     E->getInitVal().toString(ValueStr);
-    I.Members.emplace_back(E->getNameAsString(), ValueStr, ValueExpr);
+    I.Members.emplace_back(E->getNameAsString(), ValueStr.str(), ValueExpr);
+    ASTContext &Context = E->getASTContext();
+    if (RawComment *Comment =
+            E->getASTContext().getRawCommentForDeclNoCache(E)) {
+      CommentInfo CInfo;
+      Comment->setAttached();
+      if (comments::FullComment *Fc = Comment->parse(Context, nullptr, E)) {
+        EnumValueInfo &Member = I.Members.back();
+        Member.Description.emplace_back();
+        parseFullComment(Fc, Member.Description.back());
+      }
+    }
   }
 }
 
@@ -566,7 +576,7 @@ static void populateMemberTypeInfo(MemberTypeInfo &I, const FieldDecl *D) {
     return;
 
   Comment->setAttached();
-  if (comments::FullComment* fc = Comment->parse(Context, nullptr, D)) {
+  if (comments::FullComment *fc = Comment->parse(Context, nullptr, D)) {
     I.Description.emplace_back();
     parseFullComment(fc, I.Description.back());
   }
diff --git a/clang-tools-extra/test/clang-doc/enum.cpp b/clang-tools-extra/test/clang-doc/enum.cpp
index e559940a31de69a..ef768e33b456680 100644
--- a/clang-tools-extra/test/clang-doc/enum.cpp
+++ b/clang-tools-extra/test/clang-doc/enum.cpp
@@ -21,9 +21,9 @@
 enum Color {
 // MD-INDEX-LINE: *Defined at {{.*}}clang-tools-extra{{[\/]}}test{{[\/]}}clang-doc{{[\/]}}enum.cpp#[[@LINE-1]]*
 // HTML-INDEX-LINE: <p>Defined at line [[@LINE-2]] of file {{.*}}clang-tools-extra{{[\/]}}test{{[\/]}}clang-doc{{[\/]}}enum.cpp</p>
-  Red, ///< Red
-  Green, ///< Green
-  Blue ///< Blue
+  Red, ///< Comment 1
+  Green, ///< Comment 2
+  Blue ///< Comment 3
 };
 
 // MD-INDEX: ## Enums
@@ -34,11 +34,16 @@ enum Color {
 // MD-INDEX: | Blue |
 // MD-INDEX: **brief** For specifying RGB colors
 
-// HTML-INDEX: <h2 id="Enums">Enums</h2>
-// HTML-INDEX: <h3 id="{{([0-9A-F]{40})}}">enum Color</h3>
-// HTML-INDEX: <li>Red</li>
-// HTML-INDEX: <li>Green</li>
-// HTML-INDEX: <li>Blue</li>
+// HTML-INDEX: <th colspan="3">enum Color</th>
+// HTML-INDEX: <td>Red</td>
+// HTML-INDEX: <td>0</td>
+// HTML-INDEX: <p> Comment 1</p>
+// HTML-INDEX: <td>Green</td>
+// HTML-INDEX: <td>1</td>
+// HTML-INDEX: <p> Comment 2</p>
+// HTML-INDEX: <td>Blue</td>
+// HTML-INDEX: <td>2</td>
+// HTML-INDEX: <p> Comment 3</p>
 
 /**
  * @brief Shape Types
@@ -46,11 +51,12 @@ enum Color {
 enum class Shapes {
 // MD-INDEX-LINE: *Defined at {{.*}}clang-tools-extra{{[\/]}}test{{[\/]}}clang-doc{{[\/]}}enum.cpp#[[@LINE-1]]*
 // HTML-INDEX-LINE: <p>Defined at line [[@LINE-2]] of file {{.*}}clang-tools-extra{{[\/]}}test{{[\/]}}clang-doc{{[\/]}}enum.cpp</p>
-  /// Circle
+
+  /// Comment 1
   Circle,
-  /// Rectangle
+  /// Comment 2
   Rectangle,
-  /// Triangle
+  /// Comment 3
   Triangle
 };
 // MD-INDEX: | enum class Shapes |
@@ -60,10 +66,17 @@ enum class Shapes {
 // MD-INDEX: | Triangle |
 // MD-INDEX: **brief** Shape Types
 
-// HTML-INDEX: <h3 id="{{([0-9A-F]{40})}}">enum class Shapes</h3>
-// HTML-INDEX: <li>Circle</li>
-// HTML-INDEX: <li>Rectangle</li>
-// HTML-INDEX: <li>Triangle</li>
+// HTML-INDEX: <th colspan="3">enum class Shapes</th>
+// HTML-INDEX: <td>Circle</td>
+// HTML-INDEX: <td>0</td>
+// HTML-INDEX: <p> Comment 1</p>
+// HTML-INDEX: <td>Rectangle</td>
+// HTML-INDEX: <td>1</td>
+// HTML-INDEX: <p> Comment 2</p>
+// HTML-INDEX: <td>Triangle</td>
+// HTML-INDEX: <td>2</td>
+// HTML-INDEX: <p> Comment 3</p>
+
 
 
 class Animals {
@@ -76,18 +89,25 @@ class Animals {
       enum AnimalType {
 // MD-ANIMAL-LINE: *Defined at {{.*}}clang-tools-extra{{[\/]}}test{{[\/]}}clang-doc{{[\/]}}enum.cpp#[[@LINE-1]]*
 // HTML-ANIMAL-LINE: <p>Defined at line [[@LINE-2]] of file {{.*}}clang-tools-extra{{[\/]}}test{{[\/]}}clang-doc{{[\/]}}enum.cpp</p>
-          Dog, /// Man's best friend
-          Cat, /// Man's other best friend
-          Iguana /// A lizard
+          Dog, ///< Man's best friend
+          Cat, ///< Man's other best friend
+          Iguana ///< A lizard
       };
 };
 
 // HTML-ANIMAL: <h1>class Animals</h1>
 // HTML-ANIMAL: <h2 id="Enums">Enums</h2>
-// HTML-ANIMAL: <h3 id="{{([0-9A-F]{40})}}">enum AnimalType</h3>
-// HTML-ANIMAL: <li>Dog</li>
-// HTML-ANIMAL: <li>Cat</li>
-// HTML-ANIMAL: <li>Iguana</li>
+// HTML-ANIMAL: <th colspan="3">enum AnimalType</th>
+// HTML-ANIMAL: <td>Dog</td>
+// HTML-ANIMAL: <td>0</td>
+// HTML-ANIMAL: <p> Man&apos;s best friend</p>
+// HTML-ANIMAL: <td>Cat</td>
+// HTML-ANIMAL: <td>1</td>
+// HTML-ANIMAL: <p> Man&apos;s other best friend</p>
+// HTML-ANIMAL: <td>Iguana</td>
+// HTML-ANIMAL: <td>2</td>
+// HTML-ANIMAL: <p> A lizard</p>
+
 
 // MD-ANIMAL: # class Animals
 // MD-ANIMAL: ## Enums
@@ -106,10 +126,11 @@ namespace Vehicles {
     enum Car {
 // MD-VEHICLES-LINE: *Defined at {{.*}}clang-tools-extra{{[\/]}}test{{[\/]}}clang-doc{{[\/]}}enum.cpp#[[@LINE-1]]*
 // HTML-VEHICLES-LINE: <p>Defined at line [[@LINE-2]] of file {{.*}}clang-tools-extra{{[\/]}}test{{[\/]}}clang-doc{{[\/]}}enum.cpp</p>
-       Sedan, /// Sedan
-       SUV, /// SUV
-       Pickup, /// Pickup
-       Hatchback /// Hatchback
+
+       Sedan, ///< Comment 1
+       SUV, ///< Comment 2
+       Pickup, ///< Comment 3
+       Hatchback ///< Comment 4
     };
 }
 
@@ -124,9 +145,37 @@ namespace Vehicles {
 // MD-VEHICLES: **brief** specify type of car
 
 // HTML-VEHICLES: <h1>namespace Vehicles</h1>
-// HTML-VEHICLES: <h2 id="Enums">Enums</h2>
-// HTML-VEHICLES: <h3 id="{{([0-9A-F]{40})}}">enum Car</h3>
-// HTML-VEHICLES: <li>Sedan</li>
-// HTML-VEHICLES: <li>SUV</li>
-// HTML-VEHICLES: <li>Pickup</li>
-// HTML-VEHICLES: <li>Hatchback</li>
\ No newline at end of file
+// HTML-VEHICLES: <th colspan="3">enum Car</th>
+// HTML-VEHICLES: <td>Sedan</td>
+// HTML-VEHICLES: <td>0</td>
+// HTML-VEHICLES: <p> Comment 1</p>
+// HTML-VEHICLES: <td>SUV</td>
+// HTML-VEHICLES: <td>1</td>
+// HTML-VEHICLES: <p> Comment 2</p>
+// HTML-VEHICLES: <td>Pickup</td>
+// HTML-VEHICLES: <td>2</td>
+// HTML-VEHICLES: <p> Comment 3</p>
+// HTML-VEHICLES: <td>Hatchback</td>
+// HTML-VEHICLES: <td>3</td>
+// HTML-VEHICLES: <p> Comment 4</p>
+
+
+enum ColorUserSpecified {
+  RedUserSpecified = 'A',
+  GreenUserSpecified = 2,
+  BlueUserSpecified = 'C'
+};
+
+// MD-INDEX: | enum ColorUserSpecified |
+// MD-INDEX: --
+// MD-INDEX: | RedUserSpecified |
+// MD-INDEX: | GreenUserSpecified |
+// MD-INDEX: | BlueUserSpecified |
+
+// HTML-INDEX: <th colspan="2">enum ColorUserSpecified</th>
+// HTML-INDEX: <td>RedUserSpecified</td>
+// HTML-INDEX: <td>&apos;A&apos;</td>
+// HTML-INDEX: <td>GreenUserSpecified</td>
+// HTML-INDEX: <td>2</td>
+// HTML-INDEX: <td>BlueUserSpecified</td>
+// HTML-INDEX: <td>&apos;C&apos;</td>
diff --git a/clang-tools-extra/unittests/clang-doc/HTMLGeneratorTest.cpp b/clang-tools-extra/unittests/clang-doc/HTMLGeneratorTest.cpp
index bd031282b042abb..dfd31e657871451 100644
--- a/clang-tools-extra/unittests/clang-doc/HTMLGeneratorTest.cpp
+++ b/clang-tools-extra/unittests/clang-doc/HTMLGeneratorTest.cpp
@@ -92,7 +92,13 @@ TEST(HTMLGeneratorTest, emitNamespaceHTML) {
     </div>
     <h2 id="Enums">Enums</h2>
     <div>
-      <h3 id="0000000000000000000000000000000000000000">enum OneEnum</h3>
+      <table id="0000000000000000000000000000000000000000">
+        <thead>
+          <tr>
+            <th colspan="2">enum OneEnum</th>
+          </tr>
+        </thead>
+      </table>
     </div>
   </div>
   <div id="sidebar-right" class="col-xs-6 col-sm-6 col-md-2 sidebar sidebar-offcanvas-right">
@@ -214,7 +220,13 @@ TEST(HTMLGeneratorTest, emitRecordHTML) {
     </div>
     <h2 id="Enums">Enums</h2>
     <div>
-      <h3 id="0000000000000000000000000000000000000000">enum OneEnum</h3>
+      <table id="0000000000000000000000000000000000000000">
+        <thead>
+          <tr>
+            <th colspan="2">enum OneEnum</th>
+          </tr>
+        </thead>
+      </table>
     </div>
   </div>
   <div id="sidebar-right" class="col-xs-6 col-sm-6 col-md-2 sidebar sidebar-offcanvas-right">
@@ -348,10 +360,19 @@ TEST(HTMLGeneratorTest, emitEnumHTML) {
 <main>
   <div id="sidebar-left" path="" class="col-xs-6 col-sm-3 col-md-2 sidebar sidebar-offcanvas-left"></div>
   <div id="main-content" class="col-xs-12 col-sm-9 col-md-8 main-content">
-    <h3 id="0000000000000000000000000000000000000000">enum class e</h3>
-    <ul>
-      <li>X</li>
-    </ul>
+    <table id="0000000000000000000000000000000000000000">
+      <thead>
+        <tr>
+          <th colspan="2">enum class e</th>
+        </tr>
+      </thead>
+      <tbody>
+        <tr>
+          <td>X</td>
+          <td>0</td>
+        </tr>
+      </tbody>
+    </table>
     <p>
       Defined at line 
       <a href="https://www.repository.com/test.cpp#10">10</a>

From 085b04b3a2e941a25cbf713cc3969c1254469289 Mon Sep 17 00:00:00 2001
From: Vitaly Buka <vitalybuka@google.com>
Date: Fri, 16 Aug 2024 15:41:08 -0700
Subject: [PATCH 245/441] [asan] Pre-commit test with global constructor
 without any global (#104620)

In this test `@initializer()` can access globals
outside of the module, but Asan does nothing to
detect that.
---
 .../initialization-bug-no-global.cpp          | 26 ++++++++++++++
 .../instrument_initializer_without_global.ll  | 34 +++++++++++++++++++
 2 files changed, 60 insertions(+)
 create mode 100644 compiler-rt/test/asan/TestCases/initialization-bug-no-global.cpp
 create mode 100644 llvm/test/Instrumentation/AddressSanitizer/instrument_initializer_without_global.ll

diff --git a/compiler-rt/test/asan/TestCases/initialization-bug-no-global.cpp b/compiler-rt/test/asan/TestCases/initialization-bug-no-global.cpp
new file mode 100644
index 000000000000000..50698b2a7d06fc4
--- /dev/null
+++ b/compiler-rt/test/asan/TestCases/initialization-bug-no-global.cpp
@@ -0,0 +1,26 @@
+// RUN: %clangxx_asan %min_macos_deployment_target=10.11 -O0 %s %p/Helpers/initialization-bug-extra.cpp -o %t
+// RUN: %env_asan_opts=check_initialization_order=true:strict_init_order=true not %run %t 2>&1 | FileCheck %s
+
+// Not implemented.
+// XFAIL: *
+
+// Do not test with optimization -- the error may be optimized away.
+
+// FIXME: https://code.google.com/p/address-sanitizer/issues/detail?id=186
+// XFAIL: target={{.*windows-msvc.*}}
+
+#include <stdio.h>
+
+extern int y;
+
+void __attribute__((constructor)) ctor() {
+  printf("%d\n", y);
+  // CHECK: AddressSanitizer: initialization-order-fiasco
+}
+
+int main() {
+  // ASan should have caused an exit before main runs.
+  printf("PASS\n");
+  // CHECK-NOT: PASS
+  return 0;
+}
diff --git a/llvm/test/Instrumentation/AddressSanitizer/instrument_initializer_without_global.ll b/llvm/test/Instrumentation/AddressSanitizer/instrument_initializer_without_global.ll
new file mode 100644
index 000000000000000..c8a6541bacfdfa1
--- /dev/null
+++ b/llvm/test/Instrumentation/AddressSanitizer/instrument_initializer_without_global.ll
@@ -0,0 +1,34 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals all --global-value-regex ".*global_ctors.*" --version 5
+; RUN: opt < %s -passes=asan -S | FileCheck %s
+; RUN: opt < %s -passes=asan -S -asan-initialization-order=0 | FileCheck %s --check-prefixes=NOINIT
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+@llvm.global_ctors = appending global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 65535, ptr @__late_ctor, ptr null }]
+
+declare void @initializer() uwtable;
+
+;.
+; CHECK: @llvm.global_ctors = appending global [2 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 65535, ptr @__late_ctor, ptr null }, { i32, ptr, ptr } { i32 1, ptr @asan.module_ctor, ptr @asan.module_ctor }]
+;.
+; NOINIT: @llvm.global_ctors = appending global [2 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 65535, ptr @__late_ctor, ptr null }, { i32, ptr, ptr } { i32 1, ptr @asan.module_ctor, ptr @asan.module_ctor }]
+;.
+define internal void @__late_ctor() sanitize_address section ".text.startup" {
+; CHECK-LABEL: define internal void @__late_ctor(
+; CHECK-SAME: ) #[[ATTR1:[0-9]+]] section ".text.startup" {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    call void @initializer()
+; CHECK-NEXT:    ret void
+;
+; NOINIT-LABEL: define internal void @__late_ctor(
+; NOINIT-SAME: ) #[[ATTR1:[0-9]+]] section ".text.startup" {
+; NOINIT-NEXT:  [[ENTRY:.*:]]
+; NOINIT-NEXT:    call void @initializer()
+; NOINIT-NEXT:    ret void
+;
+entry:
+
+  call void @initializer()
+  ret void
+}

From e169cc162adbe89d498e774bccf4e228af989849 Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Fri, 16 Aug 2024 18:36:27 -0500
Subject: [PATCH 246/441] [Clang] Fix sema checks thinking kernels aren't
 kernels (#104460)

Summary:
Currently we have some sema checks to make sure users don't apply
kernel-only attributes to non-kernel functions. However, this currently
did not correctly check for bare NVPTX / AMDGPU kernel attributes,
making it impossible to use them at all w/o CUDA enabled. This patch
fixes that by checking for the calling convention / attributes directly.
---
 clang/lib/Sema/SemaDeclAttr.cpp               | 44 +++++++++++--------
 .../amdgpu-kernel-arg-pointer-type.cpp        |  5 +++
 2 files changed, 31 insertions(+), 18 deletions(-)

diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp
index 3b5e984f4ee7735..73d11ac972b020b 100644
--- a/clang/lib/Sema/SemaDeclAttr.cpp
+++ b/clang/lib/Sema/SemaDeclAttr.cpp
@@ -7123,6 +7123,13 @@ ProcessDeclAttribute(Sema &S, Scope *scope, Decl *D, const ParsedAttr &AL,
   }
 }
 
+static bool isKernelDecl(Decl *D) {
+  const FunctionType *FnTy = D->getFunctionType();
+  return D->hasAttr<OpenCLKernelAttr>() ||
+         (FnTy && FnTy->getCallConv() == CallingConv::CC_AMDGPUKernelCall) ||
+         D->hasAttr<CUDAGlobalAttr>() || D->getAttr<NVPTXKernelAttr>();
+}
+
 void Sema::ProcessDeclAttributeList(
     Scope *S, Decl *D, const ParsedAttributesView &AttrList,
     const ProcessDeclAttributeOptions &Options) {
@@ -7163,24 +7170,25 @@ void Sema::ProcessDeclAttributeList(
     } else if (const auto *A = D->getAttr<OpenCLIntelReqdSubGroupSizeAttr>()) {
       Diag(D->getLocation(), diag::err_opencl_kernel_attr) << A;
       D->setInvalidDecl();
-    } else if (!D->hasAttr<CUDAGlobalAttr>()) {
-      if (const auto *A = D->getAttr<AMDGPUFlatWorkGroupSizeAttr>()) {
-        Diag(D->getLocation(), diag::err_attribute_wrong_decl_type)
-            << A << A->isRegularKeywordAttribute() << ExpectedKernelFunction;
-        D->setInvalidDecl();
-      } else if (const auto *A = D->getAttr<AMDGPUWavesPerEUAttr>()) {
-        Diag(D->getLocation(), diag::err_attribute_wrong_decl_type)
-            << A << A->isRegularKeywordAttribute() << ExpectedKernelFunction;
-        D->setInvalidDecl();
-      } else if (const auto *A = D->getAttr<AMDGPUNumSGPRAttr>()) {
-        Diag(D->getLocation(), diag::err_attribute_wrong_decl_type)
-            << A << A->isRegularKeywordAttribute() << ExpectedKernelFunction;
-        D->setInvalidDecl();
-      } else if (const auto *A = D->getAttr<AMDGPUNumVGPRAttr>()) {
-        Diag(D->getLocation(), diag::err_attribute_wrong_decl_type)
-            << A << A->isRegularKeywordAttribute() << ExpectedKernelFunction;
-        D->setInvalidDecl();
-      }
+    }
+  }
+  if (!isKernelDecl(D)) {
+    if (const auto *A = D->getAttr<AMDGPUFlatWorkGroupSizeAttr>()) {
+      Diag(D->getLocation(), diag::err_attribute_wrong_decl_type)
+          << A << A->isRegularKeywordAttribute() << ExpectedKernelFunction;
+      D->setInvalidDecl();
+    } else if (const auto *A = D->getAttr<AMDGPUWavesPerEUAttr>()) {
+      Diag(D->getLocation(), diag::err_attribute_wrong_decl_type)
+          << A << A->isRegularKeywordAttribute() << ExpectedKernelFunction;
+      D->setInvalidDecl();
+    } else if (const auto *A = D->getAttr<AMDGPUNumSGPRAttr>()) {
+      Diag(D->getLocation(), diag::err_attribute_wrong_decl_type)
+          << A << A->isRegularKeywordAttribute() << ExpectedKernelFunction;
+      D->setInvalidDecl();
+    } else if (const auto *A = D->getAttr<AMDGPUNumVGPRAttr>()) {
+      Diag(D->getLocation(), diag::err_attribute_wrong_decl_type)
+          << A << A->isRegularKeywordAttribute() << ExpectedKernelFunction;
+      D->setInvalidDecl();
     }
   }
 
diff --git a/clang/test/CodeGenCXX/amdgpu-kernel-arg-pointer-type.cpp b/clang/test/CodeGenCXX/amdgpu-kernel-arg-pointer-type.cpp
index c8fb12315d0eeca..8df1d75e9ec8fa4 100644
--- a/clang/test/CodeGenCXX/amdgpu-kernel-arg-pointer-type.cpp
+++ b/clang/test/CodeGenCXX/amdgpu-kernel-arg-pointer-type.cpp
@@ -6,6 +6,10 @@
 // The original test passes the result through opt O2, but that seems to introduce invalid
 // addrspace casts which are not being fixed as part of the present change.
 
+// COMMON: define{{.*}} amdgpu_kernel void @_Z6kernelv() #[[ATTR:[0-9]+]]
+__attribute__((amdgpu_kernel, amdgpu_flat_work_group_size(1, 256))) void
+    kernel() {}
+
 // COMMON-LABEL: define{{.*}} amdgpu_kernel void @_Z7kernel1Pi(ptr {{.*}} %x)
 // CHECK-NOT: ={{.*}} addrspacecast [[TYPE:.*]] addrspace(1)* %{{.*}} to ptr
 __attribute__((amdgpu_kernel)) void kernel1(int *x) {
@@ -81,3 +85,4 @@ __attribute__((amdgpu_kernel)) void kernel8(struct SS a) {
   *a.x += 3.f;
 }
 
+// COMMON: attributes #[[ATTR]] = { {{.*}}"amdgpu-flat-work-group-size"="1,256"{{.*}} }

From 516c1a0e9a7f101e678bcc61620ccdebd5ef83d1 Mon Sep 17 00:00:00 2001
From: vporpo <vporpodas@google.com>
Date: Fri, 16 Aug 2024 16:43:01 -0700
Subject: [PATCH 247/441] [SandboxIR] Implement SwitchInst (#104641)

This patch implements sandboxir::SwitchInst mirroring llvm::SwitchInst.
---
 llvm/include/llvm/SandboxIR/SandboxIR.h       |  90 ++++++++++++
 .../llvm/SandboxIR/SandboxIRValues.def        |   1 +
 llvm/include/llvm/SandboxIR/Tracker.h         |  33 +++++
 llvm/lib/SandboxIR/SandboxIR.cpp              |  88 ++++++++++++
 llvm/lib/SandboxIR/Tracker.cpp                |  21 +++
 llvm/unittests/SandboxIR/SandboxIRTest.cpp    | 128 ++++++++++++++++++
 llvm/unittests/SandboxIR/TrackerTest.cpp      |  80 +++++++++++
 7 files changed, 441 insertions(+)

diff --git a/llvm/include/llvm/SandboxIR/SandboxIR.h b/llvm/include/llvm/SandboxIR/SandboxIR.h
index 3c568f4956857f3..a881bdf28f22c25 100644
--- a/llvm/include/llvm/SandboxIR/SandboxIR.h
+++ b/llvm/include/llvm/SandboxIR/SandboxIR.h
@@ -131,6 +131,7 @@ class CastInst;
 class PtrToIntInst;
 class BitCastInst;
 class AllocaInst;
+class SwitchInst;
 class UnaryOperator;
 class BinaryOperator;
 class AtomicRMWInst;
@@ -253,6 +254,7 @@ class Value {
   friend class InvokeInst;         // For getting `Val`.
   friend class CallBrInst;         // For getting `Val`.
   friend class GetElementPtrInst;  // For getting `Val`.
+  friend class SwitchInst;         // For getting `Val`.
   friend class UnaryOperator;      // For getting `Val`.
   friend class BinaryOperator;     // For getting `Val`.
   friend class AtomicRMWInst;      // For getting `Val`.
@@ -672,6 +674,7 @@ class Instruction : public sandboxir::User {
   friend class InvokeInst;         // For getTopmostLLVMInstruction().
   friend class CallBrInst;         // For getTopmostLLVMInstruction().
   friend class GetElementPtrInst;  // For getTopmostLLVMInstruction().
+  friend class SwitchInst;         // For getTopmostLLVMInstruction().
   friend class UnaryOperator;      // For getTopmostLLVMInstruction().
   friend class BinaryOperator;     // For getTopmostLLVMInstruction().
   friend class AtomicRMWInst;      // For getTopmostLLVMInstruction().
@@ -1477,6 +1480,91 @@ class GetElementPtrInst final
   // TODO: Add missing member functions.
 };
 
+class SwitchInst : public SingleLLVMInstructionImpl<llvm::SwitchInst> {
+public:
+  SwitchInst(llvm::SwitchInst *SI, Context &Ctx)
+      : SingleLLVMInstructionImpl(ClassID::Switch, Opcode::Switch, SI, Ctx) {}
+
+  static constexpr const unsigned DefaultPseudoIndex =
+      llvm::SwitchInst::DefaultPseudoIndex;
+
+  static SwitchInst *create(Value *V, BasicBlock *Dest, unsigned NumCases,
+                            BasicBlock::iterator WhereIt, BasicBlock *WhereBB,
+                            Context &Ctx, const Twine &Name = "");
+
+  Value *getCondition() const;
+  void setCondition(Value *V);
+  BasicBlock *getDefaultDest() const;
+  bool defaultDestUndefined() const {
+    return cast<llvm::SwitchInst>(Val)->defaultDestUndefined();
+  }
+  void setDefaultDest(BasicBlock *DefaultCase);
+  unsigned getNumCases() const {
+    return cast<llvm::SwitchInst>(Val)->getNumCases();
+  }
+
+  using CaseHandle =
+      llvm::SwitchInst::CaseHandleImpl<SwitchInst, ConstantInt, BasicBlock>;
+  using ConstCaseHandle =
+      llvm::SwitchInst::CaseHandleImpl<const SwitchInst, const ConstantInt,
+                                       const BasicBlock>;
+  using CaseIt = llvm::SwitchInst::CaseIteratorImpl<CaseHandle>;
+  using ConstCaseIt = llvm::SwitchInst::CaseIteratorImpl<ConstCaseHandle>;
+
+  /// Returns a read/write iterator that points to the first case in the
+  /// SwitchInst.
+  CaseIt case_begin() { return CaseIt(this, 0); }
+  ConstCaseIt case_begin() const { return ConstCaseIt(this, 0); }
+  /// Returns a read/write iterator that points one past the last in the
+  /// SwitchInst.
+  CaseIt case_end() { return CaseIt(this, getNumCases()); }
+  ConstCaseIt case_end() const { return ConstCaseIt(this, getNumCases()); }
+  /// Iteration adapter for range-for loops.
+  iterator_range<CaseIt> cases() {
+    return make_range(case_begin(), case_end());
+  }
+  iterator_range<ConstCaseIt> cases() const {
+    return make_range(case_begin(), case_end());
+  }
+  CaseIt case_default() { return CaseIt(this, DefaultPseudoIndex); }
+  ConstCaseIt case_default() const {
+    return ConstCaseIt(this, DefaultPseudoIndex);
+  }
+  CaseIt findCaseValue(const ConstantInt *C) {
+    return CaseIt(
+        this,
+        const_cast<const SwitchInst *>(this)->findCaseValue(C)->getCaseIndex());
+  }
+  ConstCaseIt findCaseValue(const ConstantInt *C) const {
+    ConstCaseIt I = llvm::find_if(cases(), [C](const ConstCaseHandle &Case) {
+      return Case.getCaseValue() == C;
+    });
+    if (I != case_end())
+      return I;
+    return case_default();
+  }
+  ConstantInt *findCaseDest(BasicBlock *BB);
+
+  void addCase(ConstantInt *OnVal, BasicBlock *Dest);
+  /// This method removes the specified case and its successor from the switch
+  /// instruction. Note that this operation may reorder the remaining cases at
+  /// index idx and above.
+  /// Note:
+  /// This action invalidates iterators for all cases following the one removed,
+  /// including the case_end() iterator. It returns an iterator for the next
+  /// case.
+  CaseIt removeCase(CaseIt It);
+
+  unsigned getNumSuccessors() const {
+    return cast<llvm::SwitchInst>(Val)->getNumSuccessors();
+  }
+  BasicBlock *getSuccessor(unsigned Idx) const;
+  void setSuccessor(unsigned Idx, BasicBlock *NewSucc);
+  static bool classof(const Value *From) {
+    return From->getSubclassID() == ClassID::Switch;
+  }
+};
+
 class UnaryOperator : public UnaryInstruction {
   static Opcode getUnaryOpcode(llvm::Instruction::UnaryOps UnOp) {
     switch (UnOp) {
@@ -2113,6 +2201,8 @@ class Context {
   friend CallBrInst; // For createCallBrInst()
   GetElementPtrInst *createGetElementPtrInst(llvm::GetElementPtrInst *I);
   friend GetElementPtrInst; // For createGetElementPtrInst()
+  SwitchInst *createSwitchInst(llvm::SwitchInst *I);
+  friend SwitchInst; // For createSwitchInst()
   UnaryOperator *createUnaryOperator(llvm::UnaryOperator *I);
   friend UnaryOperator; // For createUnaryOperator()
   BinaryOperator *createBinaryOperator(llvm::BinaryOperator *I);
diff --git a/llvm/include/llvm/SandboxIR/SandboxIRValues.def b/llvm/include/llvm/SandboxIR/SandboxIRValues.def
index cba5b69ebf1217c..2b9b44c529b30d8 100644
--- a/llvm/include/llvm/SandboxIR/SandboxIRValues.def
+++ b/llvm/include/llvm/SandboxIR/SandboxIRValues.def
@@ -46,6 +46,7 @@ DEF_INSTR(Call,          OP(Call),          CallInst)
 DEF_INSTR(Invoke,        OP(Invoke),        InvokeInst)
 DEF_INSTR(CallBr,        OP(CallBr),        CallBrInst)
 DEF_INSTR(GetElementPtr, OP(GetElementPtr), GetElementPtrInst)
+DEF_INSTR(Switch,        OP(Switch),        SwitchInst)
 DEF_INSTR(UnOp,          OPCODES( \
                          OP(FNeg) \
                          ),                 UnaryOperator)
diff --git a/llvm/include/llvm/SandboxIR/Tracker.h b/llvm/include/llvm/SandboxIR/Tracker.h
index 2b4dfafa3b0e5d0..22d910e1d73e728 100644
--- a/llvm/include/llvm/SandboxIR/Tracker.h
+++ b/llvm/include/llvm/SandboxIR/Tracker.h
@@ -59,6 +59,8 @@ class StoreInst;
 class Instruction;
 class Tracker;
 class AllocaInst;
+class SwitchInst;
+class ConstantInt;
 
 /// The base class for IR Change classes.
 class IRChangeBase {
@@ -261,6 +263,37 @@ class GenericSetterWithIdx final : public IRChangeBase {
 #endif
 };
 
+class SwitchAddCase : public IRChangeBase {
+  SwitchInst *Switch;
+  ConstantInt *Val;
+
+public:
+  SwitchAddCase(SwitchInst *Switch, ConstantInt *Val)
+      : Switch(Switch), Val(Val) {}
+  void revert(Tracker &Tracker) final;
+  void accept() final {}
+#ifndef NDEBUG
+  void dump(raw_ostream &OS) const final { OS << "SwitchAddCase"; }
+  LLVM_DUMP_METHOD void dump() const final;
+#endif // NDEBUG
+};
+
+class SwitchRemoveCase : public IRChangeBase {
+  SwitchInst *Switch;
+  ConstantInt *Val;
+  BasicBlock *Dest;
+
+public:
+  SwitchRemoveCase(SwitchInst *Switch, ConstantInt *Val, BasicBlock *Dest)
+      : Switch(Switch), Val(Val), Dest(Dest) {}
+  void revert(Tracker &Tracker) final;
+  void accept() final {}
+#ifndef NDEBUG
+  void dump(raw_ostream &OS) const final { OS << "SwitchRemoveCase"; }
+  LLVM_DUMP_METHOD void dump() const final;
+#endif // NDEBUG
+};
+
 class MoveInstr : public IRChangeBase {
   /// The instruction that moved.
   Instruction *MovedI;
diff --git a/llvm/lib/SandboxIR/SandboxIR.cpp b/llvm/lib/SandboxIR/SandboxIR.cpp
index 6e8c83528bded92..c243df7fc864ee5 100644
--- a/llvm/lib/SandboxIR/SandboxIR.cpp
+++ b/llvm/lib/SandboxIR/SandboxIR.cpp
@@ -1236,6 +1236,84 @@ static llvm::Instruction::UnaryOps getLLVMUnaryOp(Instruction::Opcode Opc) {
   }
 }
 
+SwitchInst *SwitchInst::create(Value *V, BasicBlock *Dest, unsigned NumCases,
+                               BasicBlock::iterator WhereIt,
+                               BasicBlock *WhereBB, Context &Ctx,
+                               const Twine &Name) {
+  auto &Builder = Ctx.getLLVMIRBuilder();
+  if (WhereIt != WhereBB->end())
+    Builder.SetInsertPoint((*WhereIt).getTopmostLLVMInstruction());
+  else
+    Builder.SetInsertPoint(cast<llvm::BasicBlock>(WhereBB->Val));
+  llvm::SwitchInst *LLVMSwitch =
+      Builder.CreateSwitch(V->Val, cast<llvm::BasicBlock>(Dest->Val), NumCases);
+  return Ctx.createSwitchInst(LLVMSwitch);
+}
+
+Value *SwitchInst::getCondition() const {
+  return Ctx.getValue(cast<llvm::SwitchInst>(Val)->getCondition());
+}
+
+void SwitchInst::setCondition(Value *V) {
+  Ctx.getTracker()
+      .emplaceIfTracking<
+          GenericSetter<&SwitchInst::getCondition, &SwitchInst::setCondition>>(
+          this);
+  cast<llvm::SwitchInst>(Val)->setCondition(V->Val);
+}
+
+BasicBlock *SwitchInst::getDefaultDest() const {
+  return cast<BasicBlock>(
+      Ctx.getValue(cast<llvm::SwitchInst>(Val)->getDefaultDest()));
+}
+
+void SwitchInst::setDefaultDest(BasicBlock *DefaultCase) {
+  Ctx.getTracker()
+      .emplaceIfTracking<GenericSetter<&SwitchInst::getDefaultDest,
+                                       &SwitchInst::setDefaultDest>>(this);
+  cast<llvm::SwitchInst>(Val)->setDefaultDest(
+      cast<llvm::BasicBlock>(DefaultCase->Val));
+}
+ConstantInt *SwitchInst::findCaseDest(BasicBlock *BB) {
+  auto *LLVMC = cast<llvm::SwitchInst>(Val)->findCaseDest(
+      cast<llvm::BasicBlock>(BB->Val));
+  return LLVMC != nullptr ? cast<ConstantInt>(Ctx.getValue(LLVMC)) : nullptr;
+}
+
+void SwitchInst::addCase(ConstantInt *OnVal, BasicBlock *Dest) {
+  Ctx.getTracker().emplaceIfTracking<SwitchAddCase>(this, OnVal);
+  // TODO: Track this!
+  cast<llvm::SwitchInst>(Val)->addCase(cast<llvm::ConstantInt>(OnVal->Val),
+                                       cast<llvm::BasicBlock>(Dest->Val));
+}
+
+SwitchInst::CaseIt SwitchInst::removeCase(CaseIt It) {
+  auto &Case = *It;
+  Ctx.getTracker().emplaceIfTracking<SwitchRemoveCase>(
+      this, Case.getCaseValue(), Case.getCaseSuccessor());
+
+  auto *LLVMSwitch = cast<llvm::SwitchInst>(Val);
+  unsigned CaseNum = It - case_begin();
+  llvm::SwitchInst::CaseIt LLVMIt(LLVMSwitch, CaseNum);
+  auto LLVMCaseIt = LLVMSwitch->removeCase(LLVMIt);
+  unsigned Num = LLVMCaseIt - LLVMSwitch->case_begin();
+  return CaseIt(this, Num);
+}
+
+BasicBlock *SwitchInst::getSuccessor(unsigned Idx) const {
+  return cast<BasicBlock>(
+      Ctx.getValue(cast<llvm::SwitchInst>(Val)->getSuccessor(Idx)));
+}
+
+void SwitchInst::setSuccessor(unsigned Idx, BasicBlock *NewSucc) {
+  Ctx.getTracker()
+      .emplaceIfTracking<GenericSetterWithIdx<&SwitchInst::getSuccessor,
+                                              &SwitchInst::setSuccessor>>(this,
+                                                                          Idx);
+  cast<llvm::SwitchInst>(Val)->setSuccessor(
+      Idx, cast<llvm::BasicBlock>(NewSucc->Val));
+}
+
 Value *UnaryOperator::create(Instruction::Opcode Op, Value *OpV,
                              BBIterator WhereIt, BasicBlock *WhereBB,
                              Context &Ctx, const Twine &Name) {
@@ -1875,6 +1953,12 @@ Value *Context::getOrCreateValueInternal(llvm::Value *LLVMV, llvm::User *U) {
         new GetElementPtrInst(LLVMGEP, *this));
     return It->second.get();
   }
+  case llvm::Instruction::Switch: {
+    auto *LLVMSwitchInst = cast<llvm::SwitchInst>(LLVMV);
+    It->second =
+        std::unique_ptr<SwitchInst>(new SwitchInst(LLVMSwitchInst, *this));
+    return It->second.get();
+  }
   case llvm::Instruction::FNeg: {
     auto *LLVMUnaryOperator = cast<llvm::UnaryOperator>(LLVMV);
     It->second = std::unique_ptr<UnaryOperator>(
@@ -2033,6 +2117,10 @@ Context::createGetElementPtrInst(llvm::GetElementPtrInst *I) {
       std::unique_ptr<GetElementPtrInst>(new GetElementPtrInst(I, *this));
   return cast<GetElementPtrInst>(registerValue(std::move(NewPtr)));
 }
+SwitchInst *Context::createSwitchInst(llvm::SwitchInst *I) {
+  auto NewPtr = std::unique_ptr<SwitchInst>(new SwitchInst(I, *this));
+  return cast<SwitchInst>(registerValue(std::move(NewPtr)));
+}
 UnaryOperator *Context::createUnaryOperator(llvm::UnaryOperator *I) {
   auto NewPtr = std::unique_ptr<UnaryOperator>(new UnaryOperator(I, *this));
   return cast<UnaryOperator>(registerValue(std::move(NewPtr)));
diff --git a/llvm/lib/SandboxIR/Tracker.cpp b/llvm/lib/SandboxIR/Tracker.cpp
index baf218aa41c1cdd..8faff72bc3e2fba 100644
--- a/llvm/lib/SandboxIR/Tracker.cpp
+++ b/llvm/lib/SandboxIR/Tracker.cpp
@@ -160,6 +160,27 @@ void RemoveFromParent::dump() const {
 }
 #endif
 
+void SwitchRemoveCase::revert(Tracker &Tracker) { Switch->addCase(Val, Dest); }
+
+#ifndef NDEBUG
+void SwitchRemoveCase::dump() const {
+  dump(dbgs());
+  dbgs() << "\n";
+}
+#endif // NDEBUG
+
+void SwitchAddCase::revert(Tracker &Tracker) {
+  auto It = Switch->findCaseValue(Val);
+  Switch->removeCase(It);
+}
+
+#ifndef NDEBUG
+void SwitchAddCase::dump() const {
+  dump(dbgs());
+  dbgs() << "\n";
+}
+#endif // NDEBUG
+
 MoveInstr::MoveInstr(Instruction *MovedI) : MovedI(MovedI) {
   if (auto *NextI = MovedI->getNextNode())
     NextInstrOrBB = NextI;
diff --git a/llvm/unittests/SandboxIR/SandboxIRTest.cpp b/llvm/unittests/SandboxIR/SandboxIRTest.cpp
index 6c0f1b19243c289..1d6a26728c9c567 100644
--- a/llvm/unittests/SandboxIR/SandboxIRTest.cpp
+++ b/llvm/unittests/SandboxIR/SandboxIRTest.cpp
@@ -1643,6 +1643,134 @@ define void @foo(i32 %arg, float %farg) {
   EXPECT_FALSE(FAdd->getFastMathFlags() != LLVMFAdd->getFastMathFlags());
 }
 
+TEST_F(SandboxIRTest, SwitchInst) {
+  parseIR(C, R"IR(
+define void @foo(i32 %cond0, i32 %cond1) {
+  entry:
+    switch i32 %cond0, label %default [ i32 0, label %bb0
+                                        i32 1, label %bb1 ]
+  bb0:
+    ret void
+  bb1:
+    ret void
+  default:
+    ret void
+}
+)IR");
+  Function &LLVMF = *M->getFunction("foo");
+  auto *LLVMEntry = getBasicBlockByName(LLVMF, "entry");
+  auto *LLVMSwitch = cast<llvm::SwitchInst>(&*LLVMEntry->begin());
+
+  sandboxir::Context Ctx(C);
+  auto &F = *Ctx.createFunction(&LLVMF);
+  auto *Cond1 = F.getArg(1);
+  auto *Entry = cast<sandboxir::BasicBlock>(Ctx.getValue(LLVMEntry));
+  auto *Switch = cast<sandboxir::SwitchInst>(&*Entry->begin());
+  auto *BB0 = cast<sandboxir::BasicBlock>(
+      Ctx.getValue(getBasicBlockByName(LLVMF, "bb0")));
+  auto *BB1 = cast<sandboxir::BasicBlock>(
+      Ctx.getValue(getBasicBlockByName(LLVMF, "bb1")));
+  auto *Default = cast<sandboxir::BasicBlock>(
+      Ctx.getValue(getBasicBlockByName(LLVMF, "default")));
+
+  // Check getCondition().
+  EXPECT_EQ(Switch->getCondition(), Ctx.getValue(LLVMSwitch->getCondition()));
+  // Check setCondition().
+  auto *OrigCond = Switch->getCondition();
+  auto *NewCond = Cond1;
+  EXPECT_NE(NewCond, OrigCond);
+  Switch->setCondition(NewCond);
+  EXPECT_EQ(Switch->getCondition(), NewCond);
+  Switch->setCondition(OrigCond);
+  EXPECT_EQ(Switch->getCondition(), OrigCond);
+  // Check getDefaultDest().
+  EXPECT_EQ(Switch->getDefaultDest(),
+            Ctx.getValue(LLVMSwitch->getDefaultDest()));
+  EXPECT_EQ(Switch->getDefaultDest(), Default);
+  // Check defaultDestUndefined().
+  EXPECT_EQ(Switch->defaultDestUndefined(), LLVMSwitch->defaultDestUndefined());
+  // Check setDefaultDest().
+  auto *OrigDefaultDest = Switch->getDefaultDest();
+  auto *NewDefaultDest = Entry;
+  EXPECT_NE(NewDefaultDest, OrigDefaultDest);
+  Switch->setDefaultDest(NewDefaultDest);
+  EXPECT_EQ(Switch->getDefaultDest(), NewDefaultDest);
+  Switch->setDefaultDest(OrigDefaultDest);
+  EXPECT_EQ(Switch->getDefaultDest(), OrigDefaultDest);
+  // Check getNumCases().
+  EXPECT_EQ(Switch->getNumCases(), LLVMSwitch->getNumCases());
+  // Check getNumSuccessors().
+  EXPECT_EQ(Switch->getNumSuccessors(), LLVMSwitch->getNumSuccessors());
+  // Check getSuccessor().
+  for (auto SuccIdx : seq<unsigned>(0, Switch->getNumSuccessors()))
+    EXPECT_EQ(Switch->getSuccessor(SuccIdx),
+              Ctx.getValue(LLVMSwitch->getSuccessor(SuccIdx)));
+  // Check setSuccessor().
+  auto *OrigSucc = Switch->getSuccessor(0);
+  auto *NewSucc = Entry;
+  EXPECT_NE(NewSucc, OrigSucc);
+  Switch->setSuccessor(0, NewSucc);
+  EXPECT_EQ(Switch->getSuccessor(0), NewSucc);
+  Switch->setSuccessor(0, OrigSucc);
+  EXPECT_EQ(Switch->getSuccessor(0), OrigSucc);
+  // Check case_begin(), case_end(), CaseIt.
+  auto *Zero = sandboxir::ConstantInt::get(Type::getInt32Ty(C), 0, Ctx);
+  auto *One = sandboxir::ConstantInt::get(Type::getInt32Ty(C), 1, Ctx);
+  auto CaseIt = Switch->case_begin();
+  {
+    sandboxir::SwitchInst::CaseHandle Case = *CaseIt++;
+    EXPECT_EQ(Case.getCaseValue(), Zero);
+    EXPECT_EQ(Case.getCaseSuccessor(), BB0);
+    EXPECT_EQ(Case.getCaseIndex(), 0u);
+    EXPECT_EQ(Case.getSuccessorIndex(), 1u);
+  }
+  {
+    sandboxir::SwitchInst::CaseHandle Case = *CaseIt++;
+    EXPECT_EQ(Case.getCaseValue(), One);
+    EXPECT_EQ(Case.getCaseSuccessor(), BB1);
+    EXPECT_EQ(Case.getCaseIndex(), 1u);
+    EXPECT_EQ(Case.getSuccessorIndex(), 2u);
+  }
+  EXPECT_EQ(CaseIt, Switch->case_end());
+  // Check cases().
+  unsigned CntCase = 0;
+  for (auto &Case : Switch->cases()) {
+    EXPECT_EQ(Case.getCaseIndex(), CntCase);
+    ++CntCase;
+  }
+  EXPECT_EQ(CntCase, 2u);
+  // Check case_default().
+  auto CaseDefault = *Switch->case_default();
+  EXPECT_EQ(CaseDefault.getCaseSuccessor(), Default);
+  EXPECT_EQ(CaseDefault.getCaseIndex(),
+            sandboxir::SwitchInst::DefaultPseudoIndex);
+  // Check findCaseValue().
+  EXPECT_EQ(Switch->findCaseValue(Zero)->getCaseIndex(), 0u);
+  EXPECT_EQ(Switch->findCaseValue(One)->getCaseIndex(), 1u);
+  // Check findCaseDest().
+  EXPECT_EQ(Switch->findCaseDest(BB0), Zero);
+  EXPECT_EQ(Switch->findCaseDest(BB1), One);
+  EXPECT_EQ(Switch->findCaseDest(Entry), nullptr);
+  // Check addCase().
+  auto *Two = sandboxir::ConstantInt::get(Type::getInt32Ty(C), 2, Ctx);
+  Switch->addCase(Two, Entry);
+  auto CaseTwoIt = Switch->findCaseValue(Two);
+  auto CaseTwo = *CaseTwoIt;
+  EXPECT_EQ(CaseTwo.getCaseValue(), Two);
+  EXPECT_EQ(CaseTwo.getCaseSuccessor(), Entry);
+  EXPECT_EQ(Switch->getNumCases(), 3u);
+  // Check removeCase().
+  auto RemovedIt = Switch->removeCase(CaseTwoIt);
+  EXPECT_EQ(RemovedIt, Switch->case_end());
+  EXPECT_EQ(Switch->getNumCases(), 2u);
+  // Check create().
+  auto NewSwitch = sandboxir::SwitchInst::create(
+      Cond1, Default, 1, Default->begin(), Default, Ctx, "NewSwitch");
+  EXPECT_TRUE(isa<sandboxir::SwitchInst>(NewSwitch));
+  EXPECT_EQ(NewSwitch->getCondition(), Cond1);
+  EXPECT_EQ(NewSwitch->getDefaultDest(), Default);
+}
+
 TEST_F(SandboxIRTest, UnaryOperator) {
   parseIR(C, R"IR(
 define void @foo(float %arg0) {
diff --git a/llvm/unittests/SandboxIR/TrackerTest.cpp b/llvm/unittests/SandboxIR/TrackerTest.cpp
index 380c90e7f0f65f6..b6770b237853cc1 100644
--- a/llvm/unittests/SandboxIR/TrackerTest.cpp
+++ b/llvm/unittests/SandboxIR/TrackerTest.cpp
@@ -644,6 +644,86 @@ define void @foo(i8 %arg) {
   EXPECT_EQ(Invoke->getSuccessor(1), ExceptionBB);
 }
 
+TEST_F(TrackerTest, SwitchInstSetters) {
+  parseIR(C, R"IR(
+define void @foo(i32 %cond0, i32 %cond1) {
+  entry:
+    switch i32 %cond0, label %default [ i32 0, label %bb0
+                                        i32 1, label %bb1 ]
+  bb0:
+    ret void
+  bb1:
+    ret void
+  default:
+    ret void
+}
+)IR");
+  Function &LLVMF = *M->getFunction("foo");
+  auto *LLVMEntry = getBasicBlockByName(LLVMF, "entry");
+
+  sandboxir::Context Ctx(C);
+  auto &F = *Ctx.createFunction(&LLVMF);
+  auto *Cond1 = F.getArg(1);
+  auto *Entry = cast<sandboxir::BasicBlock>(Ctx.getValue(LLVMEntry));
+  auto *BB0 = cast<sandboxir::BasicBlock>(
+      Ctx.getValue(getBasicBlockByName(LLVMF, "bb0")));
+  auto *BB1 = cast<sandboxir::BasicBlock>(
+      Ctx.getValue(getBasicBlockByName(LLVMF, "bb1")));
+  auto *Switch = cast<sandboxir::SwitchInst>(&*Entry->begin());
+
+  // Check setCondition().
+  auto *OrigCond = Switch->getCondition();
+  auto *NewCond = Cond1;
+  EXPECT_NE(NewCond, OrigCond);
+  Ctx.save();
+  Switch->setCondition(NewCond);
+  EXPECT_EQ(Switch->getCondition(), NewCond);
+  Ctx.revert();
+  EXPECT_EQ(Switch->getCondition(), OrigCond);
+  // Check setDefaultDest().
+  auto *OrigDefaultDest = Switch->getDefaultDest();
+  auto *NewDefaultDest = Entry;
+  EXPECT_NE(NewDefaultDest, OrigDefaultDest);
+  Ctx.save();
+  Switch->setDefaultDest(NewDefaultDest);
+  EXPECT_EQ(Switch->getDefaultDest(), NewDefaultDest);
+  Ctx.revert();
+  EXPECT_EQ(Switch->getDefaultDest(), OrigDefaultDest);
+  // Check setSuccessor().
+  auto *OrigSucc = Switch->getSuccessor(0);
+  auto *NewSucc = Entry;
+  EXPECT_NE(NewSucc, OrigSucc);
+  Ctx.save();
+  Switch->setSuccessor(0, NewSucc);
+  EXPECT_EQ(Switch->getSuccessor(0), NewSucc);
+  Ctx.revert();
+  EXPECT_EQ(Switch->getSuccessor(0), OrigSucc);
+  // Check addCase().
+  auto *Zero = sandboxir::ConstantInt::get(Type::getInt32Ty(C), 0, Ctx);
+  auto *One = sandboxir::ConstantInt::get(Type::getInt32Ty(C), 1, Ctx);
+  auto *FortyTwo = sandboxir::ConstantInt::get(Type::getInt32Ty(C), 42, Ctx);
+  Ctx.save();
+  Switch->addCase(FortyTwo, Entry);
+  EXPECT_EQ(Switch->getNumCases(), 3u);
+  EXPECT_EQ(Switch->findCaseDest(Entry), FortyTwo);
+  EXPECT_EQ(Switch->findCaseValue(FortyTwo)->getCaseSuccessor(), Entry);
+  EXPECT_EQ(Switch->findCaseDest(BB0), Zero);
+  EXPECT_EQ(Switch->findCaseDest(BB1), One);
+  Ctx.revert();
+  EXPECT_EQ(Switch->getNumCases(), 2u);
+  EXPECT_EQ(Switch->findCaseDest(BB0), Zero);
+  EXPECT_EQ(Switch->findCaseDest(BB1), One);
+  // Check removeCase().
+  Ctx.save();
+  Switch->removeCase(Switch->findCaseValue(Zero));
+  EXPECT_EQ(Switch->getNumCases(), 1u);
+  EXPECT_EQ(Switch->findCaseDest(BB1), One);
+  Ctx.revert();
+  EXPECT_EQ(Switch->getNumCases(), 2u);
+  EXPECT_EQ(Switch->findCaseDest(BB0), Zero);
+  EXPECT_EQ(Switch->findCaseDest(BB1), One);
+}
+
 TEST_F(TrackerTest, AtomicRMWSetters) {
   parseIR(C, R"IR(
 define void @foo(ptr %ptr, i8 %arg) {

From 981191aa94bfd3fce0852ee2c8ff2b23aba5a4a6 Mon Sep 17 00:00:00 2001
From: Kelvin Li <kkwli@users.noreply.github.com>
Date: Fri, 16 Aug 2024 19:56:22 -0400
Subject: [PATCH 248/441] [flang] Allow flexible name in llvm.ident (NFC)
 (#104543)

---
 flang/test/Lower/ident.f90 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flang/test/Lower/ident.f90 b/flang/test/Lower/ident.f90
index c5201edbcf83f7b..2a6190b5f99182c 100644
--- a/flang/test/Lower/ident.f90
+++ b/flang/test/Lower/ident.f90
@@ -2,4 +2,4 @@
 ! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s
 
 ! CHECK: module attributes {
-! CHECK-SAME: llvm.ident = "flang version {{.+}}"
+! CHECK-SAME: llvm.ident = "{{.*}}flang version {{.+}}"

From cd3f48df82dae19493da838afd1c0aaf98c737eb Mon Sep 17 00:00:00 2001
From: Greg Roth <grroth@microsoft.com>
Date: Fri, 16 Aug 2024 18:11:16 -0600
Subject: [PATCH 249/441] [NFC][DXIL] move replace/erase in DXIL intrinsic
 expansion to caller (#104626)

All expansions end with replacing the previous inrinsic with the new
expansion and erasing the old one. By moving this operation to the
caller, these expansion functions can be called in more contexts and a
small amount of duplicated code is consolidated.

Pre-req for #88056
---
 .../Target/DirectX/DXILIntrinsicExpansion.cpp | 137 ++++++++----------
 1 file changed, 61 insertions(+), 76 deletions(-)

diff --git a/llvm/lib/Target/DirectX/DXILIntrinsicExpansion.cpp b/llvm/lib/Target/DirectX/DXILIntrinsicExpansion.cpp
index e63633b8a1e1abc..2c481d15be5bde3 100644
--- a/llvm/lib/Target/DirectX/DXILIntrinsicExpansion.cpp
+++ b/llvm/lib/Target/DirectX/DXILIntrinsicExpansion.cpp
@@ -51,7 +51,7 @@ static bool isIntrinsicExpansion(Function &F) {
   return false;
 }
 
-static bool expandAbs(CallInst *Orig) {
+static Value *expandAbs(CallInst *Orig) {
   Value *X = Orig->getOperand(0);
   IRBuilder<> Builder(Orig->getParent());
   Builder.SetInsertPoint(Orig);
@@ -64,14 +64,11 @@ static bool expandAbs(CallInst *Orig) {
                              ConstantInt::get(EltTy, 0))
                        : ConstantInt::get(EltTy, 0);
   auto *V = Builder.CreateSub(Zero, X);
-  auto *MaxCall =
-      Builder.CreateIntrinsic(Ty, Intrinsic::smax, {X, V}, nullptr, "dx.max");
-  Orig->replaceAllUsesWith(MaxCall);
-  Orig->eraseFromParent();
-  return true;
+  return Builder.CreateIntrinsic(Ty, Intrinsic::smax, {X, V}, nullptr,
+                                 "dx.max");
 }
 
-static bool expandIntegerDot(CallInst *Orig, Intrinsic::ID DotIntrinsic) {
+static Value *expandIntegerDot(CallInst *Orig, Intrinsic::ID DotIntrinsic) {
   assert(DotIntrinsic == Intrinsic::dx_sdot ||
          DotIntrinsic == Intrinsic::dx_udot);
   Intrinsic::ID MadIntrinsic = DotIntrinsic == Intrinsic::dx_sdot
@@ -97,12 +94,10 @@ static bool expandIntegerDot(CallInst *Orig, Intrinsic::ID DotIntrinsic) {
                                      ArrayRef<Value *>{Elt0, Elt1, Result},
                                      nullptr, "dx.mad");
   }
-  Orig->replaceAllUsesWith(Result);
-  Orig->eraseFromParent();
-  return true;
+  return Result;
 }
 
-static bool expandExpIntrinsic(CallInst *Orig) {
+static Value *expandExpIntrinsic(CallInst *Orig) {
   Value *X = Orig->getOperand(0);
   IRBuilder<> Builder(Orig->getParent());
   Builder.SetInsertPoint(Orig);
@@ -119,23 +114,21 @@ static bool expandExpIntrinsic(CallInst *Orig) {
       Builder.CreateIntrinsic(Ty, Intrinsic::exp2, {NewX}, nullptr, "dx.exp2");
   Exp2Call->setTailCall(Orig->isTailCall());
   Exp2Call->setAttributes(Orig->getAttributes());
-  Orig->replaceAllUsesWith(Exp2Call);
-  Orig->eraseFromParent();
-  return true;
+  return Exp2Call;
 }
 
-static bool expandAnyIntrinsic(CallInst *Orig) {
+static Value *expandAnyIntrinsic(CallInst *Orig) {
   Value *X = Orig->getOperand(0);
   IRBuilder<> Builder(Orig->getParent());
   Builder.SetInsertPoint(Orig);
   Type *Ty = X->getType();
   Type *EltTy = Ty->getScalarType();
 
+  Value *Result = nullptr;
   if (!Ty->isVectorTy()) {
-    Value *Cond = EltTy->isFloatingPointTy()
-                      ? Builder.CreateFCmpUNE(X, ConstantFP::get(EltTy, 0))
-                      : Builder.CreateICmpNE(X, ConstantInt::get(EltTy, 0));
-    Orig->replaceAllUsesWith(Cond);
+    Result = EltTy->isFloatingPointTy()
+                 ? Builder.CreateFCmpUNE(X, ConstantFP::get(EltTy, 0))
+                 : Builder.CreateICmpNE(X, ConstantInt::get(EltTy, 0));
   } else {
     auto *XVec = dyn_cast<FixedVectorType>(Ty);
     Value *Cond =
@@ -148,18 +141,16 @@ static bool expandAnyIntrinsic(CallInst *Orig) {
                   X, ConstantVector::getSplat(
                          ElementCount::getFixed(XVec->getNumElements()),
                          ConstantInt::get(EltTy, 0)));
-    Value *Result = Builder.CreateExtractElement(Cond, (uint64_t)0);
+    Result = Builder.CreateExtractElement(Cond, (uint64_t)0);
     for (unsigned I = 1; I < XVec->getNumElements(); I++) {
       Value *Elt = Builder.CreateExtractElement(Cond, I);
       Result = Builder.CreateOr(Result, Elt);
     }
-    Orig->replaceAllUsesWith(Result);
   }
-  Orig->eraseFromParent();
-  return true;
+  return Result;
 }
 
-static bool expandLengthIntrinsic(CallInst *Orig) {
+static Value *expandLengthIntrinsic(CallInst *Orig) {
   Value *X = Orig->getOperand(0);
   IRBuilder<> Builder(Orig->getParent());
   Builder.SetInsertPoint(Orig);
@@ -182,15 +173,11 @@ static bool expandLengthIntrinsic(CallInst *Orig) {
     Value *Mul = Builder.CreateFMul(Elt, Elt);
     Sum = Builder.CreateFAdd(Sum, Mul);
   }
-  Value *Result = Builder.CreateIntrinsic(
-      EltTy, Intrinsic::sqrt, ArrayRef<Value *>{Sum}, nullptr, "elt.sqrt");
-
-  Orig->replaceAllUsesWith(Result);
-  Orig->eraseFromParent();
-  return true;
+  return Builder.CreateIntrinsic(EltTy, Intrinsic::sqrt, ArrayRef<Value *>{Sum},
+                                 nullptr, "elt.sqrt");
 }
 
-static bool expandLerpIntrinsic(CallInst *Orig) {
+static Value *expandLerpIntrinsic(CallInst *Orig) {
   Value *X = Orig->getOperand(0);
   Value *Y = Orig->getOperand(1);
   Value *S = Orig->getOperand(2);
@@ -198,14 +185,11 @@ static bool expandLerpIntrinsic(CallInst *Orig) {
   Builder.SetInsertPoint(Orig);
   auto *V = Builder.CreateFSub(Y, X);
   V = Builder.CreateFMul(S, V);
-  auto *Result = Builder.CreateFAdd(X, V, "dx.lerp");
-  Orig->replaceAllUsesWith(Result);
-  Orig->eraseFromParent();
-  return true;
+  return Builder.CreateFAdd(X, V, "dx.lerp");
 }
 
-static bool expandLogIntrinsic(CallInst *Orig,
-                               float LogConstVal = numbers::ln2f) {
+static Value *expandLogIntrinsic(CallInst *Orig,
+                                 float LogConstVal = numbers::ln2f) {
   Value *X = Orig->getOperand(0);
   IRBuilder<> Builder(Orig->getParent());
   Builder.SetInsertPoint(Orig);
@@ -221,16 +205,13 @@ static bool expandLogIntrinsic(CallInst *Orig,
       Builder.CreateIntrinsic(Ty, Intrinsic::log2, {X}, nullptr, "elt.log2");
   Log2Call->setTailCall(Orig->isTailCall());
   Log2Call->setAttributes(Orig->getAttributes());
-  auto *Result = Builder.CreateFMul(Ln2Const, Log2Call);
-  Orig->replaceAllUsesWith(Result);
-  Orig->eraseFromParent();
-  return true;
+  return Builder.CreateFMul(Ln2Const, Log2Call);
 }
-static bool expandLog10Intrinsic(CallInst *Orig) {
+static Value *expandLog10Intrinsic(CallInst *Orig) {
   return expandLogIntrinsic(Orig, numbers::ln2f / numbers::ln10f);
 }
 
-static bool expandNormalizeIntrinsic(CallInst *Orig) {
+static Value *expandNormalizeIntrinsic(CallInst *Orig) {
   Value *X = Orig->getOperand(0);
   Type *Ty = Orig->getType();
   Type *EltTy = Ty->getScalarType();
@@ -245,11 +226,7 @@ static bool expandNormalizeIntrinsic(CallInst *Orig) {
         report_fatal_error(Twine("Invalid input scalar: length is zero"),
                            /* gen_crash_diag=*/false);
     }
-    Value *Result = Builder.CreateFDiv(X, X);
-
-    Orig->replaceAllUsesWith(Result);
-    Orig->eraseFromParent();
-    return true;
+    return Builder.CreateFDiv(X, X);
   }
 
   unsigned XVecSize = XVec->getNumElements();
@@ -291,14 +268,10 @@ static bool expandNormalizeIntrinsic(CallInst *Orig) {
                                                 nullptr, "dx.rsqrt");
 
   Value *MultiplicandVec = Builder.CreateVectorSplat(XVecSize, Multiplicand);
-  Value *Result = Builder.CreateFMul(X, MultiplicandVec);
-
-  Orig->replaceAllUsesWith(Result);
-  Orig->eraseFromParent();
-  return true;
+  return Builder.CreateFMul(X, MultiplicandVec);
 }
 
-static bool expandPowIntrinsic(CallInst *Orig) {
+static Value *expandPowIntrinsic(CallInst *Orig) {
 
   Value *X = Orig->getOperand(0);
   Value *Y = Orig->getOperand(1);
@@ -313,9 +286,7 @@ static bool expandPowIntrinsic(CallInst *Orig) {
       Builder.CreateIntrinsic(Ty, Intrinsic::exp2, {Mul}, nullptr, "elt.exp2");
   Exp2Call->setTailCall(Orig->isTailCall());
   Exp2Call->setAttributes(Orig->getAttributes());
-  Orig->replaceAllUsesWith(Exp2Call);
-  Orig->eraseFromParent();
-  return true;
+  return Exp2Call;
 }
 
 static Intrinsic::ID getMaxForClamp(Type *ElemTy,
@@ -344,7 +315,8 @@ static Intrinsic::ID getMinForClamp(Type *ElemTy,
   return Intrinsic::minnum;
 }
 
-static bool expandClampIntrinsic(CallInst *Orig, Intrinsic::ID ClampIntrinsic) {
+static Value *expandClampIntrinsic(CallInst *Orig,
+                                   Intrinsic::ID ClampIntrinsic) {
   Value *X = Orig->getOperand(0);
   Value *Min = Orig->getOperand(1);
   Value *Max = Orig->getOperand(2);
@@ -353,41 +325,54 @@ static bool expandClampIntrinsic(CallInst *Orig, Intrinsic::ID ClampIntrinsic) {
   Builder.SetInsertPoint(Orig);
   auto *MaxCall = Builder.CreateIntrinsic(
       Ty, getMaxForClamp(Ty, ClampIntrinsic), {X, Min}, nullptr, "dx.max");
-  auto *MinCall =
-      Builder.CreateIntrinsic(Ty, getMinForClamp(Ty, ClampIntrinsic),
-                              {MaxCall, Max}, nullptr, "dx.min");
-
-  Orig->replaceAllUsesWith(MinCall);
-  Orig->eraseFromParent();
-  return true;
+  return Builder.CreateIntrinsic(Ty, getMinForClamp(Ty, ClampIntrinsic),
+                                 {MaxCall, Max}, nullptr, "dx.min");
 }
 
 static bool expandIntrinsic(Function &F, CallInst *Orig) {
+  Value *Result = nullptr;
   switch (F.getIntrinsicID()) {
   case Intrinsic::abs:
-    return expandAbs(Orig);
+    Result = expandAbs(Orig);
+    break;
   case Intrinsic::exp:
-    return expandExpIntrinsic(Orig);
+    Result = expandExpIntrinsic(Orig);
+    break;
   case Intrinsic::log:
-    return expandLogIntrinsic(Orig);
+    Result = expandLogIntrinsic(Orig);
+    break;
   case Intrinsic::log10:
-    return expandLog10Intrinsic(Orig);
+    Result = expandLog10Intrinsic(Orig);
+    break;
   case Intrinsic::pow:
-    return expandPowIntrinsic(Orig);
+    Result = expandPowIntrinsic(Orig);
+    break;
   case Intrinsic::dx_any:
-    return expandAnyIntrinsic(Orig);
+    Result = expandAnyIntrinsic(Orig);
+    break;
   case Intrinsic::dx_uclamp:
   case Intrinsic::dx_clamp:
-    return expandClampIntrinsic(Orig, F.getIntrinsicID());
+    Result = expandClampIntrinsic(Orig, F.getIntrinsicID());
+    break;
   case Intrinsic::dx_lerp:
-    return expandLerpIntrinsic(Orig);
+    Result = expandLerpIntrinsic(Orig);
+    break;
   case Intrinsic::dx_length:
-    return expandLengthIntrinsic(Orig);
+    Result = expandLengthIntrinsic(Orig);
+    break;
   case Intrinsic::dx_normalize:
-    return expandNormalizeIntrinsic(Orig);
+    Result = expandNormalizeIntrinsic(Orig);
+    break;
   case Intrinsic::dx_sdot:
   case Intrinsic::dx_udot:
-    return expandIntegerDot(Orig, F.getIntrinsicID());
+    Result = expandIntegerDot(Orig, F.getIntrinsicID());
+    break;
+  }
+
+  if (Result) {
+    Orig->replaceAllUsesWith(Result);
+    Orig->eraseFromParent();
+    return true;
   }
   return false;
 }

From 8d8f56da3038dd8e36713f8a4926aeffd0f1f80b Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Fri, 16 Aug 2024 18:01:47 -0700
Subject: [PATCH 250/441] [MC,AArch64] Remove unneeded STT_NOTYPE/STB_LOCAL
 code for mapping symbols and improve tests

---
 .../AArch64/MCTargetDesc/AArch64ELFStreamer.cpp |  2 --
 llvm/test/MC/AArch64/mapping-within-section.s   | 17 ++++++++++-------
 2 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp
index ed670bce594ec60..c69c87c685303c7 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp
@@ -272,8 +272,6 @@ class AArch64ELFStreamer : public MCELFStreamer {
   void emitMappingSymbol(StringRef Name) {
     auto *Symbol = cast<MCSymbolELF>(getContext().createLocalSymbol(Name));
     emitLabel(Symbol);
-    Symbol->setType(ELF::STT_NOTYPE);
-    Symbol->setBinding(ELF::STB_LOCAL);
   }
 
   DenseMap<const MCSection *, ElfMappingSymbol> LastMappingSymbols;
diff --git a/llvm/test/MC/AArch64/mapping-within-section.s b/llvm/test/MC/AArch64/mapping-within-section.s
index c84e3a4d2fe647e..791d63b1c65b890 100644
--- a/llvm/test/MC/AArch64/mapping-within-section.s
+++ b/llvm/test/MC/AArch64/mapping-within-section.s
@@ -1,4 +1,5 @@
-// RUN: llvm-mc -triple=aarch64-none-linux-gnu -filetype=obj %s | llvm-nm --no-sort --special-syms - | FileCheck %s --match-full-lines
+# RUN: llvm-mc -triple=aarch64-none-linux-gnu -filetype=obj %s -o %t
+# RUN: llvm-readelf -Ss %t | FileCheck %s
 
     .text
 // $x at 0x0000
@@ -22,9 +23,11 @@
 $d:
 $x:
 
-// CHECK:      0000000000000000 t $x
-// CHECK-NEXT: 0000000000000004 t $d
-// CHECK-NEXT: 0000000000000064 t $x
-// CHECK-NEXT: 0000000000000068 t $x
-// CHECK-NEXT: 0000000000000068 T $d
-// CHECK-NOT:  {{.}}
+# CHECK: [[#TEXT:]]] .text
+
+# CHECK:      1: 0000000000000000     0 NOTYPE  LOCAL  DEFAULT [[#TEXT]] $x
+# CHECK-NEXT: 2: 0000000000000004     0 NOTYPE  LOCAL  DEFAULT [[#TEXT]] $d
+# CHECK-NEXT: 3: 0000000000000064     0 NOTYPE  LOCAL  DEFAULT [[#TEXT]] $x
+# CHECK-NEXT: 4: 0000000000000068     0 NOTYPE  LOCAL  DEFAULT [[#TEXT]] $x
+# CHECK-NEXT: 5: 0000000000000068     0 NOTYPE  GLOBAL DEFAULT [[#TEXT]] $d
+# CHECK-NOT:  {{.}}

From 4cf9a4266167c686e7e405ead5d6c1cd389eeca5 Mon Sep 17 00:00:00 2001
From: Nathan Sidwell <nathan@acm.org>
Date: Fri, 16 Aug 2024 21:05:08 -0400
Subject: [PATCH 251/441] [clang-format] Adjust requires clause wrapping
 (#101550) (#102078)

Address #101550 by adding OwnLineWithBrace option for RequiresClausePosition. This permits placing a following '{' on the same line as the requires clause.

Thus, instead of:
```
bool Foo ()
  requires(true)
{
  return true;
}
```

we have:
```
bool Foo ()
  requires(true) {
  return true;
}
```

If the function body is empty, we'll get:
```
bool Foo ()
  requires(true) {}
```

I attempted to get a line break between the open and close braces, but
failed. Perhaps that's fine -- it's rare and only happens in the empty
body case.
---
 clang/docs/ClangFormatStyleOptions.rst    | 36 ++++++++--
 clang/include/clang/Format/Format.h       | 34 ++++++++--
 clang/lib/Format/ContinuationIndenter.cpp |  1 +
 clang/lib/Format/Format.cpp               |  1 +
 clang/lib/Format/TokenAnnotator.cpp       |  7 +-
 clang/unittests/Format/FormatTest.cpp     | 83 +++++++++++++++++++++++
 6 files changed, 150 insertions(+), 12 deletions(-)

diff --git a/clang/docs/ClangFormatStyleOptions.rst b/clang/docs/ClangFormatStyleOptions.rst
index 72e1bd19b6b520f..c79a635d86a6ef8 100644
--- a/clang/docs/ClangFormatStyleOptions.rst
+++ b/clang/docs/ClangFormatStyleOptions.rst
@@ -4187,7 +4187,8 @@ the configuration (without a prefix: ``Auto``).
 
 **IndentRequiresClause** (``Boolean``) :versionbadge:`clang-format 15` :ref:`¶ <IndentRequiresClause>`
   Indent the requires clause in a template. This only applies when
-  ``RequiresClausePosition`` is ``OwnLine``, or ``WithFollowing``.
+  ``RequiresClausePosition`` is ``OwnLine``, ``OwnLineWithBrace``,
+  or ``WithFollowing``.
 
   In clang-format 12, 13 and 14 it was named ``IndentRequires``.
 
@@ -5419,23 +5420,48 @@ the configuration (without a prefix: ``Auto``).
   Possible values:
 
   * ``RCPS_OwnLine`` (in configuration: ``OwnLine``)
-    Always put the ``requires`` clause on its own line.
+    Always put the ``requires`` clause on its own line (possibly followed by
+    a semicolon).
 
     .. code-block:: c++
 
       template <typename T>
-      requires C<T>
+        requires C<T>
       struct Foo {...
 
       template <typename T>
-      requires C<T>
+      void bar(T t)
+        requires C<T>;
+
+      template <typename T>
+        requires C<T>
       void bar(T t) {...
 
       template <typename T>
       void baz(T t)
-      requires C<T>
+        requires C<T>
       {...
 
+  * ``RCPS_OwnLineWithBrace`` (in configuration: ``OwnLineWithBrace``)
+    As with ``OwnLine``, except, unless otherwise prohibited, place a
+    following open brace (of a function definition) to follow on the same
+    line.
+
+    .. code-block:: c++
+
+      void bar(T t)
+        requires C<T> {
+        return;
+      }
+
+      void bar(T t)
+        requires C<T> {}
+
+      template <typename T>
+        requires C<T>
+      void baz(T t) {
+        ...
+
   * ``RCPS_WithPreceding`` (in configuration: ``WithPreceding``)
     Try to put the clause together with the preceding part of a declaration.
     For class templates: stick to the template declaration.
diff --git a/clang/include/clang/Format/Format.h b/clang/include/clang/Format/Format.h
index ef6c76a070bfaaa..2af1d4065c3cc19 100644
--- a/clang/include/clang/Format/Format.h
+++ b/clang/include/clang/Format/Format.h
@@ -2858,7 +2858,8 @@ struct FormatStyle {
   PPDirectiveIndentStyle IndentPPDirectives;
 
   /// Indent the requires clause in a template. This only applies when
-  /// ``RequiresClausePosition`` is ``OwnLine``, or ``WithFollowing``.
+  /// ``RequiresClausePosition`` is ``OwnLine``, ``OwnLineWithBrace``,
+  /// or ``WithFollowing``.
   ///
   /// In clang-format 12, 13 and 14 it was named ``IndentRequires``.
   /// \code
@@ -3944,22 +3945,45 @@ struct FormatStyle {
   /// ``IndentRequires`` option is only used if the ``requires`` is put on the
   /// start of a line.
   enum RequiresClausePositionStyle : int8_t {
-    /// Always put the ``requires`` clause on its own line.
+    /// Always put the ``requires`` clause on its own line (possibly followed by
+    /// a semicolon).
     /// \code
     ///   template <typename T>
-    ///   requires C<T>
+    ///     requires C<T>
     ///   struct Foo {...
     ///
     ///   template <typename T>
-    ///   requires C<T>
+    ///   void bar(T t)
+    ///     requires C<T>;
+    ///
+    ///   template <typename T>
+    ///     requires C<T>
     ///   void bar(T t) {...
     ///
     ///   template <typename T>
     ///   void baz(T t)
-    ///   requires C<T>
+    ///     requires C<T>
     ///   {...
     /// \endcode
     RCPS_OwnLine,
+    /// As with ``OwnLine``, except, unless otherwise prohibited, place a
+    /// following open brace (of a function definition) to follow on the same
+    /// line.
+    /// \code
+    ///   void bar(T t)
+    ///     requires C<T> {
+    ///     return;
+    ///   }
+    ///
+    ///   void bar(T t)
+    ///     requires C<T> {}
+    ///
+    ///   template <typename T>
+    ///     requires C<T>
+    ///   void baz(T t) {
+    ///     ...
+    /// \endcode
+    RCPS_OwnLineWithBrace,
     /// Try to put the clause together with the preceding part of a declaration.
     /// For class templates: stick to the template declaration.
     /// For function templates: stick to the template declaration.
diff --git a/clang/lib/Format/ContinuationIndenter.cpp b/clang/lib/Format/ContinuationIndenter.cpp
index 43d246b7f824198..46dafad65863dcf 100644
--- a/clang/lib/Format/ContinuationIndenter.cpp
+++ b/clang/lib/Format/ContinuationIndenter.cpp
@@ -1427,6 +1427,7 @@ unsigned ContinuationIndenter::getNewLineColumn(const LineState &State) {
     switch (Style.RequiresClausePosition) {
     case FormatStyle::RCPS_OwnLine:
     case FormatStyle::RCPS_WithFollowing:
+    case FormatStyle::RCPS_OwnLineWithBrace:
       return CurrentState.Indent;
     default:
       break;
diff --git a/clang/lib/Format/Format.cpp b/clang/lib/Format/Format.cpp
index 5358b35c19de25e..a795d869a3cb66f 100644
--- a/clang/lib/Format/Format.cpp
+++ b/clang/lib/Format/Format.cpp
@@ -540,6 +540,7 @@ struct ScalarEnumerationTraits<FormatStyle::RequiresClausePositionStyle> {
   static void enumeration(IO &IO,
                           FormatStyle::RequiresClausePositionStyle &Value) {
     IO.enumCase(Value, "OwnLine", FormatStyle::RCPS_OwnLine);
+    IO.enumCase(Value, "OwnLineWithBrace", FormatStyle::RCPS_OwnLineWithBrace);
     IO.enumCase(Value, "WithPreceding", FormatStyle::RCPS_WithPreceding);
     IO.enumCase(Value, "WithFollowing", FormatStyle::RCPS_WithFollowing);
     IO.enumCase(Value, "SingleLine", FormatStyle::RCPS_SingleLine);
diff --git a/clang/lib/Format/TokenAnnotator.cpp b/clang/lib/Format/TokenAnnotator.cpp
index 9f79fa9fc516ca6..9d4204655b8ed6e 100644
--- a/clang/lib/Format/TokenAnnotator.cpp
+++ b/clang/lib/Format/TokenAnnotator.cpp
@@ -5681,6 +5681,7 @@ bool TokenAnnotator::mustBreakBefore(const AnnotatedLine &Line,
   if (Right.is(TT_RequiresClause)) {
     switch (Style.RequiresClausePosition) {
     case FormatStyle::RCPS_OwnLine:
+    case FormatStyle::RCPS_OwnLineWithBrace:
     case FormatStyle::RCPS_WithFollowing:
       return true;
     default:
@@ -5699,11 +5700,13 @@ bool TokenAnnotator::mustBreakBefore(const AnnotatedLine &Line,
            (Style.BreakTemplateDeclarations == FormatStyle::BTDS_Leave &&
             Right.NewlinesBefore > 0);
   }
-  if (Left.ClosesRequiresClause && Right.isNot(tok::semi)) {
+  if (Left.ClosesRequiresClause) {
     switch (Style.RequiresClausePosition) {
     case FormatStyle::RCPS_OwnLine:
     case FormatStyle::RCPS_WithPreceding:
-      return true;
+      return Right.isNot(tok::semi);
+    case FormatStyle::RCPS_OwnLineWithBrace:
+      return !Right.isOneOf(tok::semi, tok::l_brace);
     default:
       break;
     }
diff --git a/clang/unittests/Format/FormatTest.cpp b/clang/unittests/Format/FormatTest.cpp
index bad1b1d662d1330..794ccab37045346 100644
--- a/clang/unittests/Format/FormatTest.cpp
+++ b/clang/unittests/Format/FormatTest.cpp
@@ -25857,7 +25857,90 @@ TEST_F(FormatTest, RequiresClausesPositions) {
                "}",
                Style);
 
+  Style.RequiresClausePosition = FormatStyle::RCPS_OwnLineWithBrace;
+  Style.IndentRequiresClause = true;
+
+  verifyFormat("template <typename T>\n"
+               "  requires(Foo<T> && std::trait<T>)\n"
+               "struct Bar;",
+               Style);
+
+  verifyFormat("template <typename T>\n"
+               "  requires(Foo<T> && std::trait<T>)\n"
+               "class Bar {\n"
+               "public:\n"
+               "  Bar(T t);\n"
+               "  bool baz();\n"
+               "};",
+               Style);
+
+  verifyFormat(
+      "template <typename T>\n"
+      "  requires requires(T &&t) {\n"
+      "             typename T::I;\n"
+      "             requires(F<typename T::I> && std::trait<typename T::I>);\n"
+      "           }\n"
+      "Bar(T) -> Bar<typename T::I>;",
+      Style);
+
+  verifyFormat("template <typename T>\n"
+               "  requires(Foo<T> && std::trait<T>)\n"
+               "constexpr T MyGlobal;",
+               Style);
+
+  verifyFormat("template <typename T>\n"
+               "  requires Foo<T> && requires(T t) {\n"
+               "                       { t.baz() } -> std::same_as<bool>;\n"
+               "                       requires std::same_as<T::Factor, int>;\n"
+               "                     }\n"
+               "inline int bar(T t) {\n"
+               "  return t.baz() ? T::Factor : 5;\n"
+               "}",
+               Style);
+
+  verifyFormat("template <typename T>\n"
+               "inline int bar(T t)\n"
+               "  requires Foo<T> && requires(T t) {\n"
+               "                       { t.baz() } -> std::same_as<bool>;\n"
+               "                       requires std::same_as<T::Factor, int>;\n"
+               "                     } {\n"
+               "  return t.baz() ? T::Factor : 5;\n"
+               "}",
+               Style);
+
+  verifyFormat("template <typename T>\n"
+               "  requires F<T>\n"
+               "int bar(T t) {\n"
+               "  return 5;\n"
+               "}",
+               Style);
+
+  verifyFormat("template <typename T>\n"
+               "int bar(T t)\n"
+               "  requires F<T> {\n"
+               "  return 5;\n"
+               "}",
+               Style);
+
+  verifyFormat("template <typename T>\n"
+               "int S::bar(T t) &&\n"
+               "  requires F<T> {\n"
+               "  return 5;\n"
+               "}",
+               Style);
+
+  verifyFormat("template <typename T>\n"
+               "int bar(T t)\n"
+               "  requires F<T>;",
+               Style);
+
+  verifyFormat("template <typename T>\n"
+               "int bar(T t)\n"
+               "  requires F<T> {}",
+               Style);
+
   Style.RequiresClausePosition = FormatStyle::RCPS_SingleLine;
+  Style.IndentRequiresClause = false;
   verifyFormat("template <typename T> requires Foo<T> struct Bar {};\n"
                "template <typename T> requires Foo<T> void bar() {}\n"
                "template <typename T> void bar() requires Foo<T> {}\n"

From 6e0fc155782ff5307245a85c7b037a2998ec6c86 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Fri, 16 Aug 2024 17:16:07 -0700
Subject: [PATCH 252/441] [RISCV] Remove feature implication from Zvknhb.

We don't have feature implications on any other Zvk extensions and
we have error messages in RISCVISAInfo if Zve or V is not enabled.
I'm working on testing and refactoring in that code so I'd like to
make it consistent.
---
 llvm/lib/Target/RISCV/RISCVFeatures.td | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVFeatures.td b/llvm/lib/Target/RISCV/RISCVFeatures.td
index e278fa3fe31764e..f2610ea7c9768ee 100644
--- a/llvm/lib/Target/RISCV/RISCVFeatures.td
+++ b/llvm/lib/Target/RISCV/RISCVFeatures.td
@@ -780,8 +780,7 @@ def HasStdExtZvknha : Predicate<"Subtarget->hasStdExtZvknha()">,
 
 def FeatureStdExtZvknhb
     : RISCVExtension<"zvknhb", 1, 0,
-                     "'Zvknhb' (Vector SHA-2 (SHA-256 and SHA-512))",
-                     [FeatureStdExtZve64x]>,
+                     "'Zvknhb' (Vector SHA-2 (SHA-256 and SHA-512))">,
       RISCVExtensionBitmask<0, 56>;
 def HasStdExtZvknhb : Predicate<"Subtarget->hasStdExtZvknhb()">,
                       AssemblerPredicate<(all_of FeatureStdExtZvknhb),

From f44f02629274ea67aad23553ebc11042537758ad Mon Sep 17 00:00:00 2001
From: Vitaly Buka <vitalybuka@google.com>
Date: Fri, 16 Aug 2024 20:59:39 -0700
Subject: [PATCH 253/441] [asan] Catch `initialization-order-fiasco` in modules
 without globals (#104621)

Those modules still can have global constructors and access
globals in other modules which are not initialized yet.
---
 .../TestCases/initialization-bug-no-global.cpp     |  3 ---
 .../Instrumentation/AddressSanitizer.cpp           | 14 +++-----------
 .../instrument_initializer_without_global.ll       |  2 ++
 3 files changed, 5 insertions(+), 14 deletions(-)

diff --git a/compiler-rt/test/asan/TestCases/initialization-bug-no-global.cpp b/compiler-rt/test/asan/TestCases/initialization-bug-no-global.cpp
index 50698b2a7d06fc4..369449303f3e082 100644
--- a/compiler-rt/test/asan/TestCases/initialization-bug-no-global.cpp
+++ b/compiler-rt/test/asan/TestCases/initialization-bug-no-global.cpp
@@ -1,9 +1,6 @@
 // RUN: %clangxx_asan %min_macos_deployment_target=10.11 -O0 %s %p/Helpers/initialization-bug-extra.cpp -o %t
 // RUN: %env_asan_opts=check_initialization_order=true:strict_init_order=true not %run %t 2>&1 | FileCheck %s
 
-// Not implemented.
-// XFAIL: *
-
 // Do not test with optimization -- the error may be optimized away.
 
 // FIXME: https://code.google.com/p/address-sanitizer/issues/detail?id=186
diff --git a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
index d1bb1334aae6a36..34366b98aed7ae7 100644
--- a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
@@ -2531,15 +2531,10 @@ void ModuleAddressSanitizer::instrumentGlobals(IRBuilder<> &IRB, Module &M,
   SmallVector<GlobalVariable *, 16> NewGlobals(n);
   SmallVector<Constant *, 16> Initializers(n);
 
-  bool HasDynamicallyInitializedGlobals = false;
-
   // We shouldn't merge same module names, as this string serves as unique
   // module ID in runtime.
-  GlobalVariable *ModuleName =
-      n != 0
-          ? createPrivateGlobalForString(M, M.getModuleIdentifier(),
-                                         /*AllowMerging*/ false, kAsanGenPrefix)
-          : nullptr;
+  GlobalVariable *ModuleName = createPrivateGlobalForString(
+      M, M.getModuleIdentifier(), /*AllowMerging*/ false, kAsanGenPrefix);
 
   for (size_t i = 0; i < n; i++) {
     GlobalVariable *G = GlobalsToChange[i];
@@ -2646,9 +2641,6 @@ void ModuleAddressSanitizer::instrumentGlobals(IRBuilder<> &IRB, Module &M,
         Constant::getNullValue(IntptrTy),
         ConstantExpr::getPointerCast(ODRIndicator, IntptrTy));
 
-    if (ClInitializers && MD.IsDynInit)
-      HasDynamicallyInitializedGlobals = true;
-
     LLVM_DEBUG(dbgs() << "NEW GLOBAL: " << *NewGlobal << "\n");
 
     Initializers[i] = Initializer;
@@ -2688,7 +2680,7 @@ void ModuleAddressSanitizer::instrumentGlobals(IRBuilder<> &IRB, Module &M,
   }
 
   // Create calls for poisoning before initializers run and unpoisoning after.
-  if (HasDynamicallyInitializedGlobals)
+  if (ClInitializers)
     createInitializerPoisonCalls(M, ModuleName);
 
   LLVM_DEBUG(dbgs() << M);
diff --git a/llvm/test/Instrumentation/AddressSanitizer/instrument_initializer_without_global.ll b/llvm/test/Instrumentation/AddressSanitizer/instrument_initializer_without_global.ll
index c8a6541bacfdfa1..b6ab4aca547a4f4 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/instrument_initializer_without_global.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/instrument_initializer_without_global.ll
@@ -18,7 +18,9 @@ define internal void @__late_ctor() sanitize_address section ".text.startup" {
 ; CHECK-LABEL: define internal void @__late_ctor(
 ; CHECK-SAME: ) #[[ATTR1:[0-9]+]] section ".text.startup" {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    call void @__asan_before_dynamic_init(i64 ptrtoint (ptr @___asan_gen_ to i64))
 ; CHECK-NEXT:    call void @initializer()
+; CHECK-NEXT:    call void @__asan_after_dynamic_init()
 ; CHECK-NEXT:    ret void
 ;
 ; NOINIT-LABEL: define internal void @__late_ctor(

From 1ff630d5c32e73f5346e5cac0392762c11d1323d Mon Sep 17 00:00:00 2001
From: Pradeep Kumar <pradeepku@nvidia.com>
Date: Sat, 17 Aug 2024 10:00:24 +0530
Subject: [PATCH 254/441] [LLVM][NVPTX] Add support for brkpt instruction
 (#104470)

This commit adds NVPTX codegen support for brkpt instruction
(https://docs.nvidia.com/cuda/parallel-thread-execution/#miscellaneous-instructions-brkpt)
with test under CodeGen/NVPTX/brkpt.ll
---
 llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp | 2 ++
 llvm/lib/Target/NVPTX/NVPTXInstrInfo.td     | 2 ++
 llvm/test/CodeGen/NVPTX/brkpt.ll            | 8 ++++++++
 3 files changed, 12 insertions(+)
 create mode 100644 llvm/test/CodeGen/NVPTX/brkpt.ll

diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index 43a3fbf4d1306a9..577141299b94963 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -648,6 +648,8 @@ NVPTXTargetLowering::NVPTXTargetLowering(const NVPTXTargetMachine &TM,
 
   // TRAP can be lowered to PTX trap
   setOperationAction(ISD::TRAP, MVT::Other, Legal);
+  // DEBUGTRAP can be lowered to PTX brkpt
+  setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal);
 
   // Register custom handling for vector loads/stores
   for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
diff --git a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
index b57c86fcf697cdb..1c116c496658dfb 100644
--- a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
+++ b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
@@ -3845,6 +3845,8 @@ def Callseq_End :
 // Emit an `exit` as well to convey to ptxas that `trap` exits the CFG.
 // This won't be necessary in a future version of ptxas.
 def trapinst : NVPTXInst<(outs), (ins), "trap; exit;", [(trap)]>;
+// brkpt instruction
+def debugtrapinst : NVPTXInst<(outs), (ins), "brkpt;", [(debugtrap)]>;
 
 // Call prototype wrapper
 def SDTCallPrototype : SDTypeProfile<0, 1, [SDTCisInt<0>]>;
diff --git a/llvm/test/CodeGen/NVPTX/brkpt.ll b/llvm/test/CodeGen/NVPTX/brkpt.ll
new file mode 100644
index 000000000000000..beb2ff6aed6c9c4
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/brkpt.ll
@@ -0,0 +1,8 @@
+; RUN: llc -o - -march=nvptx64 %s | FileCheck %s
+
+; CHECK-LABEL: .func breakpoint
+define void @breakpoint() {
+  ; CHECK: brkpt;
+  call void @llvm.debugtrap()
+  ret void
+}

From c51578eba1f88750ef189377b1bc848607449309 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Fri, 16 Aug 2024 21:46:39 -0700
Subject: [PATCH 255/441] [SelectionDAG][X86] Use getAllOnesConstant. NFC
 (#104640)

Part of an effort to make getConstant stricter about implicit truncation
when converting uint64_t to APInt.
---
 .../SelectionDAG/LegalizeIntegerTypes.cpp     |  2 +-
 llvm/lib/Target/X86/X86ISelLowering.cpp       | 19 ++++++++++---------
 2 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
index 5b66748297b7ef4..9d78044456e025d 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
@@ -4390,7 +4390,7 @@ void DAGTypeLegalizer::ExpandIntRes_MULFIX(SDNode *N, SDValue &Lo,
 
   SDValue SatMax, SatMin;
   SDValue NVTZero = DAG.getConstant(0, dl, NVT);
-  SDValue NVTNeg1 = DAG.getConstant(-1, dl, NVT);
+  SDValue NVTNeg1 = DAG.getAllOnesConstant(dl, NVT);
   EVT BoolNVT = getSetCCResultType(NVT);
 
   if (!Signed) {
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index df6295a88e13498..e0face6b6a79042 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -23599,7 +23599,7 @@ static SDValue LowerVSETCC(SDValue Op, const X86Subtarget &Subtarget,
 
       if (!FlipSigns && !Invert && ISD::isBuildVectorAllOnes(Op1.getNode())) {
         Op0 = DAG.getBitcast(MVT::v4i32, Op0);
-        Op1 = DAG.getConstant(-1, dl, MVT::v4i32);
+        Op1 = DAG.getAllOnesConstant(dl, MVT::v4i32);
 
         SDValue GT = DAG.getNode(X86ISD::PCMPGT, dl, MVT::v4i32, Op0, Op1);
         static const int MaskHi[] = { 1, 1, 3, 3 };
@@ -24391,7 +24391,7 @@ static SDValue LowerSIGN_EXTEND_Mask(SDValue Op, const SDLoc &dl,
       (Subtarget.hasBWI() && WideEltVT.getSizeInBits() <= 16)) {
     V = DAG.getNode(Op.getOpcode(), dl, WideVT, In);
   } else {
-    SDValue NegOne = DAG.getConstant(-1, dl, WideVT);
+    SDValue NegOne = DAG.getAllOnesConstant(dl, WideVT);
     SDValue Zero = DAG.getConstant(0, dl, WideVT);
     V = DAG.getSelect(dl, WideVT, In, NegOne, Zero);
   }
@@ -26095,7 +26095,7 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
                           Op.getOperand(3));
       } else {
         SDValue GenCF = DAG.getNode(X86ISD::ADD, dl, CFVTs, Op.getOperand(1),
-                                    DAG.getConstant(-1, dl, MVT::i8));
+                                    DAG.getAllOnesConstant(dl, MVT::i8));
         Res = DAG.getNode(IntrData->Opc0, dl, VTs, Op.getOperand(2),
                           Op.getOperand(3), GenCF.getValue(1));
       }
@@ -29378,7 +29378,7 @@ static SDValue LowerShiftByScalarVariable(SDValue Op, SelectionDAG &DAG,
 
         // Create the mask using vXi16 shifts. For shift-rights we need to move
         // the upper byte down before splatting the vXi8 mask.
-        SDValue BitMask = DAG.getConstant(-1, dl, ExtVT);
+        SDValue BitMask = DAG.getAllOnesConstant(dl, ExtVT);
         BitMask = getTargetVShiftNode(LogicalX86Op, dl, ExtVT, BitMask,
                                       BaseShAmt, BaseShAmtIdx, Subtarget, DAG);
         if (Opcode != ISD::SHL)
@@ -50580,7 +50580,7 @@ static SDValue combineAddOrSubToADCOrSBB(bool IsSub, const SDLoc &DL, EVT VT,
     // X - SETAE --> adc X, -1
     return DAG.getNode(IsSub ? X86ISD::ADC : X86ISD::SBB, DL,
                        DAG.getVTList(VT, MVT::i32), X,
-                       DAG.getConstant(-1, DL, VT), EFLAGS);
+                       DAG.getAllOnesConstant(DL, VT), EFLAGS);
   }
 
   if (CC == X86::COND_BE) {
@@ -50601,7 +50601,7 @@ static SDValue combineAddOrSubToADCOrSBB(bool IsSub, const SDLoc &DL, EVT VT,
       SDValue NewEFLAGS = NewSub.getValue(EFLAGS.getResNo());
       return DAG.getNode(IsSub ? X86ISD::ADC : X86ISD::SBB, DL,
                          DAG.getVTList(VT, MVT::i32), X,
-                         DAG.getConstant(-1, DL, VT), NewEFLAGS);
+                         DAG.getAllOnesConstant(DL, VT), NewEFLAGS);
     }
   }
 
@@ -50660,7 +50660,7 @@ static SDValue combineAddOrSubToADCOrSBB(bool IsSub, const SDLoc &DL, EVT VT,
   // X + (Z != 0) --> add X, (zext(setne Z, 0)) --> sbb X, -1, (cmp Z, 1)
   if (CC == X86::COND_NE)
     return DAG.getNode(IsSub ? X86ISD::ADC : X86ISD::SBB, DL, VTs, X,
-                       DAG.getConstant(-1ULL, DL, VT), Cmp1.getValue(1));
+                       DAG.getAllOnesConstant(DL, VT), Cmp1.getValue(1));
 
   // X - (Z == 0) --> sub X, (zext(sete  Z, 0)) --> sbb X, 0, (cmp Z, 1)
   // X + (Z == 0) --> add X, (zext(sete  Z, 0)) --> adc X, 0, (cmp Z, 1)
@@ -50944,8 +50944,9 @@ static SDValue foldXorTruncShiftIntoCmp(SDNode *N, SelectionDAG &DAG) {
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   EVT SetCCResultType = TLI.getSetCCResultType(DAG.getDataLayout(),
                                                *DAG.getContext(), ResultType);
-  SDValue Cond = DAG.getSetCC(DL, SetCCResultType, ShiftOp,
-                              DAG.getConstant(-1, DL, ShiftOpTy), ISD::SETGT);
+  SDValue Cond =
+      DAG.getSetCC(DL, SetCCResultType, ShiftOp,
+                   DAG.getAllOnesConstant(DL, ShiftOpTy), ISD::SETGT);
   if (SetCCResultType != ResultType)
     Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, ResultType, Cond);
   return Cond;

From 34f941a2f96b804dd24c2a25770d899b018339ff Mon Sep 17 00:00:00 2001
From: Vitaly Buka <vitalybuka@google.com>
Date: Fri, 16 Aug 2024 22:43:55 -0700
Subject: [PATCH 256/441] Revert "[asan] Catch `initialization-order-fiasco` in
 modules without globals" (#104665)

Reverts llvm/llvm-project#104621

To many bots are broken, see #104621.
---
 .../TestCases/initialization-bug-no-global.cpp     |  3 +++
 .../Instrumentation/AddressSanitizer.cpp           | 14 +++++++++++---
 .../instrument_initializer_without_global.ll       |  2 --
 3 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/compiler-rt/test/asan/TestCases/initialization-bug-no-global.cpp b/compiler-rt/test/asan/TestCases/initialization-bug-no-global.cpp
index 369449303f3e082..50698b2a7d06fc4 100644
--- a/compiler-rt/test/asan/TestCases/initialization-bug-no-global.cpp
+++ b/compiler-rt/test/asan/TestCases/initialization-bug-no-global.cpp
@@ -1,6 +1,9 @@
 // RUN: %clangxx_asan %min_macos_deployment_target=10.11 -O0 %s %p/Helpers/initialization-bug-extra.cpp -o %t
 // RUN: %env_asan_opts=check_initialization_order=true:strict_init_order=true not %run %t 2>&1 | FileCheck %s
 
+// Not implemented.
+// XFAIL: *
+
 // Do not test with optimization -- the error may be optimized away.
 
 // FIXME: https://code.google.com/p/address-sanitizer/issues/detail?id=186
diff --git a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
index 34366b98aed7ae7..d1bb1334aae6a36 100644
--- a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
@@ -2531,10 +2531,15 @@ void ModuleAddressSanitizer::instrumentGlobals(IRBuilder<> &IRB, Module &M,
   SmallVector<GlobalVariable *, 16> NewGlobals(n);
   SmallVector<Constant *, 16> Initializers(n);
 
+  bool HasDynamicallyInitializedGlobals = false;
+
   // We shouldn't merge same module names, as this string serves as unique
   // module ID in runtime.
-  GlobalVariable *ModuleName = createPrivateGlobalForString(
-      M, M.getModuleIdentifier(), /*AllowMerging*/ false, kAsanGenPrefix);
+  GlobalVariable *ModuleName =
+      n != 0
+          ? createPrivateGlobalForString(M, M.getModuleIdentifier(),
+                                         /*AllowMerging*/ false, kAsanGenPrefix)
+          : nullptr;
 
   for (size_t i = 0; i < n; i++) {
     GlobalVariable *G = GlobalsToChange[i];
@@ -2641,6 +2646,9 @@ void ModuleAddressSanitizer::instrumentGlobals(IRBuilder<> &IRB, Module &M,
         Constant::getNullValue(IntptrTy),
         ConstantExpr::getPointerCast(ODRIndicator, IntptrTy));
 
+    if (ClInitializers && MD.IsDynInit)
+      HasDynamicallyInitializedGlobals = true;
+
     LLVM_DEBUG(dbgs() << "NEW GLOBAL: " << *NewGlobal << "\n");
 
     Initializers[i] = Initializer;
@@ -2680,7 +2688,7 @@ void ModuleAddressSanitizer::instrumentGlobals(IRBuilder<> &IRB, Module &M,
   }
 
   // Create calls for poisoning before initializers run and unpoisoning after.
-  if (ClInitializers)
+  if (HasDynamicallyInitializedGlobals)
     createInitializerPoisonCalls(M, ModuleName);
 
   LLVM_DEBUG(dbgs() << M);
diff --git a/llvm/test/Instrumentation/AddressSanitizer/instrument_initializer_without_global.ll b/llvm/test/Instrumentation/AddressSanitizer/instrument_initializer_without_global.ll
index b6ab4aca547a4f4..c8a6541bacfdfa1 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/instrument_initializer_without_global.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/instrument_initializer_without_global.ll
@@ -18,9 +18,7 @@ define internal void @__late_ctor() sanitize_address section ".text.startup" {
 ; CHECK-LABEL: define internal void @__late_ctor(
 ; CHECK-SAME: ) #[[ATTR1:[0-9]+]] section ".text.startup" {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    call void @__asan_before_dynamic_init(i64 ptrtoint (ptr @___asan_gen_ to i64))
 ; CHECK-NEXT:    call void @initializer()
-; CHECK-NEXT:    call void @__asan_after_dynamic_init()
 ; CHECK-NEXT:    ret void
 ;
 ; NOINIT-LABEL: define internal void @__late_ctor(

From d257cd875873666d250e3fa7e95aafcd099f4e2f Mon Sep 17 00:00:00 2001
From: Vitaly Buka <vitalybuka@google.com>
Date: Fri, 16 Aug 2024 22:56:12 -0700
Subject: [PATCH 257/441] [CodeGen][asan] Use `%t` instead of `cd` in test

---
 clang/test/CodeGen/sanitize-ignorelist-mainfile.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/clang/test/CodeGen/sanitize-ignorelist-mainfile.c b/clang/test/CodeGen/sanitize-ignorelist-mainfile.c
index 84d58b4c018e3a2..419e0417c1d8296 100644
--- a/clang/test/CodeGen/sanitize-ignorelist-mainfile.c
+++ b/clang/test/CodeGen/sanitize-ignorelist-mainfile.c
@@ -1,8 +1,8 @@
 /// Test mainfile in a sanitizer special case list.
-// RUN: rm -rf %t && split-file %s %t && cd %t
-// RUN: %clang_cc1 -emit-llvm -triple x86_64 -fsanitize=address,alignment a.c -o - | FileCheck %s --check-prefixes=CHECK,DEFAULT
-// RUN: %clang_cc1 -emit-llvm -triple x86_64 -fsanitize=address,alignment -fsanitize-ignorelist=a.list a.c -o - | FileCheck %s --check-prefixes=CHECK,IGNORE
-// RUN: %clang_cc1 -emit-llvm -triple x86_64 -fsanitize=address,alignment -fsanitize-ignorelist=b.list a.c -o - | FileCheck %s --check-prefixes=CHECK,IGNORE
+// RUN: rm -rf %t && split-file %s %t
+// RUN: %clang_cc1 -emit-llvm -triple x86_64 -fsanitize=address,alignment %t/a.c -o - | FileCheck %s --check-prefixes=CHECK,DEFAULT
+// RUN: %clang_cc1 -emit-llvm -triple x86_64 -fsanitize=address,alignment -fsanitize-ignorelist=%t/a.list %t/a.c -o - | FileCheck %s --check-prefixes=CHECK,IGNORE
+// RUN: %clang_cc1 -emit-llvm -triple x86_64 -fsanitize=address,alignment -fsanitize-ignorelist=%t/b.list %t/a.c -o - | FileCheck %s --check-prefixes=CHECK,IGNORE
 
 //--- a.list
 mainfile:*a.c

From 2f89c1c76ce22831bd7abebea99202a6d3313110 Mon Sep 17 00:00:00 2001
From: Mariusz Sikora <mariusz.sikora@amd.com>
Date: Sat, 17 Aug 2024 08:11:03 +0200
Subject: [PATCH 258/441] [AMDGPU][NFC] Remove duplicate code by using
 getAddressableLocalMemorySize (#104604)

---
 llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
index 0ca6266cc678b69..5b41a2cd731607b 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
@@ -906,11 +906,7 @@ unsigned getWavefrontSize(const MCSubtargetInfo *STI) {
 }
 
 unsigned getLocalMemorySize(const MCSubtargetInfo *STI) {
-  unsigned BytesPerCU = 0;
-  if (STI->getFeatureBits().test(FeatureLocalMemorySize32768))
-    BytesPerCU = 32768;
-  if (STI->getFeatureBits().test(FeatureLocalMemorySize65536))
-    BytesPerCU = 65536;
+  unsigned BytesPerCU = getAddressableLocalMemorySize(STI);
 
   // "Per CU" really means "per whatever functional block the waves of a
   // workgroup must share". So the effective local memory size is doubled in

From 8041bf48337dcd8f02052c47b398216453e56168 Mon Sep 17 00:00:00 2001
From: Vitaly Buka <vitalybuka@google.com>
Date: Fri, 16 Aug 2024 23:50:44 -0700
Subject: [PATCH 259/441] [NFC][asan] Make 'Module &M' class member

---
 .../Instrumentation/AddressSanitizer.cpp      | 99 ++++++++++---------
 1 file changed, 50 insertions(+), 49 deletions(-)

diff --git a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
index d1bb1334aae6a36..cda35c5af84fff4 100644
--- a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
@@ -719,7 +719,8 @@ struct AddressSanitizer {
                    bool Recover = false, bool UseAfterScope = false,
                    AsanDetectStackUseAfterReturnMode UseAfterReturn =
                        AsanDetectStackUseAfterReturnMode::Runtime)
-      : CompileKernel(ClEnableKasan.getNumOccurrences() > 0 ? ClEnableKasan
+      : M(M),
+        CompileKernel(ClEnableKasan.getNumOccurrences() > 0 ? ClEnableKasan
                                                             : CompileKernel),
         Recover(ClRecover.getNumOccurrences() > 0 ? ClRecover : Recover),
         UseAfterScope(UseAfterScope || ClUseAfterScope),
@@ -803,7 +804,7 @@ struct AddressSanitizer {
 private:
   friend struct FunctionStackPoisoner;
 
-  void initializeCallbacks(Module &M, const TargetLibraryInfo *TLI);
+  void initializeCallbacks(const TargetLibraryInfo *TLI);
 
   bool LooksLikeCodeInBug11395(Instruction *I);
   bool GlobalIsLinkerInitialized(GlobalVariable *G);
@@ -826,6 +827,7 @@ struct AddressSanitizer {
     }
   };
 
+  Module &M;
   LLVMContext *C;
   const DataLayout *DL;
   Triple TargetTriple;
@@ -868,7 +870,8 @@ class ModuleAddressSanitizer {
                          bool UseGlobalsGC = true, bool UseOdrIndicator = true,
                          AsanDtorKind DestructorKind = AsanDtorKind::Global,
                          AsanCtorKind ConstructorKind = AsanCtorKind::Global)
-      : CompileKernel(ClEnableKasan.getNumOccurrences() > 0 ? ClEnableKasan
+      : M(M),
+        CompileKernel(ClEnableKasan.getNumOccurrences() > 0 ? ClEnableKasan
                                                             : CompileKernel),
         InsertVersionCheck(ClInsertVersionCheck.getNumOccurrences() > 0
                                ? ClInsertVersionCheck
@@ -906,45 +909,46 @@ class ModuleAddressSanitizer {
     assert(this->DestructorKind != AsanDtorKind::Invalid);
   }
 
-  bool instrumentModule(Module &);
+  bool instrumentModule();
 
 private:
-  void initializeCallbacks(Module &M);
+  void initializeCallbacks();
 
-  void instrumentGlobals(IRBuilder<> &IRB, Module &M, bool *CtorComdat);
-  void InstrumentGlobalsCOFF(IRBuilder<> &IRB, Module &M,
+  void instrumentGlobals(IRBuilder<> &IRB, bool *CtorComdat);
+  void InstrumentGlobalsCOFF(IRBuilder<> &IRB,
                              ArrayRef<GlobalVariable *> ExtendedGlobals,
                              ArrayRef<Constant *> MetadataInitializers);
-  void instrumentGlobalsELF(IRBuilder<> &IRB, Module &M,
+  void instrumentGlobalsELF(IRBuilder<> &IRB,
                             ArrayRef<GlobalVariable *> ExtendedGlobals,
                             ArrayRef<Constant *> MetadataInitializers,
                             const std::string &UniqueModuleId);
-  void InstrumentGlobalsMachO(IRBuilder<> &IRB, Module &M,
+  void InstrumentGlobalsMachO(IRBuilder<> &IRB,
                               ArrayRef<GlobalVariable *> ExtendedGlobals,
                               ArrayRef<Constant *> MetadataInitializers);
   void
-  InstrumentGlobalsWithMetadataArray(IRBuilder<> &IRB, Module &M,
+  InstrumentGlobalsWithMetadataArray(IRBuilder<> &IRB,
                                      ArrayRef<GlobalVariable *> ExtendedGlobals,
                                      ArrayRef<Constant *> MetadataInitializers);
 
-  GlobalVariable *CreateMetadataGlobal(Module &M, Constant *Initializer,
+  GlobalVariable *CreateMetadataGlobal(Constant *Initializer,
                                        StringRef OriginalName);
   void SetComdatForGlobalMetadata(GlobalVariable *G, GlobalVariable *Metadata,
                                   StringRef InternalSuffix);
-  Instruction *CreateAsanModuleDtor(Module &M);
+  Instruction *CreateAsanModuleDtor();
 
   const GlobalVariable *getExcludedAliasedGlobal(const GlobalAlias &GA) const;
   bool shouldInstrumentGlobal(GlobalVariable *G) const;
   bool ShouldUseMachOGlobalsSection() const;
   StringRef getGlobalMetadataSection() const;
   void poisonOneInitializer(Function &GlobalInit, GlobalValue *ModuleName);
-  void createInitializerPoisonCalls(Module &M, GlobalValue *ModuleName);
+  void createInitializerPoisonCalls(GlobalValue *ModuleName);
   uint64_t getMinRedzoneSizeForGlobal() const {
     return getRedzoneSizeForScale(Mapping.Scale);
   }
   uint64_t getRedzoneSizeForGlobal(uint64_t SizeInBytes) const;
-  int GetAsanVersion(const Module &M) const;
+  int GetAsanVersion() const;
 
+  Module &M;
   bool CompileKernel;
   bool InsertVersionCheck;
   bool Recover;
@@ -1271,7 +1275,7 @@ PreservedAnalyses AddressSanitizerPass::run(Module &M,
     const TargetLibraryInfo &TLI = FAM.getResult<TargetLibraryAnalysis>(F);
     Modified |= FunctionSanitizer.instrumentFunction(F, &TLI);
   }
-  Modified |= ModuleSanitizer.instrumentModule(M);
+  Modified |= ModuleSanitizer.instrumentModule();
   if (!Modified)
     return PreservedAnalyses::all();
 
@@ -1974,7 +1978,7 @@ void ModuleAddressSanitizer::poisonOneInitializer(Function &GlobalInit,
 }
 
 void ModuleAddressSanitizer::createInitializerPoisonCalls(
-    Module &M, GlobalValue *ModuleName) {
+    GlobalValue *ModuleName) {
   GlobalVariable *GV = M.getGlobalVariable("llvm.global_ctors");
   if (!GV)
     return;
@@ -2198,7 +2202,7 @@ StringRef ModuleAddressSanitizer::getGlobalMetadataSection() const {
   llvm_unreachable("unsupported object format");
 }
 
-void ModuleAddressSanitizer::initializeCallbacks(Module &M) {
+void ModuleAddressSanitizer::initializeCallbacks() {
   IRBuilder<> IRB(*C);
 
   // Declare our poisoning and unpoisoning functions.
@@ -2268,7 +2272,7 @@ void ModuleAddressSanitizer::SetComdatForGlobalMetadata(
 // Create a separate metadata global and put it in the appropriate ASan
 // global registration section.
 GlobalVariable *
-ModuleAddressSanitizer::CreateMetadataGlobal(Module &M, Constant *Initializer,
+ModuleAddressSanitizer::CreateMetadataGlobal(Constant *Initializer,
                                              StringRef OriginalName) {
   auto Linkage = TargetTriple.isOSBinFormatMachO()
                      ? GlobalVariable::InternalLinkage
@@ -2283,7 +2287,7 @@ ModuleAddressSanitizer::CreateMetadataGlobal(Module &M, Constant *Initializer,
   return Metadata;
 }
 
-Instruction *ModuleAddressSanitizer::CreateAsanModuleDtor(Module &M) {
+Instruction *ModuleAddressSanitizer::CreateAsanModuleDtor() {
   AsanDtorFunction = Function::createWithDefaultAttr(
       FunctionType::get(Type::getVoidTy(*C), false),
       GlobalValue::InternalLinkage, 0, kAsanModuleDtorName, &M);
@@ -2296,7 +2300,7 @@ Instruction *ModuleAddressSanitizer::CreateAsanModuleDtor(Module &M) {
 }
 
 void ModuleAddressSanitizer::InstrumentGlobalsCOFF(
-    IRBuilder<> &IRB, Module &M, ArrayRef<GlobalVariable *> ExtendedGlobals,
+    IRBuilder<> &IRB, ArrayRef<GlobalVariable *> ExtendedGlobals,
     ArrayRef<Constant *> MetadataInitializers) {
   assert(ExtendedGlobals.size() == MetadataInitializers.size());
   auto &DL = M.getDataLayout();
@@ -2305,8 +2309,7 @@ void ModuleAddressSanitizer::InstrumentGlobalsCOFF(
   for (size_t i = 0; i < ExtendedGlobals.size(); i++) {
     Constant *Initializer = MetadataInitializers[i];
     GlobalVariable *G = ExtendedGlobals[i];
-    GlobalVariable *Metadata =
-        CreateMetadataGlobal(M, Initializer, G->getName());
+    GlobalVariable *Metadata = CreateMetadataGlobal(Initializer, G->getName());
     MDNode *MD = MDNode::get(M.getContext(), ValueAsMetadata::get(G));
     Metadata->setMetadata(LLVMContext::MD_associated, MD);
     MetadataGlobals[i] = Metadata;
@@ -2329,7 +2332,7 @@ void ModuleAddressSanitizer::InstrumentGlobalsCOFF(
 }
 
 void ModuleAddressSanitizer::instrumentGlobalsELF(
-    IRBuilder<> &IRB, Module &M, ArrayRef<GlobalVariable *> ExtendedGlobals,
+    IRBuilder<> &IRB, ArrayRef<GlobalVariable *> ExtendedGlobals,
     ArrayRef<Constant *> MetadataInitializers,
     const std::string &UniqueModuleId) {
   assert(ExtendedGlobals.size() == MetadataInitializers.size());
@@ -2344,7 +2347,7 @@ void ModuleAddressSanitizer::instrumentGlobalsELF(
   for (size_t i = 0; i < ExtendedGlobals.size(); i++) {
     GlobalVariable *G = ExtendedGlobals[i];
     GlobalVariable *Metadata =
-        CreateMetadataGlobal(M, MetadataInitializers[i], G->getName());
+        CreateMetadataGlobal(MetadataInitializers[i], G->getName());
     MDNode *MD = MDNode::get(M.getContext(), ValueAsMetadata::get(G));
     Metadata->setMetadata(LLVMContext::MD_associated, MD);
     MetadataGlobals[i] = Metadata;
@@ -2389,7 +2392,7 @@ void ModuleAddressSanitizer::instrumentGlobalsELF(
   // We also need to unregister globals at the end, e.g., when a shared library
   // gets closed.
   if (DestructorKind != AsanDtorKind::None && !MetadataGlobals.empty()) {
-    IRBuilder<> IrbDtor(CreateAsanModuleDtor(M));
+    IRBuilder<> IrbDtor(CreateAsanModuleDtor());
     IrbDtor.CreateCall(AsanUnregisterElfGlobals,
                        {IRB.CreatePointerCast(RegisteredFlag, IntptrTy),
                         IRB.CreatePointerCast(StartELFMetadata, IntptrTy),
@@ -2398,7 +2401,7 @@ void ModuleAddressSanitizer::instrumentGlobalsELF(
 }
 
 void ModuleAddressSanitizer::InstrumentGlobalsMachO(
-    IRBuilder<> &IRB, Module &M, ArrayRef<GlobalVariable *> ExtendedGlobals,
+    IRBuilder<> &IRB, ArrayRef<GlobalVariable *> ExtendedGlobals,
     ArrayRef<Constant *> MetadataInitializers) {
   assert(ExtendedGlobals.size() == MetadataInitializers.size());
 
@@ -2411,8 +2414,7 @@ void ModuleAddressSanitizer::InstrumentGlobalsMachO(
   for (size_t i = 0; i < ExtendedGlobals.size(); i++) {
     Constant *Initializer = MetadataInitializers[i];
     GlobalVariable *G = ExtendedGlobals[i];
-    GlobalVariable *Metadata =
-        CreateMetadataGlobal(M, Initializer, G->getName());
+    GlobalVariable *Metadata = CreateMetadataGlobal(Initializer, G->getName());
 
     // On recent Mach-O platforms, we emit the global metadata in a way that
     // allows the linker to properly strip dead globals.
@@ -2451,14 +2453,14 @@ void ModuleAddressSanitizer::InstrumentGlobalsMachO(
   // We also need to unregister globals at the end, e.g., when a shared library
   // gets closed.
   if (DestructorKind != AsanDtorKind::None) {
-    IRBuilder<> IrbDtor(CreateAsanModuleDtor(M));
+    IRBuilder<> IrbDtor(CreateAsanModuleDtor());
     IrbDtor.CreateCall(AsanUnregisterImageGlobals,
                        {IRB.CreatePointerCast(RegisteredFlag, IntptrTy)});
   }
 }
 
 void ModuleAddressSanitizer::InstrumentGlobalsWithMetadataArray(
-    IRBuilder<> &IRB, Module &M, ArrayRef<GlobalVariable *> ExtendedGlobals,
+    IRBuilder<> &IRB, ArrayRef<GlobalVariable *> ExtendedGlobals,
     ArrayRef<Constant *> MetadataInitializers) {
   assert(ExtendedGlobals.size() == MetadataInitializers.size());
   unsigned N = ExtendedGlobals.size();
@@ -2482,7 +2484,7 @@ void ModuleAddressSanitizer::InstrumentGlobalsWithMetadataArray(
   // We also need to unregister globals at the end, e.g., when a shared library
   // gets closed.
   if (DestructorKind != AsanDtorKind::None) {
-    IRBuilder<> IrbDtor(CreateAsanModuleDtor(M));
+    IRBuilder<> IrbDtor(CreateAsanModuleDtor());
     IrbDtor.CreateCall(AsanUnregisterGlobals,
                        {IRB.CreatePointerCast(AllGlobals, IntptrTy),
                         ConstantInt::get(IntptrTy, N)});
@@ -2494,7 +2496,7 @@ void ModuleAddressSanitizer::InstrumentGlobalsWithMetadataArray(
 // redzones and inserts this function into llvm.global_ctors.
 // Sets *CtorComdat to true if the global registration code emitted into the
 // asan constructor is comdat-compatible.
-void ModuleAddressSanitizer::instrumentGlobals(IRBuilder<> &IRB, Module &M,
+void ModuleAddressSanitizer::instrumentGlobals(IRBuilder<> &IRB,
                                                bool *CtorComdat) {
   // Build set of globals that are aliased by some GA, where
   // getExcludedAliasedGlobal(GA) returns the relevant GlobalVariable.
@@ -2670,8 +2672,7 @@ void ModuleAddressSanitizer::instrumentGlobals(IRBuilder<> &IRB, Module &M,
     // function will be called. The module destructor is not created when n ==
     // 0.
     *CtorComdat = true;
-    instrumentGlobalsELF(IRB, M, NewGlobals, Initializers,
-                         getUniqueModuleId(&M));
+    instrumentGlobalsELF(IRB, NewGlobals, Initializers, getUniqueModuleId(&M));
   } else if (n == 0) {
     // When UseGlobalsGC is false, COMDAT can still be used if n == 0, because
     // all compile units will have identical module constructor/destructor.
@@ -2679,17 +2680,17 @@ void ModuleAddressSanitizer::instrumentGlobals(IRBuilder<> &IRB, Module &M,
   } else {
     *CtorComdat = false;
     if (UseGlobalsGC && TargetTriple.isOSBinFormatCOFF()) {
-      InstrumentGlobalsCOFF(IRB, M, NewGlobals, Initializers);
+      InstrumentGlobalsCOFF(IRB, NewGlobals, Initializers);
     } else if (UseGlobalsGC && ShouldUseMachOGlobalsSection()) {
-      InstrumentGlobalsMachO(IRB, M, NewGlobals, Initializers);
+      InstrumentGlobalsMachO(IRB, NewGlobals, Initializers);
     } else {
-      InstrumentGlobalsWithMetadataArray(IRB, M, NewGlobals, Initializers);
+      InstrumentGlobalsWithMetadataArray(IRB, NewGlobals, Initializers);
     }
   }
 
   // Create calls for poisoning before initializers run and unpoisoning after.
   if (HasDynamicallyInitializedGlobals)
-    createInitializerPoisonCalls(M, ModuleName);
+    createInitializerPoisonCalls(ModuleName);
 
   LLVM_DEBUG(dbgs() << M);
 }
@@ -2719,7 +2720,7 @@ ModuleAddressSanitizer::getRedzoneSizeForGlobal(uint64_t SizeInBytes) const {
   return RZ;
 }
 
-int ModuleAddressSanitizer::GetAsanVersion(const Module &M) const {
+int ModuleAddressSanitizer::GetAsanVersion() const {
   int LongSize = M.getDataLayout().getPointerSizeInBits();
   bool isAndroid = Triple(M.getTargetTriple()).isAndroid();
   int Version = 8;
@@ -2729,8 +2730,8 @@ int ModuleAddressSanitizer::GetAsanVersion(const Module &M) const {
   return Version;
 }
 
-bool ModuleAddressSanitizer::instrumentModule(Module &M) {
-  initializeCallbacks(M);
+bool ModuleAddressSanitizer::instrumentModule() {
+  initializeCallbacks();
 
   // Create a module constructor. A destructor is created lazily because not all
   // platforms, and not all modules need it.
@@ -2740,13 +2741,13 @@ bool ModuleAddressSanitizer::instrumentModule(Module &M) {
       // need the init and version check calls.
       AsanCtorFunction = createSanitizerCtor(M, kAsanModuleCtorName);
     } else {
-      std::string AsanVersion = std::to_string(GetAsanVersion(M));
+      std::string AsanVersion = std::to_string(GetAsanVersion());
       std::string VersionCheckName =
           InsertVersionCheck ? (kAsanVersionCheckNamePrefix + AsanVersion) : "";
       std::tie(AsanCtorFunction, std::ignore) =
-          createSanitizerCtorAndInitFunctions(M, kAsanModuleCtorName,
-                                              kAsanInitName, /*InitArgTypes=*/{},
-                                              /*InitArgs=*/{}, VersionCheckName);
+          createSanitizerCtorAndInitFunctions(
+              M, kAsanModuleCtorName, kAsanInitName, /*InitArgTypes=*/{},
+              /*InitArgs=*/{}, VersionCheckName);
     }
   }
 
@@ -2755,10 +2756,10 @@ bool ModuleAddressSanitizer::instrumentModule(Module &M) {
     assert(AsanCtorFunction || ConstructorKind == AsanCtorKind::None);
     if (AsanCtorFunction) {
       IRBuilder<> IRB(AsanCtorFunction->getEntryBlock().getTerminator());
-      instrumentGlobals(IRB, M, &CtorComdat);
+      instrumentGlobals(IRB, &CtorComdat);
     } else {
       IRBuilder<> IRB(*C);
-      instrumentGlobals(IRB, M, &CtorComdat);
+      instrumentGlobals(IRB, &CtorComdat);
     }
   }
 
@@ -2786,7 +2787,7 @@ bool ModuleAddressSanitizer::instrumentModule(Module &M) {
   return true;
 }
 
-void AddressSanitizer::initializeCallbacks(Module &M, const TargetLibraryInfo *TLI) {
+void AddressSanitizer::initializeCallbacks(const TargetLibraryInfo *TLI) {
   IRBuilder<> IRB(*C);
   // Create __asan_report* callbacks.
   // IsWrite, TypeSize and Exp are encoded in the function name.
@@ -2970,7 +2971,7 @@ bool AddressSanitizer::instrumentFunction(Function &F,
 
   LLVM_DEBUG(dbgs() << "ASAN instrumenting:\n" << F << "\n");
 
-  initializeCallbacks(*F.getParent(), TLI);
+  initializeCallbacks(TLI);
 
   FunctionStateRAII CleanupObj(this);
 

From 067f2e9f18e1c92496202dc37428a89945f57c22 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Sat, 17 Aug 2024 00:02:20 -0700
Subject: [PATCH 260/441] [SelectionDAG] Use
 getSignedConstant/getAllOnesConstant.

---
 llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp          | 2 +-
 llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp | 2 +-
 llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp         | 6 +++---
 llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp  | 6 ++++--
 llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp       | 6 +++---
 5 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index e75db2950579f6c..8064cc7963b7ac4 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -9561,7 +9561,7 @@ SDValue DAGCombiner::visitXOR(SDNode *N) {
   // A rotate left of ~1 is a nice way of achieving the desired result.
   if (TLI.isOperationLegalOrCustom(ISD::ROTL, VT) && N0Opcode == ISD::SHL &&
       isAllOnesConstant(N1) && isOneConstant(N0.getOperand(0))) {
-    return DAG.getNode(ISD::ROTL, DL, VT, DAG.getConstant(~1, DL, VT),
+    return DAG.getNode(ISD::ROTL, DL, VT, DAG.getSignedConstant(~1, DL, VT),
                        N0.getOperand(1));
   }
 
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
index 9d78044456e025d..c19a5a4995627a0 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
@@ -3302,7 +3302,7 @@ void DAGTypeLegalizer::ExpandIntRes_MINMAX(SDNode *N,
     SDValue HiNeg =
         DAG.getSetCC(DL, CCT, LHSH, DAG.getConstant(0, DL, NVT), ISD::SETLT);
     if (N->getOpcode() == ISD::SMIN) {
-      Lo = DAG.getSelect(DL, NVT, HiNeg, LHSL, DAG.getConstant(-1, DL, NVT));
+      Lo = DAG.getSelect(DL, NVT, HiNeg, LHSL, DAG.getAllOnesConstant(DL, NVT));
     } else {
       Lo = DAG.getSelect(DL, NVT, HiNeg, DAG.getConstant(0, DL, NVT), LHSL);
     }
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 2401bb5a347e2d0..2c6f7c09e5ca636 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -2443,9 +2443,9 @@ SDValue SelectionDAG::expandVAArg(SDNode *Node) {
     VAList = getNode(ISD::ADD, dl, VAList.getValueType(), VAList,
                      getConstant(MA->value() - 1, dl, VAList.getValueType()));
 
-    VAList =
-        getNode(ISD::AND, dl, VAList.getValueType(), VAList,
-                getConstant(-(int64_t)MA->value(), dl, VAList.getValueType()));
+    VAList = getNode(
+        ISD::AND, dl, VAList.getValueType(), VAList,
+        getSignedConstant(-(int64_t)MA->value(), dl, VAList.getValueType()));
   }
 
   // Increment the pointer, VAList, to the next vaarg
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 87f6b4388a728c7..b0f26b54fe42536 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -12532,8 +12532,10 @@ void SelectionDAGBuilder::visitVectorSplice(const CallInst &I) {
 
   // VECTOR_SHUFFLE doesn't support a scalable mask so use a dedicated node.
   if (VT.isScalableVector()) {
-    setValue(&I, DAG.getNode(ISD::VECTOR_SPLICE, DL, VT, V1, V2,
-                             DAG.getVectorIdxConstant(Imm, DL)));
+    setValue(
+        &I, DAG.getNode(ISD::VECTOR_SPLICE, DL, VT, V1, V2,
+                        DAG.getSignedConstant(
+                            Imm, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))));
     return;
   }
 
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 3c499ca60078082..0bf84ebb90db599 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -9185,8 +9185,8 @@ SDValue TargetLowering::expandVPCTLZ(SDNode *Node, SelectionDAG &DAG) const {
                      DAG.getNode(ISD::VP_SRL, dl, VT, Op, Tmp, Mask, VL), Mask,
                      VL);
   }
-  Op = DAG.getNode(ISD::VP_XOR, dl, VT, Op, DAG.getConstant(-1, dl, VT), Mask,
-                   VL);
+  Op = DAG.getNode(ISD::VP_XOR, dl, VT, Op, DAG.getAllOnesConstant(dl, VT),
+                   Mask, VL);
   return DAG.getNode(ISD::VP_CTPOP, dl, VT, Op, Mask, VL);
 }
 
@@ -9299,7 +9299,7 @@ SDValue TargetLowering::expandVPCTTZ(SDNode *Node, SelectionDAG &DAG) const {
 
   // Same as the vector part of expandCTTZ, use: popcount(~x & (x - 1))
   SDValue Not = DAG.getNode(ISD::VP_XOR, dl, VT, Op,
-                            DAG.getConstant(-1, dl, VT), Mask, VL);
+                            DAG.getAllOnesConstant(dl, VT), Mask, VL);
   SDValue MinusOne = DAG.getNode(ISD::VP_SUB, dl, VT, Op,
                                  DAG.getConstant(1, dl, VT), Mask, VL);
   SDValue Tmp = DAG.getNode(ISD::VP_AND, dl, VT, Not, MinusOne, Mask, VL);

From e6ceb29ab6bb5632cac79e99da4d248e9bb7d378 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Sat, 17 Aug 2024 00:14:28 -0700
Subject: [PATCH 261/441] [RISCV] Use getAllOnesConstant/getSignedConstant.

---
 llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp   | 87 ++++++++++---------
 llvm/lib/Target/RISCV/RISCVISelLowering.cpp   | 72 +++++++--------
 llvm/lib/Target/RISCV/RISCVInstrInfo.td       | 16 ++--
 .../Target/RISCV/RISCVInstrInfoVPseudos.td    |  4 +-
 llvm/lib/Target/RISCV/RISCVInstrInfoZb.td     | 12 +--
 5 files changed, 101 insertions(+), 90 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
index d95cb3cbe3e4aba..11210e6cec177ff 100644
--- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
@@ -177,7 +177,8 @@ static SDValue selectImmSeq(SelectionDAG *CurDAG, const SDLoc &DL, const MVT VT,
                             RISCVMatInt::InstSeq &Seq) {
   SDValue SrcReg = CurDAG->getRegister(RISCV::X0, VT);
   for (const RISCVMatInt::Inst &Inst : Seq) {
-    SDValue SDImm = CurDAG->getTargetConstant(Inst.getImm(), DL, VT);
+    SDValue SDImm =
+        CurDAG->getSignedConstant(Inst.getImm(), DL, VT, /*isTarget=*/true);
     SDNode *Result = nullptr;
     switch (Inst.getOpndKind()) {
     case RISCVMatInt::Imm:
@@ -208,10 +209,10 @@ static SDValue selectImm(SelectionDAG *CurDAG, const SDLoc &DL, const MVT VT,
 
   // Use a rematerializable pseudo instruction for short sequences if enabled.
   if (Seq.size() == 2 && UsePseudoMovImm)
-    return SDValue(
-        CurDAG->getMachineNode(RISCV::PseudoMovImm, DL, VT,
-                               CurDAG->getTargetConstant(Imm, DL, VT)),
-        0);
+    return SDValue(CurDAG->getMachineNode(RISCV::PseudoMovImm, DL, VT,
+                                          CurDAG->getSignedConstant(
+                                              Imm, DL, VT, /*isTarget=*/true)),
+                   0);
 
   // See if we can create this constant as (ADD (SLLI X, C), X) where X is at
   // worst an LUI+ADDIW. This will require an extra register, but avoids a
@@ -670,9 +671,9 @@ bool RISCVDAGToDAGISel::tryShrinkShlLogicImm(SDNode *Node) {
 
   unsigned ShOpc = SignExt ? RISCV::SLLIW : RISCV::SLLI;
 
-  SDNode *BinOp =
-      CurDAG->getMachineNode(BinOpc, DL, VT, Shift.getOperand(0),
-                             CurDAG->getTargetConstant(ShiftedVal, DL, VT));
+  SDNode *BinOp = CurDAG->getMachineNode(
+      BinOpc, DL, VT, Shift.getOperand(0),
+      CurDAG->getSignedConstant(ShiftedVal, DL, VT, /*isTarget=*/true));
   SDNode *SLLI =
       CurDAG->getMachineNode(ShOpc, DL, VT, SDValue(BinOp, 0),
                              CurDAG->getTargetConstant(ShAmt, DL, VT));
@@ -802,7 +803,8 @@ bool RISCVDAGToDAGISel::tryIndexedLoad(SDNode *Node) {
 
   EVT Ty = Ld->getOffset().getValueType();
   SDValue Ops[] = {Ld->getBasePtr(),
-                   CurDAG->getTargetConstant(Offset >> Shift, SDLoc(Node), Ty),
+                   CurDAG->getSignedConstant(Offset >> Shift, SDLoc(Node), Ty,
+                                             /*isTarget=*/true),
                    CurDAG->getTargetConstant(Shift, SDLoc(Node), Ty),
                    Ld->getChain()};
   SDNode *New = CurDAG->getMachineNode(Opcode, SDLoc(Node), Ld->getValueType(0),
@@ -2486,8 +2488,8 @@ bool RISCVDAGToDAGISel::SelectFrameAddrRegImm(SDValue Addr, SDValue &Base,
     if (isInt<12>(CVal)) {
       Base = CurDAG->getTargetFrameIndex(FIN->getIndex(),
                                          Subtarget->getXLenVT());
-      Offset = CurDAG->getTargetConstant(CVal, SDLoc(Addr),
-                                         Subtarget->getXLenVT());
+      Offset = CurDAG->getSignedConstant(
+          CVal, SDLoc(Addr), Subtarget->getXLenVT(), /*isTarget=*/true);
       return true;
     }
   }
@@ -2523,7 +2525,7 @@ static bool selectConstantAddr(SelectionDAG *CurDAG, const SDLoc &DL,
     } else {
       Base = CurDAG->getRegister(RISCV::X0, VT);
     }
-    Offset = CurDAG->getTargetConstant(Lo12, DL, VT);
+    Offset = CurDAG->getSignedConstant(Lo12, DL, VT, /*isTarget=*/true);
     return true;
   }
 
@@ -2543,7 +2545,7 @@ static bool selectConstantAddr(SelectionDAG *CurDAG, const SDLoc &DL,
   assert(!Seq.empty() && "Expected more instructions in sequence");
 
   Base = selectImmSeq(CurDAG, DL, VT, Seq);
-  Offset = CurDAG->getTargetConstant(Lo12, DL, VT);
+  Offset = CurDAG->getSignedConstant(Lo12, DL, VT, /*isTarget=*/true);
   return true;
 }
 
@@ -2668,7 +2670,7 @@ bool RISCVDAGToDAGISel::SelectAddrRegImm(SDValue Addr, SDValue &Base,
 
       if (auto *FIN = dyn_cast<FrameIndexSDNode>(Base))
         Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), VT);
-      Offset = CurDAG->getTargetConstant(CVal, DL, VT);
+      Offset = CurDAG->getSignedConstant(CVal, DL, VT, /*isTarget=*/true);
       return true;
     }
   }
@@ -2685,10 +2687,11 @@ bool RISCVDAGToDAGISel::SelectAddrRegImm(SDValue Addr, SDValue &Base,
     if (isInt<12>(CVal / 2) && isInt<12>(CVal - CVal / 2)) {
       int64_t Adj = CVal < 0 ? -2048 : 2047;
       Base = SDValue(
-          CurDAG->getMachineNode(RISCV::ADDI, DL, VT, Addr.getOperand(0),
-                                 CurDAG->getTargetConstant(Adj, DL, VT)),
+          CurDAG->getMachineNode(
+              RISCV::ADDI, DL, VT, Addr.getOperand(0),
+              CurDAG->getSignedConstant(Adj, DL, VT, /*isTarget=*/true)),
           0);
-      Offset = CurDAG->getTargetConstant(CVal - Adj, DL, VT);
+      Offset = CurDAG->getSignedConstant(CVal - Adj, DL, VT, /*isTarget=*/true);
       return true;
     }
 
@@ -2741,7 +2744,7 @@ bool RISCVDAGToDAGISel::SelectAddrRegImmLsb00000(SDValue Addr, SDValue &Base,
 
       if (auto *FIN = dyn_cast<FrameIndexSDNode>(Base))
         Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), VT);
-      Offset = CurDAG->getTargetConstant(CVal, DL, VT);
+      Offset = CurDAG->getSignedConstant(CVal, DL, VT, /*isTarget=*/true);
       return true;
     }
   }
@@ -2759,9 +2762,10 @@ bool RISCVDAGToDAGISel::SelectAddrRegImmLsb00000(SDValue Addr, SDValue &Base,
       int64_t AdjustedOffset = CVal - Adj;
       Base = SDValue(CurDAG->getMachineNode(
                          RISCV::ADDI, DL, VT, Addr.getOperand(0),
-                         CurDAG->getTargetConstant(AdjustedOffset, DL, VT)),
+                         CurDAG->getSignedConstant(AdjustedOffset, DL, VT,
+                                                   /*isTarget=*/true)),
                      0);
-      Offset = CurDAG->getTargetConstant(Adj, DL, VT);
+      Offset = CurDAG->getSignedConstant(Adj, DL, VT, /*isTarget=*/true);
       return true;
     }
 
@@ -2857,9 +2861,9 @@ bool RISCVDAGToDAGISel::selectShiftMask(SDValue N, unsigned ShiftWidth,
     if (Imm % ShiftWidth == ShiftWidth - 1) {
       SDLoc DL(ShAmt);
       EVT VT = ShAmt.getValueType();
-      MachineSDNode *Not =
-          CurDAG->getMachineNode(RISCV::XORI, DL, VT, ShAmt.getOperand(1),
-                                 CurDAG->getTargetConstant(-1, DL, VT));
+      MachineSDNode *Not = CurDAG->getMachineNode(
+          RISCV::XORI, DL, VT, ShAmt.getOperand(1),
+          CurDAG->getAllOnesConstant(DL, VT, /*isTarget=*/true));
       ShAmt = SDValue(Not, 0);
       return true;
     }
@@ -2906,21 +2910,21 @@ bool RISCVDAGToDAGISel::selectSETCC(SDValue N, ISD::CondCode ExpectedCCVal,
     // If the RHS is -2048, we can use xori to produce 0 if the LHS is -2048 and
     // non-zero otherwise.
     if (CVal == -2048) {
-      Val =
-          SDValue(CurDAG->getMachineNode(
-                      RISCV::XORI, DL, N->getValueType(0), LHS,
-                      CurDAG->getTargetConstant(CVal, DL, N->getValueType(0))),
-                  0);
+      Val = SDValue(CurDAG->getMachineNode(
+                        RISCV::XORI, DL, N->getValueType(0), LHS,
+                        CurDAG->getSignedConstant(CVal, DL, N->getValueType(0),
+                                                  /*isTarget=*/true)),
+                    0);
       return true;
     }
     // If the RHS is [-2047,2048], we can use addi with -RHS to produce 0 if the
     // LHS is equal to the RHS and non-zero otherwise.
     if (isInt<12>(CVal) || CVal == 2048) {
-      Val =
-          SDValue(CurDAG->getMachineNode(
-                      RISCV::ADDI, DL, N->getValueType(0), LHS,
-                      CurDAG->getTargetConstant(-CVal, DL, N->getValueType(0))),
-                  0);
+      Val = SDValue(CurDAG->getMachineNode(
+                        RISCV::ADDI, DL, N->getValueType(0), LHS,
+                        CurDAG->getSignedConstant(-CVal, DL, N->getValueType(0),
+                                                  /*isTarget=*/true)),
+                    0);
       return true;
     }
   }
@@ -3326,7 +3330,8 @@ bool RISCVDAGToDAGISel::selectSimm5Shl2(SDValue N, SDValue &Simm5,
       return false;
 
     EVT Ty = N->getValueType(0);
-    Simm5 = CurDAG->getTargetConstant(Offset >> Shift, SDLoc(N), Ty);
+    Simm5 = CurDAG->getSignedConstant(Offset >> Shift, SDLoc(N), Ty,
+                                      /*isTarget=*/true);
     Shl2 = CurDAG->getTargetConstant(Shift, SDLoc(N), Ty);
     return true;
   }
@@ -3343,16 +3348,16 @@ bool RISCVDAGToDAGISel::selectVLOp(SDValue N, SDValue &VL) {
                                    N->getValueType(0));
   } else if (C && C->isAllOnes()) {
     // Treat all ones as VLMax.
-    VL = CurDAG->getTargetConstant(RISCV::VLMaxSentinel, SDLoc(N),
-                                   N->getValueType(0));
+    VL = CurDAG->getSignedConstant(RISCV::VLMaxSentinel, SDLoc(N),
+                                   N->getValueType(0), /*isTarget=*/true);
   } else if (isa<RegisterSDNode>(N) &&
              cast<RegisterSDNode>(N)->getReg() == RISCV::X0) {
     // All our VL operands use an operand that allows GPRNoX0 or an immediate
     // as the register class. Convert X0 to a special immediate to pass the
     // MachineVerifier. This is recognized specially by the vsetvli insertion
     // pass.
-    VL = CurDAG->getTargetConstant(RISCV::VLMaxSentinel, SDLoc(N),
-                                   N->getValueType(0));
+    VL = CurDAG->getSignedConstant(RISCV::VLMaxSentinel, SDLoc(N),
+                                   N->getValueType(0), /*isTarget=*/true);
   } else {
     VL = N;
   }
@@ -3410,7 +3415,8 @@ static bool selectVSplatImmHelper(SDValue N, SDValue &SplatVal,
   if (!ValidateImm(SplatImm))
     return false;
 
-  SplatVal = DAG.getTargetConstant(SplatImm, SDLoc(N), Subtarget.getXLenVT());
+  SplatVal = DAG.getSignedConstant(SplatImm, SDLoc(N), Subtarget.getXLenVT(),
+                                   /*isTarget=*/true);
   return true;
 }
 
@@ -3504,7 +3510,8 @@ bool RISCVDAGToDAGISel::selectRVVSimm5(SDValue N, unsigned Width,
     if (!isInt<5>(ImmVal))
       return false;
 
-    Imm = CurDAG->getTargetConstant(ImmVal, SDLoc(N), Subtarget->getXLenVT());
+    Imm = CurDAG->getSignedConstant(ImmVal, SDLoc(N), Subtarget->getXLenVT(),
+                                    /*isTarget=*/true);
     return true;
   }
 
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index da937960f1ea76a..62980ae71d60b69 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -3596,7 +3596,7 @@ static SDValue lowerBuildVectorOfConstants(SDValue Op, SelectionDAG &DAG,
       if (I % NumViaIntegerBits == 0 || I == NumElts) {
         if (NumViaIntegerBits <= 32)
           Bits = SignExtend64<32>(Bits);
-        SDValue Elt = DAG.getConstant(Bits, DL, XLenVT);
+        SDValue Elt = DAG.getSignedConstant(Bits, DL, XLenVT);
         Elts[IntegerEltIdx] = Elt;
         Bits = 0;
         BitPos = 0;
@@ -3675,7 +3675,7 @@ static SDValue lowerBuildVectorOfConstants(SDValue Op, SelectionDAG &DAG,
       VID = convertFromScalableVector(VIDVT, VID, DAG, Subtarget);
       if ((StepOpcode == ISD::MUL && SplatStepVal != 1) ||
           (StepOpcode == ISD::SHL && SplatStepVal != 0)) {
-        SDValue SplatStep = DAG.getConstant(SplatStepVal, DL, VIDVT);
+        SDValue SplatStep = DAG.getSignedConstant(SplatStepVal, DL, VIDVT);
         VID = DAG.getNode(StepOpcode, DL, VIDVT, VID, SplatStep);
       }
       if (StepDenominator != 1) {
@@ -3684,7 +3684,7 @@ static SDValue lowerBuildVectorOfConstants(SDValue Op, SelectionDAG &DAG,
         VID = DAG.getNode(ISD::SRL, DL, VIDVT, VID, SplatStep);
       }
       if (Addend != 0 || Negate) {
-        SDValue SplatAddend = DAG.getConstant(Addend, DL, VIDVT);
+        SDValue SplatAddend = DAG.getSignedConstant(Addend, DL, VIDVT);
         VID = DAG.getNode(Negate ? ISD::SUB : ISD::ADD, DL, VIDVT, SplatAddend,
                           VID);
       }
@@ -3723,12 +3723,13 @@ static SDValue lowerBuildVectorOfConstants(SDValue Op, SelectionDAG &DAG,
 
     // On RV64, sign-extend from 32 to 64 bits where possible in order to
     // achieve better constant materializion.
-    if (Subtarget.is64Bit() && ViaIntVT == MVT::i32)
+    // On RV32, we need to sign-extend to use getSignedConstant.
+    if (ViaIntVT == MVT::i32)
       SplatValue = SignExtend64<32>(SplatValue);
 
     SDValue Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, ViaVecVT,
                               DAG.getUNDEF(ViaVecVT),
-                              DAG.getConstant(SplatValue, DL, XLenVT),
+                              DAG.getSignedConstant(SplatValue, DL, XLenVT),
                               DAG.getVectorIdxConstant(0, DL));
     if (ViaVecLen != 1)
       Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL,
@@ -3780,7 +3781,8 @@ static SDValue lowerBuildVectorOfConstants(SDValue Op, SelectionDAG &DAG,
 
     // On RV64, sign-extend from 32 to 64 bits where possible in order to
     // achieve better constant materializion.
-    if (Subtarget.is64Bit() && ViaIntVT == MVT::i32)
+    // On RV32, we need to sign-extend to use getSignedConstant.
+    if (ViaIntVT == MVT::i32)
       SplatValue = SignExtend64<32>(SplatValue);
 
     // Since we can't introduce illegal i64 types at this stage, we can only
@@ -3797,7 +3799,7 @@ static SDValue lowerBuildVectorOfConstants(SDValue Op, SelectionDAG &DAG,
       SDValue Splat =
           DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ViaContainerVT,
                       DAG.getUNDEF(ViaContainerVT),
-                      DAG.getConstant(SplatValue, DL, XLenVT), ViaVL);
+                      DAG.getSignedConstant(SplatValue, DL, XLenVT), ViaVL);
       Splat = convertFromScalableVector(ViaVecVT, Splat, DAG, Subtarget);
       if (ViaVecLen != RequiredVL)
         Splat = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL,
@@ -6810,7 +6812,7 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
           // Using getSetCCSwappedOperands will convert SET(U)GT->SET(U)LT.
           CCVal = ISD::getSetCCSwappedOperands(CCVal);
           SDValue SetCC = DAG.getSetCC(
-              DL, VT, LHS, DAG.getConstant(Imm + 1, DL, OpVT), CCVal);
+              DL, VT, LHS, DAG.getSignedConstant(Imm + 1, DL, OpVT), CCVal);
           return DAG.getLogicalNOT(DL, SetCC, VT);
         }
       }
@@ -7760,8 +7762,9 @@ SDValue RISCVTargetLowering::lowerFRAMEADDR(SDValue Op,
   unsigned Depth = Op.getConstantOperandVal(0);
   while (Depth--) {
     int Offset = -(XLenInBytes * 2);
-    SDValue Ptr = DAG.getNode(ISD::ADD, DL, VT, FrameAddr,
-                              DAG.getIntPtrConstant(Offset, DL));
+    SDValue Ptr = DAG.getNode(
+        ISD::ADD, DL, VT, FrameAddr,
+        DAG.getSignedConstant(Offset, DL, getPointerTy(DAG.getDataLayout())));
     FrameAddr =
         DAG.getLoad(VT, DL, DAG.getEntryNode(), Ptr, MachinePointerInfo());
   }
@@ -7786,7 +7789,7 @@ SDValue RISCVTargetLowering::lowerRETURNADDR(SDValue Op,
   if (Depth) {
     int Off = -XLenInBytes;
     SDValue FrameAddr = lowerFRAMEADDR(Op, DAG);
-    SDValue Offset = DAG.getConstant(Off, DL, VT);
+    SDValue Offset = DAG.getSignedConstant(Off, DL, VT);
     return DAG.getLoad(VT, DL, DAG.getEntryNode(),
                        DAG.getNode(ISD::ADD, DL, VT, FrameAddr, Offset),
                        MachinePointerInfo());
@@ -7815,7 +7818,7 @@ SDValue RISCVTargetLowering::lowerShiftLeftParts(SDValue Op,
 
   SDValue Zero = DAG.getConstant(0, DL, VT);
   SDValue One = DAG.getConstant(1, DL, VT);
-  SDValue MinusXLen = DAG.getConstant(-(int)Subtarget.getXLen(), DL, VT);
+  SDValue MinusXLen = DAG.getSignedConstant(-(int)Subtarget.getXLen(), DL, VT);
   SDValue XLenMinus1 = DAG.getConstant(Subtarget.getXLen() - 1, DL, VT);
   SDValue ShamtMinusXLen = DAG.getNode(ISD::ADD, DL, VT, Shamt, MinusXLen);
   SDValue XLenMinus1Shamt = DAG.getNode(ISD::SUB, DL, VT, XLenMinus1, Shamt);
@@ -7865,7 +7868,7 @@ SDValue RISCVTargetLowering::lowerShiftRightParts(SDValue Op, SelectionDAG &DAG,
 
   SDValue Zero = DAG.getConstant(0, DL, VT);
   SDValue One = DAG.getConstant(1, DL, VT);
-  SDValue MinusXLen = DAG.getConstant(-(int)Subtarget.getXLen(), DL, VT);
+  SDValue MinusXLen = DAG.getSignedConstant(-(int)Subtarget.getXLen(), DL, VT);
   SDValue XLenMinus1 = DAG.getConstant(Subtarget.getXLen() - 1, DL, VT);
   SDValue ShamtMinusXLen = DAG.getNode(ISD::ADD, DL, VT, Shamt, MinusXLen);
   SDValue XLenMinus1Shamt = DAG.getNode(ISD::SUB, DL, VT, XLenMinus1, Shamt);
@@ -7958,7 +7961,7 @@ SDValue RISCVTargetLowering::lowerVectorMaskExt(SDValue Op, SelectionDAG &DAG,
 
   if (VecVT.isScalableVector()) {
     SDValue SplatZero = DAG.getConstant(0, DL, VecVT);
-    SDValue SplatTrueVal = DAG.getConstant(ExtTrueVal, DL, VecVT);
+    SDValue SplatTrueVal = DAG.getSignedConstant(ExtTrueVal, DL, VecVT);
     return DAG.getNode(ISD::VSELECT, DL, VecVT, Src, SplatTrueVal, SplatZero);
   }
 
@@ -7972,7 +7975,7 @@ SDValue RISCVTargetLowering::lowerVectorMaskExt(SDValue Op, SelectionDAG &DAG,
 
   MVT XLenVT = Subtarget.getXLenVT();
   SDValue SplatZero = DAG.getConstant(0, DL, XLenVT);
-  SDValue SplatTrueVal = DAG.getConstant(ExtTrueVal, DL, XLenVT);
+  SDValue SplatTrueVal = DAG.getSignedConstant(ExtTrueVal, DL, XLenVT);
 
   SplatZero = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
                           DAG.getUNDEF(ContainerVT), SplatZero, VL);
@@ -8359,7 +8362,7 @@ SDValue RISCVTargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op,
     const auto *CVal = cast<ConstantSDNode>(Val);
     if (isInt<32>(CVal->getSExtValue())) {
       IsLegalInsert = true;
-      Val = DAG.getConstant(CVal->getSExtValue(), DL, MVT::i32);
+      Val = DAG.getSignedConstant(CVal->getSExtValue(), DL, MVT::i32);
     }
   }
 
@@ -10986,7 +10989,7 @@ SDValue RISCVTargetLowering::lowerVPExtMaskOp(SDValue Op,
   SDValue ZeroSplat = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
                                   DAG.getUNDEF(ContainerVT), Zero, VL);
 
-  SDValue SplatValue = DAG.getConstant(
+  SDValue SplatValue = DAG.getSignedConstant(
       Op.getOpcode() == ISD::VP_ZERO_EXTEND ? 1 : -1, DL, XLenVT);
   SDValue Splat = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
                               DAG.getUNDEF(ContainerVT), SplatValue, VL);
@@ -11116,7 +11119,7 @@ SDValue RISCVTargetLowering::lowerVPFPIntConvOp(SDValue Op,
         SDValue Zero = DAG.getConstant(0, DL, XLenVT);
         SDValue ZeroSplat = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, IntVT,
                                         DAG.getUNDEF(IntVT), Zero, VL);
-        SDValue One = DAG.getConstant(
+        SDValue One = DAG.getSignedConstant(
             RISCVISDExtOpc == RISCVISD::VZEXT_VL ? 1 : -1, DL, XLenVT);
         SDValue OneSplat = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, IntVT,
                                        DAG.getUNDEF(IntVT), One, VL);
@@ -12998,10 +13001,10 @@ static SDValue transformAddImmMulImm(SDNode *N, SelectionDAG &DAG,
   // Build new nodes (add (mul (add x, c1/c0), c0), c1%c0).
   SDLoc DL(N);
   SDValue New0 = DAG.getNode(ISD::ADD, DL, VT, N0->getOperand(0),
-                             DAG.getConstant(CA, DL, VT));
+                             DAG.getSignedConstant(CA, DL, VT));
   SDValue New1 =
-      DAG.getNode(ISD::MUL, DL, VT, New0, DAG.getConstant(C0, DL, VT));
-  return DAG.getNode(ISD::ADD, DL, VT, New1, DAG.getConstant(CB, DL, VT));
+      DAG.getNode(ISD::MUL, DL, VT, New0, DAG.getSignedConstant(C0, DL, VT));
+  return DAG.getNode(ISD::ADD, DL, VT, New1, DAG.getSignedConstant(CB, DL, VT));
 }
 
 // add (zext, zext) -> zext (add (zext, zext))
@@ -15885,10 +15888,11 @@ static SDValue performCONCAT_VECTORSCombine(SDNode *N, SelectionDAG &DAG,
     return SDValue();
 
   auto [StrideVariant, MustNegateStride] = *BaseDiff;
-  SDValue Stride = std::holds_alternative<SDValue>(StrideVariant)
-                       ? std::get<SDValue>(StrideVariant)
-                       : DAG.getConstant(std::get<int64_t>(StrideVariant), DL,
-                                         Lds[0]->getOffset().getValueType());
+  SDValue Stride =
+      std::holds_alternative<SDValue>(StrideVariant)
+          ? std::get<SDValue>(StrideVariant)
+          : DAG.getSignedConstant(std::get<int64_t>(StrideVariant), DL,
+                                  Lds[0]->getOffset().getValueType());
   if (MustNegateStride)
     Stride = DAG.getNegative(Stride, DL, Stride.getValueType());
 
@@ -16599,7 +16603,7 @@ SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N,
                         DAG.getConstant(Subtarget.getXLen() - 1, DL, VT));
         SDValue AND =
             DAG.getNode(ISD::AND, DL, VT, SRA,
-                        DAG.getConstant(TrueSImm - FalseSImm, DL, VT));
+                        DAG.getSignedConstant(TrueSImm - FalseSImm, DL, VT));
         return DAG.getNode(ISD::ADD, DL, VT, AND, FalseV);
       }
 
@@ -16754,14 +16758,14 @@ SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N,
         assert(MGN->getBasePtr()->getValueType(0) == PtrVT);
         assert(IndexType == ISD::UNSIGNED_SCALED);
         SDValue BasePtr = DAG.getNode(ISD::ADD, DL, PtrVT, MGN->getBasePtr(),
-                                      DAG.getConstant(Addend, DL, PtrVT));
+                                      DAG.getSignedConstant(Addend, DL, PtrVT));
 
         SDValue EVL = DAG.getElementCount(DL, Subtarget.getXLenVT(),
                                           VT.getVectorElementCount());
-        SDValue StridedLoad =
-            DAG.getStridedLoadVP(VT, DL, MGN->getChain(), BasePtr,
-                                 DAG.getConstant(StepNumerator, DL, XLenVT),
-                                 MGN->getMask(), EVL, MGN->getMemOperand());
+        SDValue StridedLoad = DAG.getStridedLoadVP(
+            VT, DL, MGN->getChain(), BasePtr,
+            DAG.getSignedConstant(StepNumerator, DL, XLenVT), MGN->getMask(),
+            EVL, MGN->getMemOperand());
         SDValue VPSelect = DAG.getNode(ISD::VP_SELECT, DL, VT, MGN->getMask(),
                                        StridedLoad, MGN->getPassThru(), EVL);
         return DAG.getMergeValues({VPSelect, SDValue(StridedLoad.getNode(), 1)},
@@ -17217,7 +17221,7 @@ SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N,
       EVT VT = N->getValueType(0);
       if (IntNo == Intrinsic::riscv_vfirst ||
           IntNo == Intrinsic::riscv_vfirst_mask)
-        return DAG.getConstant(-1, DL, VT);
+        return DAG.getAllOnesConstant(DL, VT);
       return DAG.getConstant(0, DL, VT);
     }
     }
@@ -20498,8 +20502,8 @@ void RISCVTargetLowering::LowerAsmOperandForConstraint(
       if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
         uint64_t CVal = C->getSExtValue();
         if (isInt<12>(CVal))
-          Ops.push_back(
-              DAG.getTargetConstant(CVal, SDLoc(Op), Subtarget.getXLenVT()));
+          Ops.push_back(DAG.getSignedConstant(
+              CVal, SDLoc(Op), Subtarget.getXLenVT(), /*isTarget=*/true));
       }
       return;
     case 'J':
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.td b/llvm/lib/Target/RISCV/RISCVInstrInfo.td
index 41ef55544e7d2df..74406bf4b104719 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.td
@@ -403,14 +403,14 @@ def AddrRegImm : ComplexPattern<iPTR, 2, "SelectAddrRegImm">;
 
 // Return the negation of an immediate value.
 def NegImm : SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant(-N->getSExtValue(), SDLoc(N),
-                                   N->getValueType(0));
+  return CurDAG->getSignedConstant(-N->getSExtValue(), SDLoc(N),
+                                   N->getValueType(0), /*isTarget=*/true);
 }]>;
 
 // Return an immediate value minus 32.
 def ImmSub32 : SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant(N->getSExtValue() - 32, SDLoc(N),
-                                   N->getValueType(0));
+  return CurDAG->getSignedConstant(N->getSExtValue() - 32, SDLoc(N),
+                                   N->getValueType(0), /*isTarget=*/true);
 }]>;
 
 // Return an immediate subtracted from XLen.
@@ -442,16 +442,16 @@ def AddiPair : PatLeaf<(imm), [{
 def AddiPairImmSmall : SDNodeXForm<imm, [{
   int64_t Imm = N->getSExtValue();
   int64_t Adj = N->getSExtValue() < 0 ? -2048 : 2047;
-  return CurDAG->getTargetConstant(Imm - Adj, SDLoc(N),
-                                   N->getValueType(0));
+  return CurDAG->getSignedConstant(Imm - Adj, SDLoc(N),
+                                   N->getValueType(0), /*isTarget=*/true);
 }]>;
 
 // Return -2048 if immediate is negative or 2047 if positive. These are the
 // largest simm12 values.
 def AddiPairImmLarge : SDNodeXForm<imm, [{
   int64_t Imm = N->getSExtValue() < 0 ? -2048 : 2047;
-  return CurDAG->getTargetConstant(Imm, SDLoc(N),
-                                   N->getValueType(0));
+  return CurDAG->getSignedConstant(Imm, SDLoc(N),
+                                   N->getValueType(0), /*isTarget=*/true);
 }]>;
 
 def TrailingZeros : SDNodeXForm<imm, [{
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
index ab4d5a7e180df00..1b4303fbbcf8093 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
@@ -92,8 +92,8 @@ def AVL : RegisterOperand<GPRNoX0> {
 def VLOp : ComplexPattern<XLenVT, 1, "selectVLOp">;
 
 def DecImm : SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant(N->getSExtValue() - 1, SDLoc(N),
-                                   N->getValueType(0));
+  return CurDAG->getSignedConstant(N->getSExtValue() - 1, SDLoc(N),
+                                   N->getValueType(0), /*isTarget=*/true);
 }]>;
 
 defvar TAIL_AGNOSTIC = 1;
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td
index 8003bd98b8706a6..0cc0605ea870148 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td
@@ -169,8 +169,8 @@ def BCLRIANDIMask : PatLeaf<(imm), [{
 }]>;
 
 def BCLRIANDIMaskLow : SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant((N->getZExtValue() & 0x7ff) | ~0x7ffull,
-                                   SDLoc(N), N->getValueType(0));
+  return CurDAG->getSignedConstant((N->getZExtValue() & 0x7ff) | ~0x7ffull,
+                                   SDLoc(N), N->getValueType(0), /*isTarget=*/true);
 }]>;
 
 def CSImm12MulBy4 : PatLeaf<(imm), [{
@@ -191,13 +191,13 @@ def CSImm12MulBy8 : PatLeaf<(imm), [{
 }]>;
 
 def SimmShiftRightBy2XForm : SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant(N->getSExtValue() >> 2, SDLoc(N),
-                                   N->getValueType(0));
+  return CurDAG->getSignedConstant(N->getSExtValue() >> 2, SDLoc(N),
+                                   N->getValueType(0), /*isTarget=*/true);
 }]>;
 
 def SimmShiftRightBy3XForm : SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant(N->getSExtValue() >> 3, SDLoc(N),
-                                   N->getValueType(0));
+  return CurDAG->getSignedConstant(N->getSExtValue() >> 3, SDLoc(N),
+                                   N->getValueType(0), /*isTarget=*/true);
 }]>;
 
 // Pattern to exclude simm12 immediates from matching, namely `non_imm12`.

From cb7614e839148196c53711fdee639e6dff933a8d Mon Sep 17 00:00:00 2001
From: Matthias Springer <me@m-sp.org>
Date: Sat, 17 Aug 2024 09:43:30 +0200
Subject: [PATCH 262/441] [mlir][Transforms] Dialect conversion: Fix bug in
 `computeNecessaryMaterializations` (#104630)

There was a typo in the code path that removes unnecessary
materializations.

Before: Update `opResult` (result of an op different from `user`) in
mapping and remove `user`.
```
replaceMaterialization(rewriterImpl, opResult, inputOperands,
                       inverseMapping);
necessaryMaterializations.remove(materializationOps.lookup(user));
```

After: Update `user->getResults()` in mapping and remove `user`.
```
replaceMaterialization(rewriterImpl, user->getResults(), inputOperands,
                       inverseMapping);
necessaryMaterializations.remove(materializationOps.lookup(user));
```
---
 mlir/lib/Transforms/Utils/DialectConversion.cpp         | 2 +-
 mlir/test/Transforms/test-legalize-type-conversion.mlir | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/mlir/lib/Transforms/Utils/DialectConversion.cpp b/mlir/lib/Transforms/Utils/DialectConversion.cpp
index 11e593cebc09b3d..8a4c7463a69a950 100644
--- a/mlir/lib/Transforms/Utils/DialectConversion.cpp
+++ b/mlir/lib/Transforms/Utils/DialectConversion.cpp
@@ -2680,7 +2680,7 @@ static void computeNecessaryMaterializations(
       if (!castOp)
         continue;
       if (castOp->getResultTypes() == inputOperands.getTypes()) {
-        replaceMaterialization(rewriterImpl, opResult, inputOperands,
+        replaceMaterialization(rewriterImpl, user->getResults(), inputOperands,
                                inverseMapping);
         necessaryMaterializations.remove(materializationOps.lookup(user));
       }
diff --git a/mlir/test/Transforms/test-legalize-type-conversion.mlir b/mlir/test/Transforms/test-legalize-type-conversion.mlir
index 252b990210a1809..cf2c9f6a8ec4417 100644
--- a/mlir/test/Transforms/test-legalize-type-conversion.mlir
+++ b/mlir/test/Transforms/test-legalize-type-conversion.mlir
@@ -99,6 +99,7 @@ func.func @test_block_argument_not_converted() {
 func.func @test_signature_conversion_no_converter() {
   "test.signature_conversion_no_converter"() ({
   // expected-error@below {{failed to legalize unresolved materialization from ('f64') to 'f32' that remained live after conversion}}
+  // expected-note@below {{see existing live user here}}
   ^bb0(%arg0: f32):
     "test.type_consumer"(%arg0) : (f32) -> ()
     "test.return"(%arg0) : (f32) -> ()

From fc6300a5f7ef430e4ec86d16be0b146de7fbd16b Mon Sep 17 00:00:00 2001
From: Carl Ritson <carl.ritson@amd.com>
Date: Sat, 17 Aug 2024 16:52:38 +0900
Subject: [PATCH 263/441] [AMDGPU] Disable inline constants for pseudo scalar
 transcendentals (#104395)

Prevent operand folding from inlining constants into pseudo scalar
transcendental f16 instructions.
However still allow literal constants.
---
 llvm/lib/Target/AMDGPU/GCNSubtarget.h         |   6 +
 llvm/lib/Target/AMDGPU/SIInstrInfo.cpp        |   4 +
 llvm/lib/Target/AMDGPU/SIInstrInfo.h          |   8 ++
 .../AMDGPU/pseudo-scalar-transcendental.mir   | 120 ++++++++++++++++++
 4 files changed, 138 insertions(+)
 create mode 100644 llvm/test/CodeGen/AMDGPU/pseudo-scalar-transcendental.mir

diff --git a/llvm/lib/Target/AMDGPU/GCNSubtarget.h b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
index def89c785b85523..902f51ae358d59c 100644
--- a/llvm/lib/Target/AMDGPU/GCNSubtarget.h
+++ b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
@@ -1289,6 +1289,12 @@ class GCNSubtarget final : public AMDGPUGenSubtargetInfo,
   /// and STOREcnt rather than VMcnt, LGKMcnt and VScnt respectively.
   bool hasExtendedWaitCounts() const { return getGeneration() >= GFX12; }
 
+  /// \returns true if inline constants are not supported for F16 pseudo
+  /// scalar transcendentals.
+  bool hasNoF16PseudoScalarTransInlineConstants() const {
+    return getGeneration() == GFX12;
+  }
+
   /// \returns The maximum number of instructions that can be enclosed in an
   /// S_CLAUSE on the given subtarget, or 0 for targets that do not support that
   /// instruction.
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 9147242046cedad..6dce41d1605fa41 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -5776,6 +5776,10 @@ bool SIInstrInfo::isOperandLegal(const MachineInstr &MI, unsigned OpIdx,
           return false;
       }
     }
+  } else if (ST.hasNoF16PseudoScalarTransInlineConstants() && !MO->isReg() &&
+             isF16PseudoScalarTrans(MI.getOpcode()) &&
+             isInlineConstant(*MO, OpInfo)) {
+    return false;
   }
 
   if (MO->isReg()) {
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index 1712dfe8d406ccb..91855fb14f6f371 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -946,6 +946,14 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
            Opcode == AMDGPU::DS_GWS_BARRIER;
   }
 
+  static bool isF16PseudoScalarTrans(unsigned Opcode) {
+    return Opcode == AMDGPU::V_S_EXP_F16_e64 ||
+           Opcode == AMDGPU::V_S_LOG_F16_e64 ||
+           Opcode == AMDGPU::V_S_RCP_F16_e64 ||
+           Opcode == AMDGPU::V_S_RSQ_F16_e64 ||
+           Opcode == AMDGPU::V_S_SQRT_F16_e64;
+  }
+
   static bool doesNotReadTiedSource(const MachineInstr &MI) {
     return MI.getDesc().TSFlags & SIInstrFlags::TiedSourceNotRead;
   }
diff --git a/llvm/test/CodeGen/AMDGPU/pseudo-scalar-transcendental.mir b/llvm/test/CodeGen/AMDGPU/pseudo-scalar-transcendental.mir
new file mode 100644
index 000000000000000..17bed38bd046d74
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/pseudo-scalar-transcendental.mir
@@ -0,0 +1,120 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx1200 -run-pass=si-fold-operands -verify-machineinstrs -o - %s | FileCheck --check-prefix=GCN %s
+
+# Do not use inline constants for f16 pseudo scalar transcendentals.
+# But allow literal constants.
+
+---
+name: exp_f16_imm
+tracksRegLiveness: true
+body: |
+  bb.0:
+    ; GCN-LABEL: name: exp_f16_imm
+    ; GCN: [[S_MOV_B32_:%[0-9]+]]:sgpr_32 = S_MOV_B32 15360
+    ; GCN-NEXT: [[V_S_EXP_F16_e64_:%[0-9]+]]:sgpr_32 = V_S_EXP_F16_e64 1, [[S_MOV_B32_]], 0, 0, implicit $mode, implicit $exec
+    %0:sgpr_32 = S_MOV_B32 15360
+    %1:sgpr_32 = V_S_EXP_F16_e64 1, %0:sgpr_32, 0, 0, implicit $mode, implicit $exec
+...
+
+---
+name: exp_f16_literal
+tracksRegLiveness: true
+body: |
+  bb.0:
+    ; GCN-LABEL: name: exp_f16_literal
+    ; GCN: [[V_S_EXP_F16_e64_:%[0-9]+]]:sgpr_32 = V_S_EXP_F16_e64 1, 16960, 0, 0, implicit $mode, implicit $exec
+    %0:sgpr_32 = S_MOV_B32 16960
+    %1:sgpr_32 = V_S_EXP_F16_e64 1, %0:sgpr_32, 0, 0, implicit $mode, implicit $exec
+...
+
+---
+name: log_f16_imm
+tracksRegLiveness: true
+body: |
+  bb.0:
+    ; GCN-LABEL: name: log_f16_imm
+    ; GCN: [[S_MOV_B32_:%[0-9]+]]:sgpr_32 = S_MOV_B32 15360
+    ; GCN-NEXT: [[V_S_LOG_F16_e64_:%[0-9]+]]:sgpr_32 = V_S_LOG_F16_e64 1, [[S_MOV_B32_]], 0, 0, implicit $mode, implicit $exec
+    %0:sgpr_32 = S_MOV_B32 15360
+    %1:sgpr_32 = V_S_LOG_F16_e64 1, %0:sgpr_32, 0, 0, implicit $mode, implicit $exec
+...
+
+---
+name: log_f16_literal
+tracksRegLiveness: true
+body: |
+  bb.0:
+    ; GCN-LABEL: name: log_f16_literal
+    ; GCN: [[V_S_LOG_F16_e64_:%[0-9]+]]:sgpr_32 = V_S_LOG_F16_e64 1, 16960, 0, 0, implicit $mode, implicit $exec
+    %0:sgpr_32 = S_MOV_B32 16960
+    %1:sgpr_32 = V_S_LOG_F16_e64 1, %0:sgpr_32, 0, 0, implicit $mode, implicit $exec
+...
+
+---
+name: rcp_f16_imm
+tracksRegLiveness: true
+body: |
+  bb.0:
+    ; GCN-LABEL: name: rcp_f16_imm
+    ; GCN: [[S_MOV_B32_:%[0-9]+]]:sgpr_32 = S_MOV_B32 15360
+    ; GCN-NEXT: [[V_S_RCP_F16_e64_:%[0-9]+]]:sgpr_32 = V_S_RCP_F16_e64 1, [[S_MOV_B32_]], 0, 0, implicit $mode, implicit $exec
+    %0:sgpr_32 = S_MOV_B32 15360
+    %1:sgpr_32 = V_S_RCP_F16_e64 1, %0:sgpr_32, 0, 0, implicit $mode, implicit $exec
+...
+
+---
+name: rcp_f16_literal
+tracksRegLiveness: true
+body: |
+  bb.0:
+    ; GCN-LABEL: name: rcp_f16_literal
+    ; GCN: [[V_S_RCP_F16_e64_:%[0-9]+]]:sgpr_32 = V_S_RCP_F16_e64 1, 16960, 0, 0, implicit $mode, implicit $exec
+    %0:sgpr_32 = S_MOV_B32 16960
+    %1:sgpr_32 = V_S_RCP_F16_e64 1, %0:sgpr_32, 0, 0, implicit $mode, implicit $exec
+...
+
+---
+name: rsq_f16_imm
+tracksRegLiveness: true
+body: |
+  bb.0:
+    ; GCN-LABEL: name: rsq_f16_imm
+    ; GCN: [[S_MOV_B32_:%[0-9]+]]:sgpr_32 = S_MOV_B32 15360
+    ; GCN-NEXT: [[V_S_RSQ_F16_e64_:%[0-9]+]]:sgpr_32 = V_S_RSQ_F16_e64 1, [[S_MOV_B32_]], 0, 0, implicit $mode, implicit $exec
+    %0:sgpr_32 = S_MOV_B32 15360
+    %1:sgpr_32 = V_S_RSQ_F16_e64 1, %0:sgpr_32, 0, 0, implicit $mode, implicit $exec
+...
+
+---
+name: rsq_f16_literal
+tracksRegLiveness: true
+body: |
+  bb.0:
+    ; GCN-LABEL: name: rsq_f16_literal
+    ; GCN: [[V_S_RSQ_F16_e64_:%[0-9]+]]:sgpr_32 = V_S_RSQ_F16_e64 1, 16960, 0, 0, implicit $mode, implicit $exec
+    %0:sgpr_32 = S_MOV_B32 16960
+    %1:sgpr_32 = V_S_RSQ_F16_e64 1, %0:sgpr_32, 0, 0, implicit $mode, implicit $exec
+...
+
+---
+name: sqrt_f16_imm
+tracksRegLiveness: true
+body: |
+  bb.0:
+    ; GCN-LABEL: name: sqrt_f16_imm
+    ; GCN: [[S_MOV_B32_:%[0-9]+]]:sgpr_32 = S_MOV_B32 15360
+    ; GCN-NEXT: [[V_S_SQRT_F16_e64_:%[0-9]+]]:sgpr_32 = V_S_SQRT_F16_e64 1, [[S_MOV_B32_]], 0, 0, implicit $mode, implicit $exec
+    %0:sgpr_32 = S_MOV_B32 15360
+    %1:sgpr_32 = V_S_SQRT_F16_e64 1, %0:sgpr_32, 0, 0, implicit $mode, implicit $exec
+...
+
+---
+name: sqrt_f16_literal
+tracksRegLiveness: true
+body: |
+  bb.0:
+    ; GCN-LABEL: name: sqrt_f16_literal
+    ; GCN: [[V_S_SQRT_F16_e64_:%[0-9]+]]:sgpr_32 = V_S_SQRT_F16_e64 1, 16960, 0, 0, implicit $mode, implicit $exec
+    %0:sgpr_32 = S_MOV_B32 16960
+    %1:sgpr_32 = V_S_SQRT_F16_e64 1, %0:sgpr_32, 0, 0, implicit $mode, implicit $exec
+...

From aad27bf534b59645f47a92f072af798687b1dd0d Mon Sep 17 00:00:00 2001
From: Giuseppe Rossini <giuseppe.rossini@amd.com>
Date: Sat, 17 Aug 2024 09:03:06 +0100
Subject: [PATCH 264/441] Add non-temporal support for LLVM masked loads
 (#104598)

This PR is adding non-temporal support to masked load as a `UnitAttr`
attribute. Non temporal load is quite an important feature for masked
loads to make the intrinsic usable from high level compilers
---
 .../mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td       | 15 ++++++++++-----
 mlir/test/Target/LLVMIR/Import/intrinsic.ll       |  2 ++
 mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir    |  5 ++++-
 3 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
index 2e1635e590cad6a..86983e95fdd33d0 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
@@ -842,22 +842,27 @@ def LLVM_GetActiveLaneMaskOp
 /// Create a call to Masked Load intrinsic.
 def LLVM_MaskedLoadOp : LLVM_OneResultIntrOp<"masked.load"> {
   let arguments = (ins LLVM_AnyPointer:$data, LLVM_VectorOf<I1>:$mask,
-                   Variadic<LLVM_AnyVector>:$pass_thru, I32Attr:$alignment);
+                   Variadic<LLVM_AnyVector>:$pass_thru, I32Attr:$alignment,
+                   UnitAttr:$nontemporal);
   let results = (outs LLVM_AnyVector:$res);
   let assemblyFormat =
     "operands attr-dict `:` functional-type(operands, results)";
 
   string llvmBuilder = [{
-    $res = $pass_thru.empty() ? builder.CreateMaskedLoad(
+    auto *inst = $pass_thru.empty() ? builder.CreateMaskedLoad(
         $_resultType, $data, llvm::Align($alignment), $mask) :
       builder.CreateMaskedLoad(
         $_resultType, $data, llvm::Align($alignment), $mask, $pass_thru[0]);
-  }];
+    $res = inst;
+  }] #setNonTemporalMetadataCode;
   string mlirBuilder = [{
+    auto *intrinInst = dyn_cast<llvm::IntrinsicInst>(inst);
+    bool nontemporal = intrinInst->hasMetadata(llvm::LLVMContext::MD_nontemporal);
     $res = $_builder.create<LLVM::MaskedLoadOp>($_location,
-      $_resultType, $data, $mask, $pass_thru, $_int_attr($alignment));
+      $_resultType, $data, $mask, $pass_thru, $_int_attr($alignment),
+        nontemporal ? $_builder.getUnitAttr() : nullptr);
   }];
-  list<int> llvmArgIndices = [0, 2, 3, 1];
+  list<int> llvmArgIndices = [0, 2, 3, 1, -1];
 }
 
 /// Create a call to Masked Store intrinsic.
diff --git a/mlir/test/Target/LLVMIR/Import/intrinsic.ll b/mlir/test/Target/LLVMIR/Import/intrinsic.ll
index 9a5528002ef5e96..0fa82cef0a0f5a5 100644
--- a/mlir/test/Target/LLVMIR/Import/intrinsic.ll
+++ b/mlir/test/Target/LLVMIR/Import/intrinsic.ll
@@ -409,6 +409,8 @@ define void @masked_load_store_intrinsics(ptr %vec, <7 x i1> %mask) {
   %1 = call <7 x float> @llvm.masked.load.v7f32.p0(ptr %vec, i32 1, <7 x i1> %mask, <7 x float> undef)
   ; CHECK:  %[[VAL2:.+]] = llvm.intr.masked.load %[[VEC]], %[[MASK]], %[[VAL1]] {alignment = 4 : i32}
   %2 = call <7 x float> @llvm.masked.load.v7f32.p0(ptr %vec, i32 4, <7 x i1> %mask, <7 x float> %1)
+  ; CHECK:  %[[VAL3:.+]] = llvm.intr.masked.load %[[VEC]], %[[MASK]], %[[VAL1]] {alignment = 4 : i32, nontemporal}
+  %3 = call <7 x float> @llvm.masked.load.v7f32.p0(ptr %vec, i32 4, <7 x i1> %mask, <7 x float> %1), !nontemporal !{i32 1}
   ; CHECK:  llvm.intr.masked.store %[[VAL2]], %[[VEC]], %[[MASK]] {alignment = 8 : i32}
   ; CHECK-SAME:  vector<7xf32>, vector<7xi1> into !llvm.ptr
   call void @llvm.masked.store.v7f32.p0(<7 x float> %2, ptr %vec, i32 8, <7 x i1> %mask)
diff --git a/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir b/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
index 7878aa5ee46d4f0..e2eadf14fc97e96 100644
--- a/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
+++ b/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
@@ -417,8 +417,11 @@ llvm.func @masked_load_store_intrinsics(%A: !llvm.ptr, %mask: vector<7xi1>) {
   // CHECK: call <7 x float> @llvm.masked.load.v7f32.p0(ptr %{{.*}}, i32 1, <7 x i1> %{{.*}}, <7 x float> poison)
   %a = llvm.intr.masked.load %A, %mask { alignment = 1: i32} :
     (!llvm.ptr, vector<7xi1>) -> vector<7xf32>
+  // CHECK: call <7 x float> @llvm.masked.load.v7f32.p0(ptr %{{.*}}, i32 1, <7 x i1> %{{.*}}, <7 x float> poison), !nontemporal !1
+  %b = llvm.intr.masked.load %A, %mask { alignment = 1: i32, nontemporal} :
+    (!llvm.ptr, vector<7xi1>) -> vector<7xf32>
   // CHECK: call <7 x float> @llvm.masked.load.v7f32.p0(ptr %{{.*}}, i32 1, <7 x i1> %{{.*}}, <7 x float> %{{.*}})
-  %b = llvm.intr.masked.load %A, %mask, %a { alignment = 1: i32} :
+  %c = llvm.intr.masked.load %A, %mask, %a { alignment = 1: i32} :
     (!llvm.ptr, vector<7xi1>, vector<7xf32>) -> vector<7xf32>
   // CHECK: call void @llvm.masked.store.v7f32.p0(<7 x float> %{{.*}}, ptr %0, i32 {{.*}}, <7 x i1> %{{.*}})
   llvm.intr.masked.store %b, %A, %mask { alignment = 1: i32} :

From 48ae61470104e9d7a8be5beb8739c24f52cc33c0 Mon Sep 17 00:00:00 2001
From: Yingwei Zheng <dtcxzyw2333@gmail.com>
Date: Sat, 17 Aug 2024 16:48:29 +0800
Subject: [PATCH 265/441] [InstCombine] Avoid infinite loop when negating phi
 nodes (#104581)

Closes https://github.com/llvm/llvm-project/issues/96012

---------

Co-authored-by: Nikita Popov <github@npopov.com>
---
 .../InstCombine/InstCombineInternal.h         |  5 +++-
 .../InstCombine/InstCombineNegator.cpp        | 11 ++++++---
 llvm/test/Transforms/InstCombine/pr96012.ll   | 24 +++++++++++++++++++
 3 files changed, 36 insertions(+), 4 deletions(-)
 create mode 100644 llvm/test/Transforms/InstCombine/pr96012.ll

diff --git a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
index 16f1c3ba15eba2d..a0f2399972b52f2 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
+++ b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
@@ -780,11 +780,14 @@ class Negator final {
   using BuilderTy = IRBuilder<TargetFolder, IRBuilderCallbackInserter>;
   BuilderTy Builder;
 
+  const DominatorTree &DT;
+
   const bool IsTrulyNegation;
 
   SmallDenseMap<Value *, Value *> NegationsCache;
 
-  Negator(LLVMContext &C, const DataLayout &DL, bool IsTrulyNegation);
+  Negator(LLVMContext &C, const DataLayout &DL, const DominatorTree &DT,
+          bool IsTrulyNegation);
 
 #if LLVM_ENABLE_STATS
   unsigned NumValuesVisitedInThisNegator = 0;
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineNegator.cpp b/llvm/lib/Transforms/InstCombine/InstCombineNegator.cpp
index 2dd1db6a4a757bb..92293ef401465b7 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineNegator.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineNegator.cpp
@@ -90,13 +90,14 @@ static cl::opt<unsigned>
                     cl::desc("What is the maximal lookup depth when trying to "
                              "check for viability of negation sinking."));
 
-Negator::Negator(LLVMContext &C, const DataLayout &DL, bool IsTrulyNegation_)
+Negator::Negator(LLVMContext &C, const DataLayout &DL, const DominatorTree &DT_,
+                 bool IsTrulyNegation_)
     : Builder(C, TargetFolder(DL),
               IRBuilderCallbackInserter([&](Instruction *I) {
                 ++NegatorNumInstructionsCreatedTotal;
                 NewInstructions.push_back(I);
               })),
-      IsTrulyNegation(IsTrulyNegation_) {}
+      DT(DT_), IsTrulyNegation(IsTrulyNegation_) {}
 
 #if LLVM_ENABLE_STATS
 Negator::~Negator() {
@@ -309,6 +310,9 @@ std::array<Value *, 2> Negator::getSortedOperandsOfBinOp(Instruction *I) {
     auto *PHI = cast<PHINode>(I);
     SmallVector<Value *, 4> NegatedIncomingValues(PHI->getNumOperands());
     for (auto I : zip(PHI->incoming_values(), NegatedIncomingValues)) {
+      // Don't negate indvars to avoid infinite loops.
+      if (DT.dominates(PHI->getParent(), std::get<0>(I)))
+        return nullptr;
       if (!(std::get<1>(I) =
                 negate(std::get<0>(I), IsNSW, Depth + 1))) // Early return.
         return nullptr;
@@ -537,7 +541,8 @@ std::array<Value *, 2> Negator::getSortedOperandsOfBinOp(Instruction *I) {
   if (!NegatorEnabled || !DebugCounter::shouldExecute(NegatorCounter))
     return nullptr;
 
-  Negator N(Root->getContext(), IC.getDataLayout(), LHSIsZero);
+  Negator N(Root->getContext(), IC.getDataLayout(), IC.getDominatorTree(),
+            LHSIsZero);
   std::optional<Result> Res = N.run(Root, IsNSW);
   if (!Res) { // Negation failed.
     LLVM_DEBUG(dbgs() << "Negator: failed to sink negation into " << *Root
diff --git a/llvm/test/Transforms/InstCombine/pr96012.ll b/llvm/test/Transforms/InstCombine/pr96012.ll
new file mode 100644
index 000000000000000..1a1356f07421609
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pr96012.ll
@@ -0,0 +1,24 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -passes=instcombine < %s | FileCheck %s
+
+target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128"
+
+; Make sure we don't hang here.
+define i32 @pr96012() {
+; CHECK-LABEL: define i32 @pr96012() {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br label %[[FOR_COND:.*]]
+; CHECK:       [[FOR_COND]]:
+; CHECK-NEXT:    br label %[[FOR_COND]]
+;
+entry:
+  br label %for.cond
+
+for.cond:
+  %indvar = phi i32 [ 1, %entry ], [ %shl, %for.cond ]
+  %conv1 = trunc i32 %indvar to i8
+  %neg = sub i8 0, %conv1
+  %conv2 = zext i8 %neg to i32
+  %shl = shl nuw i32 %conv2, 24
+  br label %for.cond
+}

From 0da2ba811ac8a01509bc533428941fb9519c0715 Mon Sep 17 00:00:00 2001
From: Daniil Fukalov <dfukalov@gmail.com>
Date: Sat, 17 Aug 2024 13:11:18 +0200
Subject: [PATCH 266/441] [NFC] Cleanup in ADT and Analysis headers. (#104484)

Remove unused directly includes and forward declarations in ADT and
Analysis headers.
---
 llvm/include/llvm/ADT/APFixedPoint.h                      | 1 -
 llvm/include/llvm/ADT/DynamicAPInt.h                      | 4 +++-
 llvm/include/llvm/ADT/SlowDynamicAPInt.h                  | 2 +-
 llvm/include/llvm/ADT/SmallPtrSet.h                       | 1 -
 llvm/include/llvm/ADT/SmallVector.h                       | 1 -
 llvm/include/llvm/ADT/Statistic.h                         | 1 -
 llvm/include/llvm/Analysis/AliasAnalysis.h                | 5 -----
 llvm/include/llvm/Analysis/AliasSetTracker.h              | 5 +++--
 llvm/include/llvm/Analysis/AssumeBundleQueries.h          | 1 -
 llvm/include/llvm/Analysis/BlockFrequencyInfo.h           | 1 -
 llvm/include/llvm/Analysis/BranchProbabilityInfo.h        | 1 -
 llvm/include/llvm/Analysis/CGSCCPassManager.h             | 1 -
 llvm/include/llvm/Analysis/CallGraph.h                    | 1 -
 llvm/include/llvm/Analysis/CaptureTracking.h              | 2 +-
 llvm/include/llvm/Analysis/CodeMetrics.h                  | 1 -
 llvm/include/llvm/Analysis/ConstraintSystem.h             | 1 -
 llvm/include/llvm/Analysis/CtxProfAnalysis.h              | 2 --
 llvm/include/llvm/Analysis/CycleAnalysis.h                | 1 -
 llvm/include/llvm/Analysis/DXILMetadataAnalysis.h         | 1 -
 llvm/include/llvm/Analysis/DXILResource.h                 | 6 ++++--
 llvm/include/llvm/Analysis/DomConditionCache.h            | 2 --
 llvm/include/llvm/Analysis/DomPrinter.h                   | 1 -
 llvm/include/llvm/Analysis/DomTreeUpdater.h               | 2 --
 llvm/include/llvm/Analysis/DominanceFrontier.h            | 1 -
 llvm/include/llvm/Analysis/FunctionPropertiesAnalysis.h   | 5 +++--
 llvm/include/llvm/Analysis/IRSimilarityIdentifier.h       | 1 -
 llvm/include/llvm/Analysis/IVDescriptors.h                | 1 -
 llvm/include/llvm/Analysis/InlineAdvisor.h                | 1 -
 llvm/include/llvm/Analysis/InlineOrder.h                  | 2 +-
 llvm/include/llvm/Analysis/InstructionSimplify.h          | 1 -
 llvm/include/llvm/Analysis/InteractiveModelRunner.h       | 3 ---
 llvm/include/llvm/Analysis/LazyBlockFrequencyInfo.h       | 3 +--
 llvm/include/llvm/Analysis/LazyBranchProbabilityInfo.h    | 3 +--
 llvm/include/llvm/Analysis/LazyCallGraph.h                | 4 ----
 llvm/include/llvm/Analysis/LazyValueInfo.h                | 2 --
 llvm/include/llvm/Analysis/LoopAccessAnalysis.h           | 8 ++------
 llvm/include/llvm/Analysis/LoopInfo.h                     | 5 -----
 llvm/include/llvm/Analysis/MLInlineAdvisor.h              | 3 +--
 llvm/include/llvm/Analysis/MemoryBuiltins.h               | 1 -
 llvm/include/llvm/Analysis/MemoryDependenceAnalysis.h     | 1 -
 llvm/include/llvm/Analysis/MemoryLocation.h               | 1 -
 llvm/include/llvm/Analysis/MemoryProfileInfo.h            | 3 ---
 llvm/include/llvm/Analysis/MemorySSA.h                    | 4 ----
 llvm/include/llvm/Analysis/MustExecute.h                  | 1 -
 llvm/include/llvm/Analysis/NoInferenceModelRunner.h       | 4 ++--
 llvm/include/llvm/Analysis/ObjCARCAliasAnalysis.h         | 1 -
 llvm/include/llvm/Analysis/OptimizationRemarkEmitter.h    | 4 +---
 llvm/include/llvm/Analysis/ProfileSummaryInfo.h           | 5 ++---
 llvm/include/llvm/Analysis/PtrUseVisitor.h                | 1 -
 llvm/include/llvm/Analysis/RegionInfo.h                   | 2 --
 llvm/include/llvm/Analysis/RegionPrinter.h                | 4 ++--
 llvm/include/llvm/Analysis/ReplayInlineAdvisor.h          | 1 -
 llvm/include/llvm/Analysis/ScalarEvolution.h              | 4 ----
 llvm/include/llvm/Analysis/SyntheticCountsUtils.h         | 2 +-
 llvm/include/llvm/Analysis/TargetLibraryInfo.h            | 3 ---
 llvm/include/llvm/Analysis/TargetTransformInfo.h          | 5 +----
 llvm/include/llvm/Analysis/UniformityAnalysis.h           | 4 +++-
 llvm/include/llvm/Analysis/ValueLattice.h                 | 5 +----
 llvm/include/llvm/Analysis/ValueTracking.h                | 6 +-----
 llvm/include/llvm/Analysis/VectorUtils.h                  | 2 --
 llvm/lib/Analysis/InstructionSimplify.cpp                 | 1 +
 llvm/lib/Analysis/MemoryProfileInfo.cpp                   | 1 +
 llvm/lib/Analysis/ValueLattice.cpp                        | 1 +
 llvm/lib/CodeGen/CodeGenPrepare.cpp                       | 1 +
 llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp                 | 3 ++-
 llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp          | 2 +-
 llvm/lib/Transforms/Utils/SizeOpts.cpp                    | 1 +
 .../Transforms/Vectorize/LoopVectorizationLegality.cpp    | 1 +
 llvm/unittests/Analysis/MemoryProfileInfoTest.cpp         | 1 +
 mlir/include/mlir/Analysis/Presburger/Fraction.h          | 1 +
 70 files changed, 46 insertions(+), 118 deletions(-)

diff --git a/llvm/include/llvm/ADT/APFixedPoint.h b/llvm/include/llvm/ADT/APFixedPoint.h
index 0c014e76aa71260..ae40db96e4818c7 100644
--- a/llvm/include/llvm/ADT/APFixedPoint.h
+++ b/llvm/include/llvm/ADT/APFixedPoint.h
@@ -17,7 +17,6 @@
 #define LLVM_ADT_APFIXEDPOINT_H
 
 #include "llvm/ADT/APSInt.h"
-#include "llvm/ADT/DenseMapInfo.h"
 #include "llvm/ADT/Hashing.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/Support/raw_ostream.h"
diff --git a/llvm/include/llvm/ADT/DynamicAPInt.h b/llvm/include/llvm/ADT/DynamicAPInt.h
index 4444d52527def13..ff958d48e773179 100644
--- a/llvm/include/llvm/ADT/DynamicAPInt.h
+++ b/llvm/include/llvm/ADT/DynamicAPInt.h
@@ -18,10 +18,12 @@
 
 #include "llvm/ADT/SlowDynamicAPInt.h"
 #include "llvm/Support/MathExtras.h"
-#include "llvm/Support/raw_ostream.h"
 #include <numeric>
 
 namespace llvm {
+
+class raw_ostream;
+
 /// This class provides support for dynamic arbitrary-precision arithmetic.
 ///
 /// Unlike APInt, this extends the precision as necessary to prevent overflows
diff --git a/llvm/include/llvm/ADT/SlowDynamicAPInt.h b/llvm/include/llvm/ADT/SlowDynamicAPInt.h
index cda5f39eb98c359..ec1021892cf4ddd 100644
--- a/llvm/include/llvm/ADT/SlowDynamicAPInt.h
+++ b/llvm/include/llvm/ADT/SlowDynamicAPInt.h
@@ -19,10 +19,10 @@
 #define LLVM_ADT_SLOWDYNAMICAPINT_H
 
 #include "llvm/ADT/APInt.h"
-#include "llvm/Support/raw_ostream.h"
 
 namespace llvm {
 class DynamicAPInt;
+class raw_ostream;
 } // namespace llvm
 
 namespace llvm::detail {
diff --git a/llvm/include/llvm/ADT/SmallPtrSet.h b/llvm/include/llvm/ADT/SmallPtrSet.h
index 78383f495b6a58d..e668df08c3e6363 100644
--- a/llvm/include/llvm/ADT/SmallPtrSet.h
+++ b/llvm/include/llvm/ADT/SmallPtrSet.h
@@ -16,7 +16,6 @@
 #define LLVM_ADT_SMALLPTRSET_H
 
 #include "llvm/ADT/EpochTracker.h"
-#include "llvm/Support/Compiler.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/ReverseIteration.h"
 #include "llvm/Support/type_traits.h"
diff --git a/llvm/include/llvm/ADT/SmallVector.h b/llvm/include/llvm/ADT/SmallVector.h
index 17444147b102a96..730f84ca038d60c 100644
--- a/llvm/include/llvm/ADT/SmallVector.h
+++ b/llvm/include/llvm/ADT/SmallVector.h
@@ -15,7 +15,6 @@
 #define LLVM_ADT_SMALLVECTOR_H
 
 #include "llvm/Support/Compiler.h"
-#include "llvm/Support/type_traits.h"
 #include <algorithm>
 #include <cassert>
 #include <cstddef>
diff --git a/llvm/include/llvm/ADT/Statistic.h b/llvm/include/llvm/ADT/Statistic.h
index 9475b6697031a46..a64cfce3ba44a0c 100644
--- a/llvm/include/llvm/ADT/Statistic.h
+++ b/llvm/include/llvm/ADT/Statistic.h
@@ -27,7 +27,6 @@
 #define LLVM_ADT_STATISTIC_H
 
 #include "llvm/Config/llvm-config.h"
-#include "llvm/Support/Compiler.h"
 #include <atomic>
 #include <memory>
 #include <vector>
diff --git a/llvm/include/llvm/Analysis/AliasAnalysis.h b/llvm/include/llvm/Analysis/AliasAnalysis.h
index 4140387a1f34109..1990172116582b2 100644
--- a/llvm/include/llvm/Analysis/AliasAnalysis.h
+++ b/llvm/include/llvm/Analysis/AliasAnalysis.h
@@ -38,7 +38,6 @@
 #define LLVM_ANALYSIS_ALIASANALYSIS_H
 
 #include "llvm/ADT/DenseMap.h"
-#include "llvm/ADT/Sequence.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Analysis/MemoryLocation.h"
 #include "llvm/IR/Function.h"
@@ -53,18 +52,14 @@
 
 namespace llvm {
 
-class AnalysisUsage;
 class AtomicCmpXchgInst;
 class BasicBlock;
 class CatchPadInst;
 class CatchReturnInst;
 class DominatorTree;
 class FenceInst;
-class Function;
 class LoopInfo;
-class PreservedAnalyses;
 class TargetLibraryInfo;
-class Value;
 
 /// The possible results of an alias query.
 ///
diff --git a/llvm/include/llvm/Analysis/AliasSetTracker.h b/llvm/include/llvm/Analysis/AliasSetTracker.h
index b8e8e7ef047f963..e5817d2409bc651 100644
--- a/llvm/include/llvm/Analysis/AliasSetTracker.h
+++ b/llvm/include/llvm/Analysis/AliasSetTracker.h
@@ -23,7 +23,6 @@
 #include "llvm/ADT/ilist.h"
 #include "llvm/ADT/ilist_node.h"
 #include "llvm/Analysis/MemoryLocation.h"
-#include "llvm/IR/Instruction.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/IR/ValueHandle.h"
 #include <cassert>
@@ -37,10 +36,12 @@ class AnyMemSetInst;
 class AnyMemTransferInst;
 class BasicBlock;
 class BatchAAResults;
+class Function;
+class Instruction;
+class StoreInst;
 class LoadInst;
 enum class ModRefInfo : uint8_t;
 class raw_ostream;
-class StoreInst;
 class VAArgInst;
 class Value;
 
diff --git a/llvm/include/llvm/Analysis/AssumeBundleQueries.h b/llvm/include/llvm/Analysis/AssumeBundleQueries.h
index df54fd2e9d655d9..f7a893708758c5a 100644
--- a/llvm/include/llvm/Analysis/AssumeBundleQueries.h
+++ b/llvm/include/llvm/Analysis/AssumeBundleQueries.h
@@ -21,7 +21,6 @@ namespace llvm {
 class AssumptionCache;
 class DominatorTree;
 class Instruction;
-class Value;
 
 /// Index of elements in the operand bundle.
 /// If the element exist it is guaranteed to be what is specified in this enum
diff --git a/llvm/include/llvm/Analysis/BlockFrequencyInfo.h b/llvm/include/llvm/Analysis/BlockFrequencyInfo.h
index 179fd06addec875..5191a964403354d 100644
--- a/llvm/include/llvm/Analysis/BlockFrequencyInfo.h
+++ b/llvm/include/llvm/Analysis/BlockFrequencyInfo.h
@@ -25,7 +25,6 @@ namespace llvm {
 
 class BasicBlock;
 class BranchProbabilityInfo;
-class Function;
 class LoopInfo;
 class Module;
 class raw_ostream;
diff --git a/llvm/include/llvm/Analysis/BranchProbabilityInfo.h b/llvm/include/llvm/Analysis/BranchProbabilityInfo.h
index eab7fec6b6e08f4..1ac5694b758feee 100644
--- a/llvm/include/llvm/Analysis/BranchProbabilityInfo.h
+++ b/llvm/include/llvm/Analysis/BranchProbabilityInfo.h
@@ -22,7 +22,6 @@
 #include "llvm/IR/ValueHandle.h"
 #include "llvm/Pass.h"
 #include "llvm/Support/BranchProbability.h"
-#include <algorithm>
 #include <cassert>
 #include <cstdint>
 #include <memory>
diff --git a/llvm/include/llvm/Analysis/CGSCCPassManager.h b/llvm/include/llvm/Analysis/CGSCCPassManager.h
index 406d3492136fcc7..15b7f226fd8283a 100644
--- a/llvm/include/llvm/Analysis/CGSCCPassManager.h
+++ b/llvm/include/llvm/Analysis/CGSCCPassManager.h
@@ -99,7 +99,6 @@
 namespace llvm {
 
 class Function;
-class Value;
 template <typename T, unsigned int N> class SmallPriorityWorklist;
 struct CGSCCUpdateResult;
 
diff --git a/llvm/include/llvm/Analysis/CallGraph.h b/llvm/include/llvm/Analysis/CallGraph.h
index 887743774175dc3..f5ce2322b76d95a 100644
--- a/llvm/include/llvm/Analysis/CallGraph.h
+++ b/llvm/include/llvm/Analysis/CallGraph.h
@@ -46,7 +46,6 @@
 #define LLVM_ANALYSIS_CALLGRAPH_H
 
 #include "llvm/IR/InstrTypes.h"
-#include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/IR/ValueHandle.h"
 #include "llvm/Pass.h"
diff --git a/llvm/include/llvm/Analysis/CaptureTracking.h b/llvm/include/llvm/Analysis/CaptureTracking.h
index 2825399e2cc4ddb..06a00d9ae789908 100644
--- a/llvm/include/llvm/Analysis/CaptureTracking.h
+++ b/llvm/include/llvm/Analysis/CaptureTracking.h
@@ -14,7 +14,6 @@
 #define LLVM_ANALYSIS_CAPTURETRACKING_H
 
 #include "llvm/ADT/DenseMap.h"
-#include "llvm/ADT/STLFunctionalExtras.h"
 
 namespace llvm {
 
@@ -25,6 +24,7 @@ namespace llvm {
   class DominatorTree;
   class LoopInfo;
   class Function;
+  template <typename Fn> class function_ref;
 
   /// getDefaultMaxUsesToExploreForCaptureTracking - Return default value of
   /// the maximal number of uses to explore before giving up. It is used by
diff --git a/llvm/include/llvm/Analysis/CodeMetrics.h b/llvm/include/llvm/Analysis/CodeMetrics.h
index d09018daf954884..a51d923eb44ed98 100644
--- a/llvm/include/llvm/Analysis/CodeMetrics.h
+++ b/llvm/include/llvm/Analysis/CodeMetrics.h
@@ -20,7 +20,6 @@
 namespace llvm {
 class AssumptionCache;
 class BasicBlock;
-class Instruction;
 class Loop;
 class Function;
 template <class T> class SmallPtrSetImpl;
diff --git a/llvm/include/llvm/Analysis/ConstraintSystem.h b/llvm/include/llvm/Analysis/ConstraintSystem.h
index 7b02b618f7cb447..449852343964cad 100644
--- a/llvm/include/llvm/Analysis/ConstraintSystem.h
+++ b/llvm/include/llvm/Analysis/ConstraintSystem.h
@@ -9,7 +9,6 @@
 #ifndef LLVM_ANALYSIS_CONSTRAINTSYSTEM_H
 #define LLVM_ANALYSIS_CONSTRAINTSYSTEM_H
 
-#include "llvm/ADT/APInt.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SmallVector.h"
diff --git a/llvm/include/llvm/Analysis/CtxProfAnalysis.h b/llvm/include/llvm/Analysis/CtxProfAnalysis.h
index d0fb99fe1966a65..8707e83574dd84c 100644
--- a/llvm/include/llvm/Analysis/CtxProfAnalysis.h
+++ b/llvm/include/llvm/Analysis/CtxProfAnalysis.h
@@ -9,8 +9,6 @@
 #ifndef LLVM_ANALYSIS_CTXPROFANALYSIS_H
 #define LLVM_ANALYSIS_CTXPROFANALYSIS_H
 
-#include "llvm/ADT/DenseMap.h"
-#include "llvm/IR/GlobalValue.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/ProfileData/PGOCtxProfReader.h"
 
diff --git a/llvm/include/llvm/Analysis/CycleAnalysis.h b/llvm/include/llvm/Analysis/CycleAnalysis.h
index ce939eff8ff8f32..6b45cece199913f 100644
--- a/llvm/include/llvm/Analysis/CycleAnalysis.h
+++ b/llvm/include/llvm/Analysis/CycleAnalysis.h
@@ -17,7 +17,6 @@
 
 #include "llvm/IR/CycleInfo.h"
 #include "llvm/IR/PassManager.h"
-#include "llvm/IR/SSAContext.h"
 #include "llvm/Pass.h"
 
 namespace llvm {
diff --git a/llvm/include/llvm/Analysis/DXILMetadataAnalysis.h b/llvm/include/llvm/Analysis/DXILMetadataAnalysis.h
index e5c6ae5db559eef..f325828495458b8 100644
--- a/llvm/include/llvm/Analysis/DXILMetadataAnalysis.h
+++ b/llvm/include/llvm/Analysis/DXILMetadataAnalysis.h
@@ -10,7 +10,6 @@
 #define LLVM_ANALYSIS_DXILMETADATA_H
 
 #include "llvm/IR/PassManager.h"
-#include "llvm/IR/Value.h"
 #include "llvm/Pass.h"
 #include "llvm/Support/VersionTuple.h"
 #include "llvm/TargetParser/Triple.h"
diff --git a/llvm/include/llvm/Analysis/DXILResource.h b/llvm/include/llvm/Analysis/DXILResource.h
index 3ba0ae5de61d515..14cf03c9a3acee2 100644
--- a/llvm/include/llvm/Analysis/DXILResource.h
+++ b/llvm/include/llvm/Analysis/DXILResource.h
@@ -10,15 +10,17 @@
 #define LLVM_ANALYSIS_DXILRESOURCE_H
 
 #include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/StringRef.h"
 #include "llvm/IR/PassManager.h"
-#include "llvm/IR/Value.h"
 #include "llvm/Pass.h"
+#include "llvm/Support/Alignment.h"
 #include "llvm/Support/DXILABI.h"
 
 namespace llvm {
 class CallInst;
+class LLVMContext;
 class MDTuple;
-class TargetExtType;
+class Value;
 
 namespace dxil {
 
diff --git a/llvm/include/llvm/Analysis/DomConditionCache.h b/llvm/include/llvm/Analysis/DomConditionCache.h
index 68e34f773af8ae8..ac25803143f49e8 100644
--- a/llvm/include/llvm/Analysis/DomConditionCache.h
+++ b/llvm/include/llvm/Analysis/DomConditionCache.h
@@ -19,9 +19,7 @@
 
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseMap.h"
-#include "llvm/ADT/DenseMapInfo.h"
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/IR/ValueHandle.h"
 
 namespace llvm {
 
diff --git a/llvm/include/llvm/Analysis/DomPrinter.h b/llvm/include/llvm/Analysis/DomPrinter.h
index 83fe721346ab2ed..80fdfcd56c366f6 100644
--- a/llvm/include/llvm/Analysis/DomPrinter.h
+++ b/llvm/include/llvm/Analysis/DomPrinter.h
@@ -17,7 +17,6 @@
 #include "llvm/Analysis/DOTGraphTraitsPass.h"
 #include "llvm/Analysis/PostDominators.h"
 #include "llvm/IR/Dominators.h"
-#include "llvm/IR/PassManager.h"
 
 namespace llvm {
 
diff --git a/llvm/include/llvm/Analysis/DomTreeUpdater.h b/llvm/include/llvm/Analysis/DomTreeUpdater.h
index e07d2bf2b0df17c..c120a6cc6ce5abe 100644
--- a/llvm/include/llvm/Analysis/DomTreeUpdater.h
+++ b/llvm/include/llvm/Analysis/DomTreeUpdater.h
@@ -14,12 +14,10 @@
 #ifndef LLVM_ANALYSIS_DOMTREEUPDATER_H
 #define LLVM_ANALYSIS_DOMTREEUPDATER_H
 
-#include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/Analysis/GenericDomTreeUpdater.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/ValueHandle.h"
 #include "llvm/Support/Compiler.h"
-#include <cstddef>
 #include <functional>
 #include <vector>
 
diff --git a/llvm/include/llvm/Analysis/DominanceFrontier.h b/llvm/include/llvm/Analysis/DominanceFrontier.h
index 1339b4869bc29b1..8b3ab57ed4d07b5 100644
--- a/llvm/include/llvm/Analysis/DominanceFrontier.h
+++ b/llvm/include/llvm/Analysis/DominanceFrontier.h
@@ -20,7 +20,6 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/GraphTraits.h"
 #include "llvm/ADT/SetVector.h"
-#include "llvm/Config/llvm-config.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/Pass.h"
 #include "llvm/Support/GenericDomTree.h"
diff --git a/llvm/include/llvm/Analysis/FunctionPropertiesAnalysis.h b/llvm/include/llvm/Analysis/FunctionPropertiesAnalysis.h
index f5fbbdcb7143d76..ee447d3e4ebb6ab 100644
--- a/llvm/include/llvm/Analysis/FunctionPropertiesAnalysis.h
+++ b/llvm/include/llvm/Analysis/FunctionPropertiesAnalysis.h
@@ -14,11 +14,12 @@
 #ifndef LLVM_ANALYSIS_FUNCTIONPROPERTIESANALYSIS_H
 #define LLVM_ANALYSIS_FUNCTIONPROPERTIESANALYSIS_H
 
-#include "llvm/ADT/iterator_range.h"
-#include "llvm/IR/InstrTypes.h"
+#include "llvm/ADT/DenseSet.h"
 #include "llvm/IR/PassManager.h"
 
 namespace llvm {
+class BasicBlock;
+class CallBase;
 class DominatorTree;
 class Function;
 class LoopInfo;
diff --git a/llvm/include/llvm/Analysis/IRSimilarityIdentifier.h b/llvm/include/llvm/Analysis/IRSimilarityIdentifier.h
index 0d19de6edc2a77a..9da465ddf75f9b8 100644
--- a/llvm/include/llvm/Analysis/IRSimilarityIdentifier.h
+++ b/llvm/include/llvm/Analysis/IRSimilarityIdentifier.h
@@ -57,7 +57,6 @@
 #include <optional>
 
 namespace llvm {
-class Module;
 
 namespace IRSimilarity {
 
diff --git a/llvm/include/llvm/Analysis/IVDescriptors.h b/llvm/include/llvm/Analysis/IVDescriptors.h
index 5c7b613ac48c400..379b114b79cdfaf 100644
--- a/llvm/include/llvm/Analysis/IVDescriptors.h
+++ b/llvm/include/llvm/Analysis/IVDescriptors.h
@@ -23,7 +23,6 @@ namespace llvm {
 class AssumptionCache;
 class DemandedBits;
 class DominatorTree;
-class Instruction;
 class Loop;
 class PredicatedScalarEvolution;
 class ScalarEvolution;
diff --git a/llvm/include/llvm/Analysis/InlineAdvisor.h b/llvm/include/llvm/Analysis/InlineAdvisor.h
index 5f36ee6f68abbb7..871a6e97861e297 100644
--- a/llvm/include/llvm/Analysis/InlineAdvisor.h
+++ b/llvm/include/llvm/Analysis/InlineAdvisor.h
@@ -12,7 +12,6 @@
 #include "llvm/Analysis/CGSCCPassManager.h"
 #include "llvm/Analysis/InlineCost.h"
 #include "llvm/Analysis/LazyCallGraph.h"
-#include "llvm/Config/llvm-config.h"
 #include "llvm/IR/PassManager.h"
 #include <memory>
 
diff --git a/llvm/include/llvm/Analysis/InlineOrder.h b/llvm/include/llvm/Analysis/InlineOrder.h
index a1d25a25b8568f7..2fa2d6091303ad8 100644
--- a/llvm/include/llvm/Analysis/InlineOrder.h
+++ b/llvm/include/llvm/Analysis/InlineOrder.h
@@ -9,12 +9,12 @@
 #ifndef LLVM_ANALYSIS_INLINEORDER_H
 #define LLVM_ANALYSIS_INLINEORDER_H
 
-#include "llvm/ADT/STLFunctionalExtras.h"
 #include "llvm/Analysis/InlineCost.h"
 #include <utility>
 
 namespace llvm {
 class CallBase;
+template <typename Fn> class function_ref;
 
 template <typename T> class InlineOrder {
 public:
diff --git a/llvm/include/llvm/Analysis/InstructionSimplify.h b/llvm/include/llvm/Analysis/InstructionSimplify.h
index 053367a78bfcf03..cf7d3e044188a60 100644
--- a/llvm/include/llvm/Analysis/InstructionSimplify.h
+++ b/llvm/include/llvm/Analysis/InstructionSimplify.h
@@ -39,7 +39,6 @@ namespace llvm {
 template <typename T, typename... TArgs> class AnalysisManager;
 template <class T> class ArrayRef;
 class AssumptionCache;
-class BinaryOperator;
 class CallBase;
 class DataLayout;
 class DominatorTree;
diff --git a/llvm/include/llvm/Analysis/InteractiveModelRunner.h b/llvm/include/llvm/Analysis/InteractiveModelRunner.h
index 680dc4249d80b17..798a249b9e4302b 100644
--- a/llvm/include/llvm/Analysis/InteractiveModelRunner.h
+++ b/llvm/include/llvm/Analysis/InteractiveModelRunner.h
@@ -13,9 +13,6 @@
 #include "llvm/Analysis/MLModelRunner.h"
 #include "llvm/Analysis/TensorSpec.h"
 #include "llvm/Analysis/Utils/TrainingLogger.h"
-#include "llvm/Config/llvm-config.h"
-#include "llvm/Support/FileSystem.h"
-#include "llvm/Support/raw_ostream.h"
 #include <system_error>
 
 namespace llvm {
diff --git a/llvm/include/llvm/Analysis/LazyBlockFrequencyInfo.h b/llvm/include/llvm/Analysis/LazyBlockFrequencyInfo.h
index a6d8b76b12aea24..8ad3c6bafc26d3c 100644
--- a/llvm/include/llvm/Analysis/LazyBlockFrequencyInfo.h
+++ b/llvm/include/llvm/Analysis/LazyBlockFrequencyInfo.h
@@ -21,7 +21,6 @@
 #include "llvm/Pass.h"
 
 namespace llvm {
-class AnalysisUsage;
 class Function;
 class LoopInfo;
 
@@ -125,5 +124,5 @@ class LazyBlockFrequencyInfoPass : public FunctionPass {
 
 /// Helper for client passes to initialize dependent passes for LBFI.
 void initializeLazyBFIPassPass(PassRegistry &Registry);
-}
+} // namespace llvm
 #endif
diff --git a/llvm/include/llvm/Analysis/LazyBranchProbabilityInfo.h b/llvm/include/llvm/Analysis/LazyBranchProbabilityInfo.h
index bad7423616b453b..fe1f7b25c0f159c 100644
--- a/llvm/include/llvm/Analysis/LazyBranchProbabilityInfo.h
+++ b/llvm/include/llvm/Analysis/LazyBranchProbabilityInfo.h
@@ -20,7 +20,6 @@
 #include "llvm/Pass.h"
 
 namespace llvm {
-class AnalysisUsage;
 class Function;
 class LoopInfo;
 class TargetLibraryInfo;
@@ -119,5 +118,5 @@ template <> struct BPIPassTrait<LazyBranchProbabilityInfoPass> {
     return P->getBPI();
   }
 };
-}
+} // namespace llvm
 #endif
diff --git a/llvm/include/llvm/Analysis/LazyCallGraph.h b/llvm/include/llvm/Analysis/LazyCallGraph.h
index a8bbf2c578af9b4..e67a0388bcaa605 100644
--- a/llvm/include/llvm/Analysis/LazyCallGraph.h
+++ b/llvm/include/llvm/Analysis/LazyCallGraph.h
@@ -55,11 +55,8 @@
 namespace llvm {
 
 class Constant;
-class Function;
 template <class GraphType> struct GraphTraits;
 class Module;
-class TargetLibraryInfo;
-class Value;
 
 /// A lazily constructed view of the call graph of a module.
 ///
@@ -111,7 +108,6 @@ class LazyCallGraph {
 public:
   class Node;
   class EdgeSequence;
-  class SCC;
   class RefSCC;
 
   /// A class used to represent edges in the call graph.
diff --git a/llvm/include/llvm/Analysis/LazyValueInfo.h b/llvm/include/llvm/Analysis/LazyValueInfo.h
index 3ca3e5a146e737f..52c7f68361a5b9f 100644
--- a/llvm/include/llvm/Analysis/LazyValueInfo.h
+++ b/llvm/include/llvm/Analysis/LazyValueInfo.h
@@ -22,11 +22,9 @@ namespace llvm {
   class AssumptionCache;
   class BasicBlock;
   class Constant;
-  class ConstantRange;
   class DataLayout;
   class DominatorTree;
   class Instruction;
-  class TargetLibraryInfo;
   class Value;
   class Use;
   class LazyValueInfoImpl;
diff --git a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
index 1975d4aef9a6d70..87c70abba30fcdb 100644
--- a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
+++ b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h
@@ -15,8 +15,7 @@
 #define LLVM_ANALYSIS_LOOPACCESSANALYSIS_H
 
 #include "llvm/ADT/EquivalenceClasses.h"
-#include "llvm/Analysis/LoopAnalysisManager.h"
-#include "llvm/Analysis/ScalarEvolutionExpressions.h"
+#include "llvm/Analysis/ScalarEvolution.h"
 #include "llvm/IR/DiagnosticInfo.h"
 #include <optional>
 #include <variant>
@@ -26,11 +25,8 @@ namespace llvm {
 class AAResults;
 class DataLayout;
 class Loop;
-class LoopAccessInfo;
 class raw_ostream;
-class SCEV;
-class SCEVUnionPredicate;
-class Value;
+class TargetTransformInfo;
 
 /// Collection of parameters shared beetween the Loop Vectorizer and the
 /// Loop Access Analysis.
diff --git a/llvm/include/llvm/Analysis/LoopInfo.h b/llvm/include/llvm/Analysis/LoopInfo.h
index 054206925483c59..748b7e11cbde929 100644
--- a/llvm/include/llvm/Analysis/LoopInfo.h
+++ b/llvm/include/llvm/Analysis/LoopInfo.h
@@ -14,13 +14,10 @@
 #define LLVM_ANALYSIS_LOOPINFO_H
 
 #include "llvm/ADT/GraphTraits.h"
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/IR/CFG.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/Pass.h"
 #include "llvm/Support/GenericLoopInfo.h"
-#include <algorithm>
 #include <optional>
 #include <utility>
 
@@ -28,10 +25,8 @@ namespace llvm {
 
 class DominatorTree;
 class InductionDescriptor;
-class Instruction;
 class LoopInfo;
 class Loop;
-class MDNode;
 class MemorySSAUpdater;
 class ScalarEvolution;
 class raw_ostream;
diff --git a/llvm/include/llvm/Analysis/MLInlineAdvisor.h b/llvm/include/llvm/Analysis/MLInlineAdvisor.h
index 7d726a246ca3cb7..580dd5e95d76043 100644
--- a/llvm/include/llvm/Analysis/MLInlineAdvisor.h
+++ b/llvm/include/llvm/Analysis/MLInlineAdvisor.h
@@ -13,10 +13,8 @@
 #include "llvm/Analysis/InlineAdvisor.h"
 #include "llvm/Analysis/LazyCallGraph.h"
 #include "llvm/Analysis/MLModelRunner.h"
-#include "llvm/Analysis/ProfileSummaryInfo.h"
 #include "llvm/IR/PassManager.h"
 
-#include <deque>
 #include <map>
 #include <memory>
 #include <optional>
@@ -25,6 +23,7 @@ namespace llvm {
 class DiagnosticInfoOptimizationBase;
 class Module;
 class MLInlineAdvice;
+class ProfileSummaryInfo;
 
 class MLInlineAdvisor : public InlineAdvisor {
 public:
diff --git a/llvm/include/llvm/Analysis/MemoryBuiltins.h b/llvm/include/llvm/Analysis/MemoryBuiltins.h
index bb282a1b73d320b..7b48844cc9e8e9b 100644
--- a/llvm/include/llvm/Analysis/MemoryBuiltins.h
+++ b/llvm/include/llvm/Analysis/MemoryBuiltins.h
@@ -18,7 +18,6 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/Analysis/TargetFolder.h"
-#include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/InstVisitor.h"
 #include "llvm/IR/ValueHandle.h"
diff --git a/llvm/include/llvm/Analysis/MemoryDependenceAnalysis.h b/llvm/include/llvm/Analysis/MemoryDependenceAnalysis.h
index d5b2eb6253db959..decb33e6af6bcb5 100644
--- a/llvm/include/llvm/Analysis/MemoryDependenceAnalysis.h
+++ b/llvm/include/llvm/Analysis/MemoryDependenceAnalysis.h
@@ -29,7 +29,6 @@
 namespace llvm {
 
 class AssumptionCache;
-class BatchAAResults;
 class DominatorTree;
 class PHITransAddr;
 
diff --git a/llvm/include/llvm/Analysis/MemoryLocation.h b/llvm/include/llvm/Analysis/MemoryLocation.h
index 7d896c44f46795d..ea29e21bd18f2fa 100644
--- a/llvm/include/llvm/Analysis/MemoryLocation.h
+++ b/llvm/include/llvm/Analysis/MemoryLocation.h
@@ -37,7 +37,6 @@ class AnyMemTransferInst;
 class AnyMemIntrinsic;
 class TargetLibraryInfo;
 class VAArgInst;
-class Value;
 
 // Represents the size of a MemoryLocation. Logically, it's an
 // std::optional<uint63_t> that also carries a bit to represent whether the
diff --git a/llvm/include/llvm/Analysis/MemoryProfileInfo.h b/llvm/include/llvm/Analysis/MemoryProfileInfo.h
index 75ddf5cacf74d5c..edbce706953d18d 100644
--- a/llvm/include/llvm/Analysis/MemoryProfileInfo.h
+++ b/llvm/include/llvm/Analysis/MemoryProfileInfo.h
@@ -13,10 +13,7 @@
 #ifndef LLVM_ANALYSIS_MEMORYPROFILEINFO_H
 #define LLVM_ANALYSIS_MEMORYPROFILEINFO_H
 
-#include "llvm/IR/Constants.h"
-#include "llvm/IR/InstrTypes.h"
 #include "llvm/IR/Metadata.h"
-#include "llvm/IR/Module.h"
 #include "llvm/IR/ModuleSummaryIndex.h"
 #include <map>
 
diff --git a/llvm/include/llvm/Analysis/MemorySSA.h b/llvm/include/llvm/Analysis/MemorySSA.h
index ac828021dd2ae1c..43fea6ba27ec4f4 100644
--- a/llvm/include/llvm/Analysis/MemorySSA.h
+++ b/llvm/include/llvm/Analysis/MemorySSA.h
@@ -108,16 +108,12 @@
 namespace llvm {
 
 template <class GraphType> struct GraphTraits;
-class BasicBlock;
 class Function;
 class Loop;
-class Instruction;
 class LLVMContext;
 class MemoryAccess;
 class MemorySSAWalker;
 class Module;
-class Use;
-class Value;
 class raw_ostream;
 
 namespace MSSAHelpers {
diff --git a/llvm/include/llvm/Analysis/MustExecute.h b/llvm/include/llvm/Analysis/MustExecute.h
index 468d94e7cd68b91..8ac3c5eb653cd4f 100644
--- a/llvm/include/llvm/Analysis/MustExecute.h
+++ b/llvm/include/llvm/Analysis/MustExecute.h
@@ -37,7 +37,6 @@ template <typename T> using GetterTy = std::function<T *(const Function &F)>;
 
 class BasicBlock;
 class DominatorTree;
-class Instruction;
 class Loop;
 class LoopInfo;
 class PostDominatorTree;
diff --git a/llvm/include/llvm/Analysis/NoInferenceModelRunner.h b/llvm/include/llvm/Analysis/NoInferenceModelRunner.h
index 980b40500d7cc3e..035fb2234b24579 100644
--- a/llvm/include/llvm/Analysis/NoInferenceModelRunner.h
+++ b/llvm/include/llvm/Analysis/NoInferenceModelRunner.h
@@ -11,9 +11,9 @@
 #define LLVM_ANALYSIS_NOINFERENCEMODELRUNNER_H
 
 #include "llvm/Analysis/MLModelRunner.h"
-#include "llvm/Analysis/TensorSpec.h"
-#include "llvm/Config/llvm-config.h"
 namespace llvm {
+class TensorSpec;
+
 /// A pseudo model runner. We use it to store feature values when collecting
 /// logs for the default policy, in 'development' mode, but never ask it to
 /// 'run'.
diff --git a/llvm/include/llvm/Analysis/ObjCARCAliasAnalysis.h b/llvm/include/llvm/Analysis/ObjCARCAliasAnalysis.h
index 1a154c648fe6115..8195fb93938b063 100644
--- a/llvm/include/llvm/Analysis/ObjCARCAliasAnalysis.h
+++ b/llvm/include/llvm/Analysis/ObjCARCAliasAnalysis.h
@@ -23,7 +23,6 @@
 #define LLVM_ANALYSIS_OBJCARCALIASANALYSIS_H
 
 #include "llvm/Analysis/AliasAnalysis.h"
-#include "llvm/Pass.h"
 
 namespace llvm {
 namespace objcarc {
diff --git a/llvm/include/llvm/Analysis/OptimizationRemarkEmitter.h b/llvm/include/llvm/Analysis/OptimizationRemarkEmitter.h
index f14344e743759f6..7b14e55782adbcb 100644
--- a/llvm/include/llvm/Analysis/OptimizationRemarkEmitter.h
+++ b/llvm/include/llvm/Analysis/OptimizationRemarkEmitter.h
@@ -22,8 +22,6 @@
 #include <optional>
 
 namespace llvm {
-class Function;
-class Value;
 
 /// The optimization diagnostic interface.
 ///
@@ -174,5 +172,5 @@ class OptimizationRemarkEmitterAnalysis
   /// Run the analysis pass over a function and produce BFI.
   Result run(Function &F, FunctionAnalysisManager &AM);
 };
-}
+} // namespace llvm
 #endif // LLVM_ANALYSIS_OPTIMIZATIONREMARKEMITTER_H
diff --git a/llvm/include/llvm/Analysis/ProfileSummaryInfo.h b/llvm/include/llvm/Analysis/ProfileSummaryInfo.h
index 73be9e1d74a33ba..ceae3e8a0ddb959 100644
--- a/llvm/include/llvm/Analysis/ProfileSummaryInfo.h
+++ b/llvm/include/llvm/Analysis/ProfileSummaryInfo.h
@@ -15,18 +15,17 @@
 #define LLVM_ANALYSIS_PROFILESUMMARYINFO_H
 
 #include "llvm/ADT/DenseMap.h"
-#include "llvm/Analysis/BlockFrequencyInfo.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/IR/ProfileSummary.h"
 #include "llvm/Pass.h"
+#include "llvm/Support/BlockFrequency.h"
 #include <memory>
 #include <optional>
 
 namespace llvm {
-class BasicBlock;
-class CallBase;
+class BlockFrequencyInfo;
 class MachineFunction;
 
 /// Analysis providing profile information.
diff --git a/llvm/include/llvm/Analysis/PtrUseVisitor.h b/llvm/include/llvm/Analysis/PtrUseVisitor.h
index 86206b2d5e9f883..b6cc14d2077af03 100644
--- a/llvm/include/llvm/Analysis/PtrUseVisitor.h
+++ b/llvm/include/llvm/Analysis/PtrUseVisitor.h
@@ -34,7 +34,6 @@
 
 namespace llvm {
 class DataLayout;
-class Use;
 
 namespace detail {
 
diff --git a/llvm/include/llvm/Analysis/RegionInfo.h b/llvm/include/llvm/Analysis/RegionInfo.h
index fc8df36ec287db9..474bd3a9483e717 100644
--- a/llvm/include/llvm/Analysis/RegionInfo.h
+++ b/llvm/include/llvm/Analysis/RegionInfo.h
@@ -41,7 +41,6 @@
 #include "llvm/ADT/GraphTraits.h"
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/iterator_range.h"
-#include "llvm/Config/llvm-config.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/Pass.h"
@@ -56,7 +55,6 @@
 
 namespace llvm {
 
-class BasicBlock;
 class DominanceFrontier;
 class Loop;
 class LoopInfo;
diff --git a/llvm/include/llvm/Analysis/RegionPrinter.h b/llvm/include/llvm/Analysis/RegionPrinter.h
index 501a5406236e926..814d085baf0ad64 100644
--- a/llvm/include/llvm/Analysis/RegionPrinter.h
+++ b/llvm/include/llvm/Analysis/RegionPrinter.h
@@ -14,13 +14,13 @@
 #ifndef LLVM_ANALYSIS_REGIONPRINTER_H
 #define LLVM_ANALYSIS_REGIONPRINTER_H
 
-#include "llvm/Analysis/DOTGraphTraitsPass.h"
-#include "llvm/Analysis/RegionInfo.h"
+#include "llvm/Support/DOTGraphTraits.h"
 
 namespace llvm {
   class FunctionPass;
   class Function;
   class RegionInfo;
+  class RegionNode;
 
   FunctionPass *createRegionViewerPass();
   FunctionPass *createRegionOnlyViewerPass();
diff --git a/llvm/include/llvm/Analysis/ReplayInlineAdvisor.h b/llvm/include/llvm/Analysis/ReplayInlineAdvisor.h
index 0c5b566f60a4704..6f6ce763746a16c 100644
--- a/llvm/include/llvm/Analysis/ReplayInlineAdvisor.h
+++ b/llvm/include/llvm/Analysis/ReplayInlineAdvisor.h
@@ -14,7 +14,6 @@
 
 namespace llvm {
 class CallBase;
-class Function;
 class LLVMContext;
 class Module;
 
diff --git a/llvm/include/llvm/Analysis/ScalarEvolution.h b/llvm/include/llvm/Analysis/ScalarEvolution.h
index f413e77877f9038..5154e2f6659c121 100644
--- a/llvm/include/llvm/Analysis/ScalarEvolution.h
+++ b/llvm/include/llvm/Analysis/ScalarEvolution.h
@@ -30,7 +30,6 @@
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/IR/ConstantRange.h"
-#include "llvm/IR/InstrTypes.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/IR/ValueHandle.h"
@@ -51,9 +50,7 @@ class Constant;
 class ConstantInt;
 class DataLayout;
 class DominatorTree;
-class Function;
 class GEPOperator;
-class Instruction;
 class LLVMContext;
 class Loop;
 class LoopInfo;
@@ -64,7 +61,6 @@ class SCEVUnknown;
 class StructType;
 class TargetLibraryInfo;
 class Type;
-class Value;
 enum SCEVTypes : unsigned short;
 
 extern bool VerifySCEV;
diff --git a/llvm/include/llvm/Analysis/SyntheticCountsUtils.h b/llvm/include/llvm/Analysis/SyntheticCountsUtils.h
index 5a6c7dd53c7e557..8b61ac5ed8ca22d 100644
--- a/llvm/include/llvm/Analysis/SyntheticCountsUtils.h
+++ b/llvm/include/llvm/Analysis/SyntheticCountsUtils.h
@@ -13,8 +13,8 @@
 #ifndef LLVM_ANALYSIS_SYNTHETICCOUNTSUTILS_H
 #define LLVM_ANALYSIS_SYNTHETICCOUNTSUTILS_H
 
+#include "llvm/ADT/GraphTraits.h"
 #include "llvm/ADT/STLFunctionalExtras.h"
-#include "llvm/Analysis/CallGraph.h"
 #include "llvm/Support/ScaledNumber.h"
 
 namespace llvm {
diff --git a/llvm/include/llvm/Analysis/TargetLibraryInfo.h b/llvm/include/llvm/Analysis/TargetLibraryInfo.h
index d0506a89b88ef46..8d848696bab874c 100644
--- a/llvm/include/llvm/Analysis/TargetLibraryInfo.h
+++ b/llvm/include/llvm/Analysis/TargetLibraryInfo.h
@@ -20,9 +20,6 @@
 namespace llvm {
 
 template <typename T> class ArrayRef;
-class Function;
-class Module;
-class Triple;
 
 /// Provides info so a possible vectorization of a function can be
 /// computed. Function 'VectorFnName' is equivalent to 'ScalarFnName'
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 38e8b9da213974b..dde90abc06cd9ce 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -22,7 +22,6 @@
 #define LLVM_ANALYSIS_TARGETTRANSFORMINFO_H
 
 #include "llvm/ADT/APInt.h"
-#include "llvm/ADT/SmallBitVector.h"
 #include "llvm/IR/FMF.h"
 #include "llvm/IR/InstrTypes.h"
 #include "llvm/IR/PassManager.h"
@@ -45,7 +44,6 @@ class AssumptionCache;
 class BlockFrequencyInfo;
 class DominatorTree;
 class BranchInst;
-class CallBase;
 class Function;
 class GlobalValue;
 class InstCombiner;
@@ -60,12 +58,11 @@ class ProfileSummaryInfo;
 class RecurrenceDescriptor;
 class SCEV;
 class ScalarEvolution;
+class SmallBitVector;
 class StoreInst;
 class SwitchInst;
 class TargetLibraryInfo;
 class Type;
-class User;
-class Value;
 class VPIntrinsic;
 struct KnownBits;
 
diff --git a/llvm/include/llvm/Analysis/UniformityAnalysis.h b/llvm/include/llvm/Analysis/UniformityAnalysis.h
index c38d100d88b8a7c..a067a9b42893cd0 100644
--- a/llvm/include/llvm/Analysis/UniformityAnalysis.h
+++ b/llvm/include/llvm/Analysis/UniformityAnalysis.h
@@ -15,7 +15,9 @@
 #define LLVM_ANALYSIS_UNIFORMITYANALYSIS_H
 
 #include "llvm/ADT/GenericUniformityInfo.h"
-#include "llvm/Analysis/CycleAnalysis.h"
+#include "llvm/IR/PassManager.h"
+#include "llvm/IR/SSAContext.h"
+#include "llvm/Pass.h"
 
 namespace llvm {
 
diff --git a/llvm/include/llvm/Analysis/ValueLattice.h b/llvm/include/llvm/Analysis/ValueLattice.h
index fa56d838a385980..a704cce0c082b9f 100644
--- a/llvm/include/llvm/Analysis/ValueLattice.h
+++ b/llvm/include/llvm/Analysis/ValueLattice.h
@@ -9,9 +9,8 @@
 #ifndef LLVM_ANALYSIS_VALUELATTICE_H
 #define LLVM_ANALYSIS_VALUELATTICE_H
 
-#include "llvm/IR/Constants.h"
 #include "llvm/IR/ConstantRange.h"
-#include "llvm/IR/Instructions.h"
+#include "llvm/IR/Constants.h"
 
 //===----------------------------------------------------------------------===//
 //                               ValueLatticeElement
@@ -19,8 +18,6 @@
 
 namespace llvm {
 
-class Constant;
-
 /// This class represents lattice values for constants.
 ///
 /// FIXME: This is basically just for bringup, this can be made a lot more rich
diff --git a/llvm/include/llvm/Analysis/ValueTracking.h b/llvm/include/llvm/Analysis/ValueTracking.h
index 96fa16970584da3..00ead1181d76258 100644
--- a/llvm/include/llvm/Analysis/ValueTracking.h
+++ b/llvm/include/llvm/Analysis/ValueTracking.h
@@ -14,7 +14,6 @@
 #ifndef LLVM_ANALYSIS_VALUETRACKING_H
 #define LLVM_ANALYSIS_VALUETRACKING_H
 
-#include "llvm/ADT/ArrayRef.h"
 #include "llvm/Analysis/SimplifyQuery.h"
 #include "llvm/Analysis/WithCache.h"
 #include "llvm/IR/Constants.h"
@@ -30,12 +29,9 @@ namespace llvm {
 
 class Operator;
 class AddOperator;
-class AllocaInst;
-class APInt;
 class AssumptionCache;
 class DominatorTree;
 class GEPOperator;
-class LoadInst;
 class WithOverflowInst;
 struct KnownBits;
 class Loop;
@@ -43,7 +39,7 @@ class LoopInfo;
 class MDNode;
 class StringRef;
 class TargetLibraryInfo;
-class Value;
+template <typename T> class ArrayRef;
 
 constexpr unsigned MaxAnalysisRecursionDepth = 6;
 
diff --git a/llvm/include/llvm/Analysis/VectorUtils.h b/llvm/include/llvm/Analysis/VectorUtils.h
index 35283637027db84..e2dd4976f39065c 100644
--- a/llvm/include/llvm/Analysis/VectorUtils.h
+++ b/llvm/include/llvm/Analysis/VectorUtils.h
@@ -120,9 +120,7 @@ class DemandedBits;
 template <typename InstTy> class InterleaveGroup;
 class IRBuilderBase;
 class Loop;
-class ScalarEvolution;
 class TargetTransformInfo;
-class Type;
 class Value;
 
 namespace Intrinsic {
diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp
index c4c174977c47589..eff8a7cfc8ce6c9 100644
--- a/llvm/lib/Analysis/InstructionSimplify.cpp
+++ b/llvm/lib/Analysis/InstructionSimplify.cpp
@@ -30,6 +30,7 @@
 #include "llvm/Analysis/LoopAnalysisManager.h"
 #include "llvm/Analysis/MemoryBuiltins.h"
 #include "llvm/Analysis/OverflowInstAnalysis.h"
+#include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/Analysis/VectorUtils.h"
 #include "llvm/IR/ConstantRange.h"
diff --git a/llvm/lib/Analysis/MemoryProfileInfo.cpp b/llvm/lib/Analysis/MemoryProfileInfo.cpp
index c81c65fd4e691e3..3a752d42241a95e 100644
--- a/llvm/lib/Analysis/MemoryProfileInfo.cpp
+++ b/llvm/lib/Analysis/MemoryProfileInfo.cpp
@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Analysis/MemoryProfileInfo.h"
+#include "llvm/IR/Constants.h"
 #include "llvm/Support/CommandLine.h"
 
 using namespace llvm;
diff --git a/llvm/lib/Analysis/ValueLattice.cpp b/llvm/lib/Analysis/ValueLattice.cpp
index 1d2177a92eb4654..5001fc2d310dd43 100644
--- a/llvm/lib/Analysis/ValueLattice.cpp
+++ b/llvm/lib/Analysis/ValueLattice.cpp
@@ -8,6 +8,7 @@
 
 #include "llvm/Analysis/ValueLattice.h"
 #include "llvm/Analysis/ConstantFolding.h"
+#include "llvm/IR/Instructions.h"
 
 namespace llvm {
 Constant *
diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp
index ec5b7579c66c262..48253a613b41d2b 100644
--- a/llvm/lib/CodeGen/CodeGenPrepare.cpp
+++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp
@@ -27,6 +27,7 @@
 #include "llvm/Analysis/InstructionSimplify.h"
 #include "llvm/Analysis/LoopInfo.h"
 #include "llvm/Analysis/ProfileSummaryInfo.h"
+#include "llvm/Analysis/ScalarEvolutionExpressions.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/Analysis/ValueTracking.h"
diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index bba3c5f3229f5cf..70a6e74b94d55cf 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -13,6 +13,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
+#include "llvm/ADT/SmallBitVector.h"
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
@@ -39,8 +40,8 @@
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/MDBuilder.h"
 #include "llvm/IR/Metadata.h"
-#include "llvm/IR/PassManager.h"
 #include "llvm/IR/PassInstrumentation.h"
+#include "llvm/IR/PassManager.h"
 #include "llvm/IR/ReplaceConstant.h"
 #include "llvm/IR/Value.h"
 #include "llvm/MC/TargetRegistry.h"
diff --git a/llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp b/llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp
index 2e66c81c3324492..273bd7b2b361e70 100644
--- a/llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp
@@ -21,7 +21,7 @@
 #include "llvm/Analysis/AssumptionCache.h"
 #include "llvm/Analysis/InstSimplifyFolder.h"
 #include "llvm/Analysis/InstructionSimplify.h"
-#include "llvm/Analysis/ScalarEvolution.h"
+#include "llvm/Analysis/ScalarEvolutionExpressions.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/Analysis/VectorUtils.h"
diff --git a/llvm/lib/Transforms/Utils/SizeOpts.cpp b/llvm/lib/Transforms/Utils/SizeOpts.cpp
index 1ca2e0e6ebb9043..09c4c1c3c511ff8 100644
--- a/llvm/lib/Transforms/Utils/SizeOpts.cpp
+++ b/llvm/lib/Transforms/Utils/SizeOpts.cpp
@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Transforms/Utils/SizeOpts.h"
+#include "llvm/Analysis/BlockFrequencyInfo.h"
 
 using namespace llvm;
 
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 0417916546bb109..66a779da8c25bca 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -18,6 +18,7 @@
 #include "llvm/Analysis/Loads.h"
 #include "llvm/Analysis/LoopInfo.h"
 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
+#include "llvm/Analysis/ScalarEvolutionExpressions.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/Analysis/ValueTracking.h"
diff --git a/llvm/unittests/Analysis/MemoryProfileInfoTest.cpp b/llvm/unittests/Analysis/MemoryProfileInfoTest.cpp
index 1ba402276e036db..4c177ae8446905a 100644
--- a/llvm/unittests/Analysis/MemoryProfileInfoTest.cpp
+++ b/llvm/unittests/Analysis/MemoryProfileInfoTest.cpp
@@ -8,6 +8,7 @@
 
 #include "llvm/Analysis/MemoryProfileInfo.h"
 #include "llvm/AsmParser/Parser.h"
+#include "llvm/IR/Constants.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Module.h"
diff --git a/mlir/include/mlir/Analysis/Presburger/Fraction.h b/mlir/include/mlir/Analysis/Presburger/Fraction.h
index f4f1be97147bf63..585a6d48506d245 100644
--- a/mlir/include/mlir/Analysis/Presburger/Fraction.h
+++ b/mlir/include/mlir/Analysis/Presburger/Fraction.h
@@ -15,6 +15,7 @@
 #define MLIR_ANALYSIS_PRESBURGER_FRACTION_H
 
 #include "llvm/ADT/DynamicAPInt.h"
+#include "llvm/Support/raw_ostream.h"
 
 namespace mlir {
 namespace presburger {

From 27d37ee4d067f42e9a46a0871d3cb961323e5c85 Mon Sep 17 00:00:00 2001
From: Vlad Serebrennikov <serebrennikov.vladislav@gmail.com>
Date: Sat, 17 Aug 2024 03:32:07 +0300
Subject: [PATCH 267/441] [clang][NFC] Clean up `Sema` headers

When various `Sema*.h` and `Sema*.cpp` files were created, cleanup of
`Sema.h` includes and forward declarations was left for the later.
Now's the time. This commit touches `Sema.h` and Sema components:
1. Unused includes are removed.
2. Unused forward declarations are removed.
3. Missing includes are added (those files are largely IWYU-clean now).
4. Includes were converted into forward declarations where possible.

As this commit focuses on headers, all changes to `.cpp` files were
minimal, and were aiming at keeping everything buildable.
---
 clang/include/clang/AST/Attr.h                |   2 +-
 clang/include/clang/Sema/Sema.h               | 156 +++++++-----------
 clang/include/clang/Sema/SemaAMDGPU.h         |   9 +-
 clang/include/clang/Sema/SemaARM.h            |  19 ++-
 clang/include/clang/Sema/SemaAVR.h            |   2 +-
 clang/include/clang/Sema/SemaBPF.h            |   4 +-
 clang/include/clang/Sema/SemaCUDA.h           |  20 ++-
 clang/include/clang/Sema/SemaCodeCompletion.h |  10 +-
 clang/include/clang/Sema/SemaHLSL.h           |  11 +-
 clang/include/clang/Sema/SemaHexagon.h        |   2 +-
 clang/include/clang/Sema/SemaLoongArch.h      |   5 +-
 clang/include/clang/Sema/SemaM68k.h           |   2 +-
 clang/include/clang/Sema/SemaMIPS.h           |   5 +-
 clang/include/clang/Sema/SemaMSP430.h         |   2 +-
 clang/include/clang/Sema/SemaNVPTX.h          |   5 +-
 clang/include/clang/Sema/SemaObjC.h           |  16 +-
 clang/include/clang/Sema/SemaOpenACC.h        |   6 +
 clang/include/clang/Sema/SemaOpenCL.h         |   2 +-
 clang/include/clang/Sema/SemaOpenMP.h         |  19 ++-
 clang/include/clang/Sema/SemaPPC.h            |   5 +-
 clang/include/clang/Sema/SemaPseudoObject.h   |   4 +-
 clang/include/clang/Sema/SemaRISCV.h          |  13 +-
 clang/include/clang/Sema/SemaSYCL.h           |   2 +-
 clang/include/clang/Sema/SemaSwift.h          |   4 +-
 clang/include/clang/Sema/SemaSystemZ.h        |   2 +-
 clang/include/clang/Sema/SemaWasm.h           |   9 +-
 clang/include/clang/Sema/SemaX86.h            |   5 +-
 clang/lib/Parse/ParseExpr.cpp                 |   2 +
 clang/lib/Parse/ParseObjc.cpp                 |   1 +
 clang/lib/Sema/Sema.cpp                       |   1 +
 clang/lib/Sema/SemaARM.cpp                    |   1 +
 clang/lib/Sema/SemaDecl.cpp                   |   1 +
 clang/lib/Sema/SemaExpr.cpp                   |   1 +
 clang/lib/Sema/SemaExprObjC.cpp               |   1 +
 clang/lib/Sema/SemaLambda.cpp                 |   1 +
 clang/lib/Sema/SemaMIPS.cpp                   |   1 +
 clang/lib/Sema/SemaObjC.cpp                   |   1 +
 clang/lib/Sema/SemaPPC.cpp                    |   1 +
 clang/lib/Sema/SemaSYCL.cpp                   |   1 +
 clang/lib/Sema/SemaTemplateDeduction.cpp      |   1 +
 .../lib/Sema/SemaTemplateInstantiateDecl.cpp  |   1 +
 clang/lib/Sema/SemaType.cpp                   |   2 +
 clang/lib/Sema/SemaX86.cpp                    |   1 +
 43 files changed, 178 insertions(+), 181 deletions(-)

diff --git a/clang/include/clang/AST/Attr.h b/clang/include/clang/AST/Attr.h
index 8e9b7ad8b468264..bd1851a26ce2e18 100644
--- a/clang/include/clang/AST/Attr.h
+++ b/clang/include/clang/AST/Attr.h
@@ -372,7 +372,7 @@ class ParamIdx {
 static_assert(sizeof(ParamIdx) == sizeof(ParamIdx::SerialType),
               "ParamIdx does not fit its serialization type");
 
-#include "clang/AST/Attrs.inc"
+#include "clang/AST/Attrs.inc" // IWYU pragma: export
 
 inline const StreamingDiagnostic &operator<<(const StreamingDiagnostic &DB,
                                              const Attr *At) {
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index ea2977838466410..299a916b9abf8d9 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -15,36 +15,42 @@
 #define LLVM_CLANG_SEMA_SEMA_H
 
 #include "clang/APINotes/APINotesManager.h"
-#include "clang/AST/ASTConcept.h"
 #include "clang/AST/ASTFwd.h"
 #include "clang/AST/Attr.h"
-#include "clang/AST/Availability.h"
-#include "clang/AST/ComparisonCategories.h"
+#include "clang/AST/AttrIterator.h"
+#include "clang/AST/CharUnits.h"
+#include "clang/AST/DeclBase.h"
+#include "clang/AST/DeclCXX.h"
 #include "clang/AST/DeclTemplate.h"
 #include "clang/AST/DeclarationName.h"
 #include "clang/AST/Expr.h"
 #include "clang/AST/ExprCXX.h"
 #include "clang/AST/ExprConcepts.h"
-#include "clang/AST/ExprObjC.h"
 #include "clang/AST/ExternalASTSource.h"
-#include "clang/AST/LocInfoType.h"
-#include "clang/AST/MangleNumberingContext.h"
-#include "clang/AST/NSAPI.h"
-#include "clang/AST/PrettyPrinter.h"
+#include "clang/AST/NestedNameSpecifier.h"
+#include "clang/AST/OperationKinds.h"
 #include "clang/AST/StmtCXX.h"
+#include "clang/AST/Type.h"
 #include "clang/AST/TypeLoc.h"
-#include "clang/AST/TypeOrdering.h"
-#include "clang/Basic/BitmaskEnum.h"
+#include "clang/Basic/AttrSubjectMatchRules.h"
 #include "clang/Basic/Builtins.h"
+#include "clang/Basic/CapturedStmt.h"
 #include "clang/Basic/Cuda.h"
-#include "clang/Basic/DarwinSDKInfo.h"
+#include "clang/Basic/DiagnosticSema.h"
+#include "clang/Basic/ExceptionSpecificationType.h"
 #include "clang/Basic/ExpressionTraits.h"
-#include "clang/Basic/IdentifierTable.h"
+#include "clang/Basic/LLVM.h"
+#include "clang/Basic/Lambda.h"
+#include "clang/Basic/LangOptions.h"
 #include "clang/Basic/Module.h"
 #include "clang/Basic/OpenCLOptions.h"
+#include "clang/Basic/OperatorKinds.h"
+#include "clang/Basic/PartialDiagnostic.h"
 #include "clang/Basic/PragmaKinds.h"
+#include "clang/Basic/SourceLocation.h"
 #include "clang/Basic/Specifiers.h"
 #include "clang/Basic/TemplateKinds.h"
+#include "clang/Basic/TokenKinds.h"
 #include "clang/Basic/TypeTraits.h"
 #include "clang/Sema/AnalysisBasedWarnings.h"
 #include "clang/Sema/Attr.h"
@@ -52,124 +58,100 @@
 #include "clang/Sema/DeclSpec.h"
 #include "clang/Sema/ExternalSemaSource.h"
 #include "clang/Sema/IdentifierResolver.h"
-#include "clang/Sema/ObjCMethodList.h"
 #include "clang/Sema/Ownership.h"
+#include "clang/Sema/ParsedAttr.h"
 #include "clang/Sema/Redeclaration.h"
 #include "clang/Sema/Scope.h"
 #include "clang/Sema/SemaBase.h"
-#include "clang/Sema/SemaConcept.h"
-#include "clang/Sema/SemaDiagnostic.h"
 #include "clang/Sema/TypoCorrection.h"
 #include "clang/Sema/Weak.h"
+#include "llvm/ADT/APInt.h"
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/BitmaskEnum.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/FloatingPointMode.h"
+#include "llvm/ADT/FoldingSet.h"
+#include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/PointerIntPair.h"
+#include "llvm/ADT/PointerUnion.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/STLForwardCompat.h"
+#include "llvm/ADT/STLFunctionalExtras.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallBitVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringExtras.h"
+#include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/TinyPtrVector.h"
+#include "llvm/Support/Allocator.h"
+#include "llvm/Support/Compiler.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/ErrorHandling.h"
+#include <cassert>
+#include <climits>
+#include <cstddef>
+#include <cstdint>
 #include <deque>
+#include <functional>
+#include <iterator>
 #include <memory>
 #include <optional>
 #include <string>
 #include <tuple>
+#include <type_traits>
+#include <utility>
 #include <vector>
 
 namespace llvm {
-class APSInt;
-template <typename ValueT, typename ValueInfoT> class DenseSet;
-class SmallBitVector;
 struct InlineAsmIdentifierInfo;
 } // namespace llvm
 
 namespace clang {
 class ADLResult;
+class APValue;
+struct ASTConstraintSatisfaction;
 class ASTConsumer;
 class ASTContext;
+class ASTDeclReader;
 class ASTMutationListener;
 class ASTReader;
 class ASTWriter;
-class ArrayType;
-class ParsedAttr;
-class BindingDecl;
-class BlockDecl;
-class CapturedDecl;
 class CXXBasePath;
 class CXXBasePaths;
-class CXXBindTemporaryExpr;
-typedef SmallVector<CXXBaseSpecifier *, 4> CXXCastPath;
-class CXXConstructorDecl;
-class CXXConversionDecl;
-class CXXDeleteExpr;
-class CXXDestructorDecl;
 class CXXFieldCollector;
-class CXXMemberCallExpr;
-class CXXMethodDecl;
-class CXXScopeSpec;
-class CXXTemporary;
-class CXXTryStmt;
-class CallExpr;
-class ClassTemplateDecl;
-class ClassTemplatePartialSpecializationDecl;
-class ClassTemplateSpecializationDecl;
-class VarTemplatePartialSpecializationDecl;
 class CodeCompleteConsumer;
-class CodeCompletionAllocator;
-class CodeCompletionTUInfo;
-class CodeCompletionResult;
-class CoroutineBodyStmt;
-class Decl;
-class DeclAccessPair;
-class DeclContext;
-class DeclRefExpr;
-class DeclaratorDecl;
+enum class ComparisonCategoryType : unsigned char;
+class ConstraintSatisfaction;
+class DarwinSDKInfo;
+class DeclGroupRef;
 class DeducedTemplateArgument;
+struct DeductionFailureInfo;
 class DependentDiagnostic;
-class DesignatedInitExpr;
 class Designation;
-class EnableIfAttr;
-class EnumConstantDecl;
-class Expr;
-class ExtVectorType;
-class FormatAttr;
-class FriendDecl;
-class FunctionDecl;
-class FunctionProtoType;
-class FunctionTemplateDecl;
+class IdentifierInfo;
 class ImplicitConversionSequence;
 typedef MutableArrayRef<ImplicitConversionSequence> ConversionSequenceList;
-class InitListExpr;
 class InitializationKind;
 class InitializationSequence;
 class InitializedEntity;
-class IntegerLiteral;
-class LabelStmt;
-class LambdaExpr;
-class LangOptions;
+enum class LangAS : unsigned int;
 class LocalInstantiationScope;
 class LookupResult;
-class MacroInfo;
+class MangleNumberingContext;
 typedef ArrayRef<std::pair<IdentifierInfo *, SourceLocation>> ModuleIdPath;
 class ModuleLoader;
 class MultiLevelTemplateArgumentList;
-class NamedDecl;
-class ObjCImplementationDecl;
+struct NormalizedConstraint;
 class ObjCInterfaceDecl;
 class ObjCMethodDecl;
-class ObjCProtocolDecl;
 struct OverloadCandidate;
 enum class OverloadCandidateParamOrder : char;
 enum OverloadCandidateRewriteKind : unsigned;
 class OverloadCandidateSet;
-class OverloadExpr;
-class ParenListExpr;
-class ParmVarDecl;
 class Preprocessor;
-class PseudoDestructorTypeStorage;
-class PseudoObjectExpr;
-class QualType;
 class SemaAMDGPU;
 class SemaARM;
 class SemaAVR;
@@ -196,41 +178,19 @@ class SemaSystemZ;
 class SemaWasm;
 class SemaX86;
 class StandardConversionSequence;
-class Stmt;
-class StringLiteral;
-class SwitchStmt;
 class TemplateArgument;
-class TemplateArgumentList;
 class TemplateArgumentLoc;
-class TemplateDecl;
 class TemplateInstantiationCallback;
-class TemplateParameterList;
 class TemplatePartialOrderingContext;
-class TemplateTemplateParmDecl;
+class TemplateSpecCandidateSet;
 class Token;
-class TypeAliasDecl;
-class TypedefDecl;
-class TypedefNameDecl;
-class TypeLoc;
+class TypeConstraint;
 class TypoCorrectionConsumer;
-class UnqualifiedId;
-class UnresolvedLookupExpr;
-class UnresolvedMemberExpr;
 class UnresolvedSetImpl;
 class UnresolvedSetIterator;
-class UsingDecl;
-class UsingShadowDecl;
-class ValueDecl;
-class VarDecl;
-class VarTemplateSpecializationDecl;
-class VisibilityAttr;
 class VisibleDeclConsumer;
-class IndirectFieldDecl;
-struct DeductionFailureInfo;
-class TemplateSpecCandidateSet;
 
 namespace sema {
-class AccessedEntity;
 class BlockScopeInfo;
 class Capture;
 class CapturedRegionScopeInfo;
@@ -240,8 +200,6 @@ class DelayedDiagnostic;
 class DelayedDiagnosticPool;
 class FunctionScopeInfo;
 class LambdaScopeInfo;
-class PossiblyUnreachableDiag;
-class RISCVIntrinsicManager;
 class SemaPPCallbacks;
 class TemplateDeductionInfo;
 } // namespace sema
diff --git a/clang/include/clang/Sema/SemaAMDGPU.h b/clang/include/clang/Sema/SemaAMDGPU.h
index 969078f552c6ab7..3fdb39202610d04 100644
--- a/clang/include/clang/Sema/SemaAMDGPU.h
+++ b/clang/include/clang/Sema/SemaAMDGPU.h
@@ -13,14 +13,13 @@
 #ifndef LLVM_CLANG_SEMA_SEMAAMDGPU_H
 #define LLVM_CLANG_SEMA_SEMAAMDGPU_H
 
-#include "clang/AST/Attr.h"
-#include "clang/AST/DeclBase.h"
-#include "clang/AST/Expr.h"
-#include "clang/Basic/AttributeCommonInfo.h"
-#include "clang/Sema/ParsedAttr.h"
+#include "clang/AST/ASTFwd.h"
 #include "clang/Sema/SemaBase.h"
 
 namespace clang {
+class AttributeCommonInfo;
+class ParsedAttr;
+
 class SemaAMDGPU : public SemaBase {
 public:
   SemaAMDGPU(Sema &S);
diff --git a/clang/include/clang/Sema/SemaARM.h b/clang/include/clang/Sema/SemaARM.h
index fedc7df7908f17c..b8196a3170d63c6 100644
--- a/clang/include/clang/Sema/SemaARM.h
+++ b/clang/include/clang/Sema/SemaARM.h
@@ -13,15 +13,18 @@
 #ifndef LLVM_CLANG_SEMA_SEMAARM_H
 #define LLVM_CLANG_SEMA_SEMAARM_H
 
-#include "clang/AST/DeclBase.h"
-#include "clang/AST/Expr.h"
-#include "clang/Basic/TargetInfo.h"
+#include "clang/AST/ASTFwd.h"
 #include "clang/Sema/SemaBase.h"
-#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
 #include <tuple>
 
+namespace llvm {
+template <typename T, unsigned N> class SmallVector;
+} // namespace llvm
+
 namespace clang {
 class ParsedAttr;
+class TargetInfo;
 
 class SemaARM : public SemaBase {
 public:
@@ -60,10 +63,10 @@ class SemaARM : public SemaBase {
                             unsigned ExpectedFieldNum, bool AllowName);
   bool BuiltinARMMemoryTaggingCall(unsigned BuiltinID, CallExpr *TheCall);
 
-  bool MveAliasValid(unsigned BuiltinID, StringRef AliasName);
-  bool CdeAliasValid(unsigned BuiltinID, StringRef AliasName);
-  bool SveAliasValid(unsigned BuiltinID, StringRef AliasName);
-  bool SmeAliasValid(unsigned BuiltinID, StringRef AliasName);
+  bool MveAliasValid(unsigned BuiltinID, llvm::StringRef AliasName);
+  bool CdeAliasValid(unsigned BuiltinID, llvm::StringRef AliasName);
+  bool SveAliasValid(unsigned BuiltinID, llvm::StringRef AliasName);
+  bool SmeAliasValid(unsigned BuiltinID, llvm::StringRef AliasName);
   void handleBuiltinAliasAttr(Decl *D, const ParsedAttr &AL);
   void handleNewAttr(Decl *D, const ParsedAttr &AL);
   void handleCmseNSEntryAttr(Decl *D, const ParsedAttr &AL);
diff --git a/clang/include/clang/Sema/SemaAVR.h b/clang/include/clang/Sema/SemaAVR.h
index 708da3a6026acee..d89ce054b864d3e 100644
--- a/clang/include/clang/Sema/SemaAVR.h
+++ b/clang/include/clang/Sema/SemaAVR.h
@@ -13,10 +13,10 @@
 #ifndef LLVM_CLANG_SEMA_SEMAAVR_H
 #define LLVM_CLANG_SEMA_SEMAAVR_H
 
+#include "clang/AST/ASTFwd.h"
 #include "clang/Sema/SemaBase.h"
 
 namespace clang {
-class Decl;
 class ParsedAttr;
 
 class SemaAVR : public SemaBase {
diff --git a/clang/include/clang/Sema/SemaBPF.h b/clang/include/clang/Sema/SemaBPF.h
index 0182ccfe508a7be..d7577aea7012274 100644
--- a/clang/include/clang/Sema/SemaBPF.h
+++ b/clang/include/clang/Sema/SemaBPF.h
@@ -13,9 +13,7 @@
 #ifndef LLVM_CLANG_SEMA_SEMABPF_H
 #define LLVM_CLANG_SEMA_SEMABPF_H
 
-#include "clang/AST/Decl.h"
-#include "clang/AST/DeclBase.h"
-#include "clang/AST/Expr.h"
+#include "clang/AST/ASTFwd.h"
 #include "clang/Sema/SemaBase.h"
 
 namespace clang {
diff --git a/clang/include/clang/Sema/SemaCUDA.h b/clang/include/clang/Sema/SemaCUDA.h
index 63dc3f4da240b36..71f05e88fb539cf 100644
--- a/clang/include/clang/Sema/SemaCUDA.h
+++ b/clang/include/clang/Sema/SemaCUDA.h
@@ -13,24 +13,34 @@
 #ifndef LLVM_CLANG_SEMA_SEMACUDA_H
 #define LLVM_CLANG_SEMA_SEMACUDA_H
 
-#include "clang/AST/Decl.h"
-#include "clang/AST/DeclCXX.h"
+#include "clang/AST/ASTFwd.h"
+#include "clang/AST/DeclAccessPair.h"
 #include "clang/AST/Redeclarable.h"
 #include "clang/Basic/Cuda.h"
+#include "clang/Basic/LLVM.h"
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Sema/Lookup.h"
 #include "clang/Sema/Ownership.h"
-#include "clang/Sema/ParsedAttr.h"
-#include "clang/Sema/Scope.h"
-#include "clang/Sema/ScopeInfo.h"
 #include "clang/Sema/SemaBase.h"
 #include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/DenseMapInfo.h"
+#include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/Hashing.h"
 #include "llvm/ADT/SmallVector.h"
 #include <string>
+#include <utility>
 
 namespace clang {
+namespace sema {
+class Capture;
+} // namespace sema
 
+class ASTReader;
+class ASTWriter;
 enum class CUDAFunctionTarget;
+enum class CXXSpecialMemberKind;
+class ParsedAttributesView;
+class Scope;
 
 class SemaCUDA : public SemaBase {
 public:
diff --git a/clang/include/clang/Sema/SemaCodeCompletion.h b/clang/include/clang/Sema/SemaCodeCompletion.h
index 6bf0f2473cbf437..50409439389b068 100644
--- a/clang/include/clang/Sema/SemaCodeCompletion.h
+++ b/clang/include/clang/Sema/SemaCodeCompletion.h
@@ -13,26 +13,26 @@
 #ifndef LLVM_CLANG_SEMA_SEMACODECOMPLETION_H
 #define LLVM_CLANG_SEMA_SEMACODECOMPLETION_H
 
-#include "clang/AST/Decl.h"
-#include "clang/AST/DeclCXX.h"
-#include "clang/AST/Expr.h"
+#include "clang/AST/ASTFwd.h"
 #include "clang/AST/Type.h"
 #include "clang/Basic/AttributeCommonInfo.h"
 #include "clang/Basic/IdentifierTable.h"
 #include "clang/Basic/LLVM.h"
 #include "clang/Basic/SourceLocation.h"
-#include "clang/Lex/MacroInfo.h"
 #include "clang/Lex/ModuleLoader.h"
 #include "clang/Sema/CodeCompleteConsumer.h"
 #include "clang/Sema/DeclSpec.h"
 #include "clang/Sema/Designator.h"
 #include "clang/Sema/Ownership.h"
-#include "clang/Sema/Scope.h"
 #include "clang/Sema/SemaBase.h"
 #include "llvm/ADT/StringRef.h"
 #include <optional>
 
 namespace clang {
+class DeclGroupRef;
+class MacroInfo;
+class Scope;
+class TemplateName;
 
 class SemaCodeCompletion : public SemaBase {
 public:
diff --git a/clang/include/clang/Sema/SemaHLSL.h b/clang/include/clang/Sema/SemaHLSL.h
index d60cb2a57d49187..3aae3383c215b55 100644
--- a/clang/include/clang/Sema/SemaHLSL.h
+++ b/clang/include/clang/Sema/SemaHLSL.h
@@ -13,19 +13,18 @@
 #ifndef LLVM_CLANG_SEMA_SEMAHLSL_H
 #define LLVM_CLANG_SEMA_SEMAHLSL_H
 
+#include "clang/AST/ASTFwd.h"
 #include "clang/AST/Attr.h"
-#include "clang/AST/Decl.h"
-#include "clang/AST/DeclBase.h"
-#include "clang/AST/Expr.h"
-#include "clang/Basic/AttributeCommonInfo.h"
-#include "clang/Basic/IdentifierTable.h"
 #include "clang/Basic/SourceLocation.h"
-#include "clang/Sema/Scope.h"
 #include "clang/Sema/SemaBase.h"
+#include "llvm/TargetParser/Triple.h"
 #include <initializer_list>
 
 namespace clang {
+class AttributeCommonInfo;
+class IdentifierInfo;
 class ParsedAttr;
+class Scope;
 
 class SemaHLSL : public SemaBase {
 public:
diff --git a/clang/include/clang/Sema/SemaHexagon.h b/clang/include/clang/Sema/SemaHexagon.h
index 2d4a04f824bc2ef..82e811dfc42e484 100644
--- a/clang/include/clang/Sema/SemaHexagon.h
+++ b/clang/include/clang/Sema/SemaHexagon.h
@@ -13,7 +13,7 @@
 #ifndef LLVM_CLANG_SEMA_SEMAHEXAGON_H
 #define LLVM_CLANG_SEMA_SEMAHEXAGON_H
 
-#include "clang/AST/Expr.h"
+#include "clang/AST/ASTFwd.h"
 #include "clang/Sema/SemaBase.h"
 
 namespace clang {
diff --git a/clang/include/clang/Sema/SemaLoongArch.h b/clang/include/clang/Sema/SemaLoongArch.h
index aef0df9e8710ff0..98a7e99e31572a2 100644
--- a/clang/include/clang/Sema/SemaLoongArch.h
+++ b/clang/include/clang/Sema/SemaLoongArch.h
@@ -13,11 +13,12 @@
 #ifndef LLVM_CLANG_SEMA_SEMALOONGARCH_H
 #define LLVM_CLANG_SEMA_SEMALOONGARCH_H
 
-#include "clang/AST/Expr.h"
-#include "clang/Basic/TargetInfo.h"
+#include "clang/AST/ASTFwd.h"
 #include "clang/Sema/SemaBase.h"
 
 namespace clang {
+class TargetInfo;
+
 class SemaLoongArch : public SemaBase {
 public:
   SemaLoongArch(Sema &S);
diff --git a/clang/include/clang/Sema/SemaM68k.h b/clang/include/clang/Sema/SemaM68k.h
index 5a9767d5ea52163..dece033c22030a9 100644
--- a/clang/include/clang/Sema/SemaM68k.h
+++ b/clang/include/clang/Sema/SemaM68k.h
@@ -13,10 +13,10 @@
 #ifndef LLVM_CLANG_SEMA_SEMAM68K_H
 #define LLVM_CLANG_SEMA_SEMAM68K_H
 
+#include "clang/AST/ASTFwd.h"
 #include "clang/Sema/SemaBase.h"
 
 namespace clang {
-class Decl;
 class ParsedAttr;
 
 class SemaM68k : public SemaBase {
diff --git a/clang/include/clang/Sema/SemaMIPS.h b/clang/include/clang/Sema/SemaMIPS.h
index 6366dce57626af8..41feeb69b80594b 100644
--- a/clang/include/clang/Sema/SemaMIPS.h
+++ b/clang/include/clang/Sema/SemaMIPS.h
@@ -13,13 +13,12 @@
 #ifndef LLVM_CLANG_SEMA_SEMAMIPS_H
 #define LLVM_CLANG_SEMA_SEMAMIPS_H
 
-#include "clang/AST/DeclBase.h"
-#include "clang/AST/Expr.h"
-#include "clang/Basic/TargetInfo.h"
+#include "clang/AST/ASTFwd.h"
 #include "clang/Sema/SemaBase.h"
 
 namespace clang {
 class ParsedAttr;
+class TargetInfo;
 
 class SemaMIPS : public SemaBase {
 public:
diff --git a/clang/include/clang/Sema/SemaMSP430.h b/clang/include/clang/Sema/SemaMSP430.h
index e1034aefe881640..30eb8eaf6aa1a39 100644
--- a/clang/include/clang/Sema/SemaMSP430.h
+++ b/clang/include/clang/Sema/SemaMSP430.h
@@ -13,10 +13,10 @@
 #ifndef LLVM_CLANG_SEMA_SEMAMSP430_H
 #define LLVM_CLANG_SEMA_SEMAMSP430_H
 
+#include "clang/AST/ASTFwd.h"
 #include "clang/Sema/SemaBase.h"
 
 namespace clang {
-class Decl;
 class ParsedAttr;
 
 class SemaMSP430 : public SemaBase {
diff --git a/clang/include/clang/Sema/SemaNVPTX.h b/clang/include/clang/Sema/SemaNVPTX.h
index a663c694179d98a..de47997d2f8bf16 100644
--- a/clang/include/clang/Sema/SemaNVPTX.h
+++ b/clang/include/clang/Sema/SemaNVPTX.h
@@ -13,11 +13,12 @@
 #ifndef LLVM_CLANG_SEMA_SEMANVPTX_H
 #define LLVM_CLANG_SEMA_SEMANVPTX_H
 
-#include "clang/AST/Expr.h"
-#include "clang/Basic/TargetInfo.h"
+#include "clang/AST/ASTFwd.h"
 #include "clang/Sema/SemaBase.h"
 
 namespace clang {
+class TargetInfo;
+
 class SemaNVPTX : public SemaBase {
 public:
   SemaNVPTX(Sema &S);
diff --git a/clang/include/clang/Sema/SemaObjC.h b/clang/include/clang/Sema/SemaObjC.h
index 07c3c1a06be1603..213c37b5091fe0f 100644
--- a/clang/include/clang/Sema/SemaObjC.h
+++ b/clang/include/clang/Sema/SemaObjC.h
@@ -13,11 +13,8 @@
 #ifndef LLVM_CLANG_SEMA_SEMAOBJC_H
 #define LLVM_CLANG_SEMA_SEMAOBJC_H
 
-#include "clang/AST/Decl.h"
-#include "clang/AST/DeclBase.h"
+#include "clang/AST/ASTFwd.h"
 #include "clang/AST/DeclObjC.h"
-#include "clang/AST/Expr.h"
-#include "clang/AST/ExprObjC.h"
 #include "clang/AST/NSAPI.h"
 #include "clang/AST/OperationKinds.h"
 #include "clang/AST/Type.h"
@@ -27,24 +24,29 @@
 #include "clang/Basic/Specifiers.h"
 #include "clang/Basic/TokenKinds.h"
 #include "clang/Sema/DeclSpec.h"
-#include "clang/Sema/Lookup.h"
 #include "clang/Sema/ObjCMethodList.h"
 #include "clang/Sema/Ownership.h"
 #include "clang/Sema/Redeclaration.h"
-#include "clang/Sema/Scope.h"
+#include "clang/Sema/Sema.h"
 #include "clang/Sema/SemaBase.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include <memory>
 #include <optional>
-#include <type_traits>
 #include <utility>
 
 namespace clang {
 
+class AttributeCommonInfo;
+class AvailabilitySpec;
 enum class CheckedConversionKind;
+class DeclGroupRef;
+class LookupResult;
+struct ObjCDictionaryElement;
 class ParsedAttr;
+class ParsedAttributesView;
+class Scope;
 struct SkipBodyInfo;
 
 class SemaObjC : public SemaBase {
diff --git a/clang/include/clang/Sema/SemaOpenACC.h b/clang/include/clang/Sema/SemaOpenACC.h
index a5f2a8bf7465777..0ca76842e5f9028 100644
--- a/clang/include/clang/Sema/SemaOpenACC.h
+++ b/clang/include/clang/Sema/SemaOpenACC.h
@@ -16,13 +16,19 @@
 
 #include "clang/AST/DeclGroup.h"
 #include "clang/AST/StmtOpenACC.h"
+#include "clang/Basic/LLVM.h"
 #include "clang/Basic/OpenACCKinds.h"
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Sema/Ownership.h"
 #include "clang/Sema/SemaBase.h"
+#include "llvm/ADT/SmallVector.h"
+#include <cassert>
+#include <optional>
+#include <utility>
 #include <variant>
 
 namespace clang {
+class IdentifierInfo;
 class OpenACCClause;
 
 class SemaOpenACC : public SemaBase {
diff --git a/clang/include/clang/Sema/SemaOpenCL.h b/clang/include/clang/Sema/SemaOpenCL.h
index 7d6b4b0dec09c25..04b2b617fb12fe6 100644
--- a/clang/include/clang/Sema/SemaOpenCL.h
+++ b/clang/include/clang/Sema/SemaOpenCL.h
@@ -13,10 +13,10 @@
 #ifndef LLVM_CLANG_SEMA_SEMAOPENCL_H
 #define LLVM_CLANG_SEMA_SEMAOPENCL_H
 
+#include "clang/AST/ASTFwd.h"
 #include "clang/Sema/SemaBase.h"
 
 namespace clang {
-class Decl;
 class ParsedAttr;
 
 class SemaOpenCL : public SemaBase {
diff --git a/clang/include/clang/Sema/SemaOpenMP.h b/clang/include/clang/Sema/SemaOpenMP.h
index e55731212c4a41e..53191e7bb4272bb 100644
--- a/clang/include/clang/Sema/SemaOpenMP.h
+++ b/clang/include/clang/Sema/SemaOpenMP.h
@@ -14,17 +14,12 @@
 #ifndef LLVM_CLANG_SEMA_SEMAOPENMP_H
 #define LLVM_CLANG_SEMA_SEMAOPENMP_H
 
+#include "clang/AST/ASTFwd.h"
 #include "clang/AST/Attr.h"
-#include "clang/AST/Decl.h"
-#include "clang/AST/DeclBase.h"
-#include "clang/AST/DeclOpenMP.h"
 #include "clang/AST/DeclarationName.h"
-#include "clang/AST/Expr.h"
 #include "clang/AST/ExprOpenMP.h"
 #include "clang/AST/OpenMPClause.h"
-#include "clang/AST/Stmt.h"
 #include "clang/AST/StmtOpenMP.h"
-#include "clang/AST/Type.h"
 #include "clang/Basic/IdentifierTable.h"
 #include "clang/Basic/LLVM.h"
 #include "clang/Basic/OpenMPKinds.h"
@@ -32,17 +27,23 @@
 #include "clang/Basic/Specifiers.h"
 #include "clang/Sema/DeclSpec.h"
 #include "clang/Sema/Ownership.h"
-#include "clang/Sema/Scope.h"
-#include "clang/Sema/ScopeInfo.h"
 #include "clang/Sema/SemaBase.h"
 #include "llvm/ADT/DenseMap.h"
-#include "llvm/ADT/PointerUnion.h"
+#include "llvm/Frontend/OpenMP/OMP.h.inc"
+#include "llvm/Frontend/OpenMP/OMPConstants.h"
 #include <optional>
 #include <string>
 #include <utility>
 
 namespace clang {
+namespace sema {
+class FunctionScopeInfo;
+} // namespace sema
+
+class DeclContext;
+class DeclGroupRef;
 class ParsedAttr;
+class Scope;
 
 class SemaOpenMP : public SemaBase {
 public:
diff --git a/clang/include/clang/Sema/SemaPPC.h b/clang/include/clang/Sema/SemaPPC.h
index 3e8929d5b6dedab..f8edecc4fcb7bff 100644
--- a/clang/include/clang/Sema/SemaPPC.h
+++ b/clang/include/clang/Sema/SemaPPC.h
@@ -13,13 +13,14 @@
 #ifndef LLVM_CLANG_SEMA_SEMAPPC_H
 #define LLVM_CLANG_SEMA_SEMAPPC_H
 
-#include "clang/AST/Expr.h"
+#include "clang/AST/ASTFwd.h"
 #include "clang/AST/Type.h"
 #include "clang/Basic/SourceLocation.h"
-#include "clang/Basic/TargetInfo.h"
 #include "clang/Sema/SemaBase.h"
 
 namespace clang {
+class TargetInfo;
+
 class SemaPPC : public SemaBase {
 public:
   SemaPPC(Sema &S);
diff --git a/clang/include/clang/Sema/SemaPseudoObject.h b/clang/include/clang/Sema/SemaPseudoObject.h
index 22d8be2b3726ee1..4753987070b750a 100644
--- a/clang/include/clang/Sema/SemaPseudoObject.h
+++ b/clang/include/clang/Sema/SemaPseudoObject.h
@@ -14,14 +14,14 @@
 #ifndef LLVM_CLANG_SEMA_SEMAPSEUDOOBJECT_H
 #define LLVM_CLANG_SEMA_SEMAPSEUDOOBJECT_H
 
-#include "clang/AST/Expr.h"
+#include "clang/AST/ASTFwd.h"
 #include "clang/AST/OperationKinds.h"
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Sema/Ownership.h"
-#include "clang/Sema/Scope.h"
 #include "clang/Sema/SemaBase.h"
 
 namespace clang {
+class Scope;
 
 class SemaPseudoObject : public SemaBase {
 public:
diff --git a/clang/include/clang/Sema/SemaRISCV.h b/clang/include/clang/Sema/SemaRISCV.h
index 48d15c411bddde2..d62fca8128b2a3a 100644
--- a/clang/include/clang/Sema/SemaRISCV.h
+++ b/clang/include/clang/Sema/SemaRISCV.h
@@ -13,18 +13,21 @@
 #ifndef LLVM_CLANG_SEMA_SEMARISCV_H
 #define LLVM_CLANG_SEMA_SEMARISCV_H
 
-#include "clang/AST/DeclBase.h"
-#include "clang/AST/Expr.h"
+#include "clang/AST/ASTFwd.h"
 #include "clang/AST/Type.h"
 #include "clang/Basic/SourceLocation.h"
-#include "clang/Basic/TargetInfo.h"
-#include "clang/Sema/RISCVIntrinsicManager.h"
 #include "clang/Sema/SemaBase.h"
 #include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/StringRef.h"
 #include <memory>
 
 namespace clang {
+namespace sema {
+class RISCVIntrinsicManager;
+} // namespace sema
+
 class ParsedAttr;
+class TargetInfo;
 
 class SemaRISCV : public SemaBase {
 public:
@@ -39,7 +42,7 @@ class SemaRISCV : public SemaBase {
   bool isValidRVVBitcast(QualType srcType, QualType destType);
 
   void handleInterruptAttr(Decl *D, const ParsedAttr &AL);
-  bool isAliasValid(unsigned BuiltinID, StringRef AliasName);
+  bool isAliasValid(unsigned BuiltinID, llvm::StringRef AliasName);
 
   /// Indicate RISC-V vector builtin functions enabled or not.
   bool DeclareRVVBuiltins = false;
diff --git a/clang/include/clang/Sema/SemaSYCL.h b/clang/include/clang/Sema/SemaSYCL.h
index 363178546a236c5..27c42b540183073 100644
--- a/clang/include/clang/Sema/SemaSYCL.h
+++ b/clang/include/clang/Sema/SemaSYCL.h
@@ -13,7 +13,7 @@
 #ifndef LLVM_CLANG_SEMA_SEMASYCL_H
 #define LLVM_CLANG_SEMA_SEMASYCL_H
 
-#include "clang/AST/Decl.h"
+#include "clang/AST/ASTFwd.h"
 #include "clang/AST/Type.h"
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Sema/Ownership.h"
diff --git a/clang/include/clang/Sema/SemaSwift.h b/clang/include/clang/Sema/SemaSwift.h
index a5561d756affd10..8d8f1467054ac5b 100644
--- a/clang/include/clang/Sema/SemaSwift.h
+++ b/clang/include/clang/Sema/SemaSwift.h
@@ -13,15 +13,15 @@
 #ifndef LLVM_CLANG_SEMA_SEMASWIFT_H
 #define LLVM_CLANG_SEMA_SEMASWIFT_H
 
-#include "clang/AST/Attr.h"
+#include "clang/AST/ASTFwd.h"
 #include "clang/Basic/LLVM.h"
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Sema/SemaBase.h"
-#include "llvm/ADT/StringRef.h"
 
 namespace clang {
 class AttributeCommonInfo;
 class Decl;
+enum class ParameterABI;
 class ParsedAttr;
 class SwiftNameAttr;
 
diff --git a/clang/include/clang/Sema/SemaSystemZ.h b/clang/include/clang/Sema/SemaSystemZ.h
index 8945471d53d638f..1a3e29f7a9312a6 100644
--- a/clang/include/clang/Sema/SemaSystemZ.h
+++ b/clang/include/clang/Sema/SemaSystemZ.h
@@ -13,7 +13,7 @@
 #ifndef LLVM_CLANG_SEMA_SEMASYSTEMZ_H
 #define LLVM_CLANG_SEMA_SEMASYSTEMZ_H
 
-#include "clang/AST/Expr.h"
+#include "clang/AST/ASTFwd.h"
 #include "clang/Sema/SemaBase.h"
 
 namespace clang {
diff --git a/clang/include/clang/Sema/SemaWasm.h b/clang/include/clang/Sema/SemaWasm.h
index c3c781535024aef..8841fdff2303538 100644
--- a/clang/include/clang/Sema/SemaWasm.h
+++ b/clang/include/clang/Sema/SemaWasm.h
@@ -13,14 +13,13 @@
 #ifndef LLVM_CLANG_SEMA_SEMAWASM_H
 #define LLVM_CLANG_SEMA_SEMAWASM_H
 
-#include "clang/AST/Attr.h"
-#include "clang/AST/DeclBase.h"
-#include "clang/AST/Expr.h"
-#include "clang/Basic/TargetInfo.h"
-#include "clang/Sema/ParsedAttr.h"
+#include "clang/AST/ASTFwd.h"
 #include "clang/Sema/SemaBase.h"
 
 namespace clang {
+class ParsedAttr;
+class TargetInfo;
+
 class SemaWasm : public SemaBase {
 public:
   SemaWasm(Sema &S);
diff --git a/clang/include/clang/Sema/SemaX86.h b/clang/include/clang/Sema/SemaX86.h
index e53aaf229c38dda..b5a23f1bede047e 100644
--- a/clang/include/clang/Sema/SemaX86.h
+++ b/clang/include/clang/Sema/SemaX86.h
@@ -13,14 +13,13 @@
 #ifndef LLVM_CLANG_SEMA_SEMAX86_H
 #define LLVM_CLANG_SEMA_SEMAX86_H
 
-#include "clang/AST/DeclBase.h"
-#include "clang/AST/Expr.h"
+#include "clang/AST/ASTFwd.h"
 #include "clang/Basic/LLVM.h"
-#include "clang/Basic/TargetInfo.h"
 #include "clang/Sema/SemaBase.h"
 
 namespace clang {
 class ParsedAttr;
+class TargetInfo;
 
 class SemaX86 : public SemaBase {
 public:
diff --git a/clang/lib/Parse/ParseExpr.cpp b/clang/lib/Parse/ParseExpr.cpp
index b8e15b1aa6ce9e7..1405aef700bec5e 100644
--- a/clang/lib/Parse/ParseExpr.cpp
+++ b/clang/lib/Parse/ParseExpr.cpp
@@ -21,7 +21,9 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/AST/ASTContext.h"
+#include "clang/AST/Availability.h"
 #include "clang/AST/ExprCXX.h"
+#include "clang/AST/LocInfoType.h"
 #include "clang/Basic/PrettyStackTrace.h"
 #include "clang/Lex/LiteralSupport.h"
 #include "clang/Parse/Parser.h"
diff --git a/clang/lib/Parse/ParseObjc.cpp b/clang/lib/Parse/ParseObjc.cpp
index 6a2088a73c55b91..dad39d8dc736de1 100644
--- a/clang/lib/Parse/ParseObjc.cpp
+++ b/clang/lib/Parse/ParseObjc.cpp
@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/AST/ASTContext.h"
+#include "clang/AST/ExprObjC.h"
 #include "clang/AST/ODRDiagsEmitter.h"
 #include "clang/AST/PrettyDeclStackTrace.h"
 #include "clang/Basic/CharInfo.h"
diff --git a/clang/lib/Sema/Sema.cpp b/clang/lib/Sema/Sema.cpp
index 633b8220ffbf11d..29acd06af603be3 100644
--- a/clang/lib/Sema/Sema.cpp
+++ b/clang/lib/Sema/Sema.cpp
@@ -22,6 +22,7 @@
 #include "clang/AST/ExprCXX.h"
 #include "clang/AST/PrettyDeclStackTrace.h"
 #include "clang/AST/StmtCXX.h"
+#include "clang/AST/TypeOrdering.h"
 #include "clang/Basic/DarwinSDKInfo.h"
 #include "clang/Basic/DiagnosticOptions.h"
 #include "clang/Basic/PartialDiagnostic.h"
diff --git a/clang/lib/Sema/SemaARM.cpp b/clang/lib/Sema/SemaARM.cpp
index d8dd4fe16e3af00..e18872f0dc551e4 100644
--- a/clang/lib/Sema/SemaARM.cpp
+++ b/clang/lib/Sema/SemaARM.cpp
@@ -13,6 +13,7 @@
 #include "clang/Sema/SemaARM.h"
 #include "clang/Basic/DiagnosticSema.h"
 #include "clang/Basic/TargetBuiltins.h"
+#include "clang/Basic/TargetInfo.h"
 #include "clang/Sema/Initialization.h"
 #include "clang/Sema/ParsedAttr.h"
 #include "clang/Sema/Sema.h"
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index 1d85341bc9a8b65..9c15b5fa571a394 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -25,6 +25,7 @@
 #include "clang/AST/Expr.h"
 #include "clang/AST/ExprCXX.h"
 #include "clang/AST/NonTrivialTypeVisitor.h"
+#include "clang/AST/MangleNumberingContext.h"
 #include "clang/AST/Randstruct.h"
 #include "clang/AST/StmtCXX.h"
 #include "clang/AST/Type.h"
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index 0f58eb2840211de..f495658fe586416 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -25,6 +25,7 @@
 #include "clang/AST/ExprCXX.h"
 #include "clang/AST/ExprObjC.h"
 #include "clang/AST/ExprOpenMP.h"
+#include "clang/AST/MangleNumberingContext.h"
 #include "clang/AST/OperationKinds.h"
 #include "clang/AST/ParentMapContext.h"
 #include "clang/AST/RecursiveASTVisitor.h"
diff --git a/clang/lib/Sema/SemaExprObjC.cpp b/clang/lib/Sema/SemaExprObjC.cpp
index 2751c7cec2842e3..2f914ddc22a38b8 100644
--- a/clang/lib/Sema/SemaExprObjC.cpp
+++ b/clang/lib/Sema/SemaExprObjC.cpp
@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/AST/ASTContext.h"
+#include "clang/AST/Availability.h"
 #include "clang/AST/DeclObjC.h"
 #include "clang/AST/ExprObjC.h"
 #include "clang/AST/StmtVisitor.h"
diff --git a/clang/lib/Sema/SemaLambda.cpp b/clang/lib/Sema/SemaLambda.cpp
index 05d406a8630fbe4..a8461a7f6cf7123 100644
--- a/clang/lib/Sema/SemaLambda.cpp
+++ b/clang/lib/Sema/SemaLambda.cpp
@@ -14,6 +14,7 @@
 #include "clang/AST/ASTLambda.h"
 #include "clang/AST/CXXInheritance.h"
 #include "clang/AST/ExprCXX.h"
+#include "clang/AST/MangleNumberingContext.h"
 #include "clang/Basic/TargetInfo.h"
 #include "clang/Sema/DeclSpec.h"
 #include "clang/Sema/Initialization.h"
diff --git a/clang/lib/Sema/SemaMIPS.cpp b/clang/lib/Sema/SemaMIPS.cpp
index 269d927903c5d60..50d210e5b381405 100644
--- a/clang/lib/Sema/SemaMIPS.cpp
+++ b/clang/lib/Sema/SemaMIPS.cpp
@@ -13,6 +13,7 @@
 #include "clang/Sema/SemaMIPS.h"
 #include "clang/Basic/DiagnosticSema.h"
 #include "clang/Basic/TargetBuiltins.h"
+#include "clang/Basic/TargetInfo.h"
 #include "clang/Sema/Attr.h"
 #include "clang/Sema/ParsedAttr.h"
 #include "clang/Sema/Sema.h"
diff --git a/clang/lib/Sema/SemaObjC.cpp b/clang/lib/Sema/SemaObjC.cpp
index 75233689769c5bc..0359d18dd945251 100644
--- a/clang/lib/Sema/SemaObjC.cpp
+++ b/clang/lib/Sema/SemaObjC.cpp
@@ -17,6 +17,7 @@
 #include "clang/Basic/DiagnosticSema.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Sema/Attr.h"
+#include "clang/Sema/Lookup.h"
 #include "clang/Sema/ParsedAttr.h"
 #include "clang/Sema/ScopeInfo.h"
 #include "clang/Sema/Sema.h"
diff --git a/clang/lib/Sema/SemaPPC.cpp b/clang/lib/Sema/SemaPPC.cpp
index e0a978e6cf5b97c..b6802d6890ed177 100644
--- a/clang/lib/Sema/SemaPPC.cpp
+++ b/clang/lib/Sema/SemaPPC.cpp
@@ -19,6 +19,7 @@
 #include "clang/Basic/DiagnosticSema.h"
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Basic/TargetBuiltins.h"
+#include "clang/Basic/TargetInfo.h"
 #include "clang/Sema/Sema.h"
 #include "llvm/ADT/APSInt.h"
 
diff --git a/clang/lib/Sema/SemaSYCL.cpp b/clang/lib/Sema/SemaSYCL.cpp
index 2b55c598d55c038..f2d13d456c25fc6 100644
--- a/clang/lib/Sema/SemaSYCL.cpp
+++ b/clang/lib/Sema/SemaSYCL.cpp
@@ -10,6 +10,7 @@
 
 #include "clang/Sema/SemaSYCL.h"
 #include "clang/AST/Mangle.h"
+#include "clang/AST/TypeOrdering.h"
 #include "clang/Sema/Attr.h"
 #include "clang/Sema/ParsedAttr.h"
 #include "clang/Sema/Sema.h"
diff --git a/clang/lib/Sema/SemaTemplateDeduction.cpp b/clang/lib/Sema/SemaTemplateDeduction.cpp
index fc883e423f47105..35bc8cd713f06c6 100644
--- a/clang/lib/Sema/SemaTemplateDeduction.cpp
+++ b/clang/lib/Sema/SemaTemplateDeduction.cpp
@@ -28,6 +28,7 @@
 #include "clang/AST/TemplateName.h"
 #include "clang/AST/Type.h"
 #include "clang/AST/TypeLoc.h"
+#include "clang/AST/TypeOrdering.h"
 #include "clang/AST/UnresolvedSet.h"
 #include "clang/Basic/AddressSpaces.h"
 #include "clang/Basic/ExceptionSpecificationType.h"
diff --git a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
index 14ca29f1bc3f1aa..a58854acb21fa5b 100644
--- a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
@@ -19,6 +19,7 @@
 #include "clang/AST/Expr.h"
 #include "clang/AST/ExprCXX.h"
 #include "clang/AST/PrettyDeclStackTrace.h"
+#include "clang/AST/TypeOrdering.h"
 #include "clang/AST/TypeLoc.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Basic/TargetInfo.h"
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index 08020f9f889270a..4b1bb05464f7dca 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -20,6 +20,8 @@
 #include "clang/AST/DeclObjC.h"
 #include "clang/AST/DeclTemplate.h"
 #include "clang/AST/Expr.h"
+#include "clang/AST/ExprObjC.h"
+#include "clang/AST/LocInfoType.h"
 #include "clang/AST/Type.h"
 #include "clang/AST/TypeLoc.h"
 #include "clang/AST/TypeLocVisitor.h"
diff --git a/clang/lib/Sema/SemaX86.cpp b/clang/lib/Sema/SemaX86.cpp
index a0756f167deae61..45c4a1c80b40835 100644
--- a/clang/lib/Sema/SemaX86.cpp
+++ b/clang/lib/Sema/SemaX86.cpp
@@ -13,6 +13,7 @@
 #include "clang/Sema/SemaX86.h"
 #include "clang/Basic/DiagnosticSema.h"
 #include "clang/Basic/TargetBuiltins.h"
+#include "clang/Basic/TargetInfo.h"
 #include "clang/Sema/Attr.h"
 #include "clang/Sema/ParsedAttr.h"
 #include "clang/Sema/Sema.h"

From 362142c4bb5cc657151f592e507f552d5b9f7dde Mon Sep 17 00:00:00 2001
From: Tomas Matheson <Tomas.Matheson@arm.com>
Date: Sat, 17 Aug 2024 13:36:40 +0100
Subject: [PATCH 268/441] [AArch64] Add a check for invalid default features
 (#104435)

This adds a check that all ExtensionWithMArch which are marked as
implied features for an architecture are also present in the list of
default features. It doesn't make sense to have something mandatory but
not on by default.

There were a number of existing cases that violated this rule, and some
changes to which features are mandatory (indicated by the Implies
field).

This resulted in a bug where if a feature was marked as `Implies` but
was not added to `DefaultExt`, then for `-march=base_arch+nofeat` the
Driver would consider `feat` to have never been added and therefore
would do nothing to disable it (no `-target-feature -feat` would be
added, but the backend would enable the feature by default because of
`Implies`). See
clang/test/Driver/aarch64-negative-modifiers-for-default-features.c.

Note that the processor definitions do not respect the architecture
DefaultExts. These apply only when specifying `-march=<some architecture
version>`. So when a feature is moved from `Implies` to `DefaultExts` on
the Architecture definition, the feature needs to be added to all
processor definitions (that are based on that architecture) in order to
preserve the existing behaviour. I have checked the TRMs for many cases
(see specific commit messages) but in other cases I have just kept the
current behaviour and not tried to fix it.
---
 clang/test/CodeGen/aarch64-targetattr.c       | 12 +--
 ...-negative-modifiers-for-default-features.c | 12 +++
 clang/test/Driver/arm-sb.c                    |  2 +-
 .../aarch64-apple-a12.c                       |  1 -
 .../aarch64-apple-a13.c                       |  1 -
 .../aarch64-apple-a14.c                       |  1 -
 .../aarch64-apple-a15.c                       |  1 -
 .../aarch64-apple-a16.c                       |  1 -
 .../aarch64-apple-a17.c                       |  1 -
 .../aarch64-apple-m4.c                        |  2 -
 .../aarch64-cortex-r82.c                      |  1 -
 .../aarch64-cortex-r82ae.c                    |  1 -
 llvm/lib/Target/AArch64/AArch64Features.td    | 19 ++--
 llvm/lib/Target/AArch64/AArch64Processors.td  | 31 +++++-
 llvm/test/MC/AArch64/arm64-system-encoding.s  |  2 +-
 llvm/test/MC/AArch64/armv8.5a-ssbs-error.s    |  2 +-
 llvm/test/MC/AArch64/armv8.5a-ssbs.s          |  2 +-
 .../MC/Disassembler/AArch64/armv8.5a-ssbs.txt |  2 +-
 .../AArch64/basic-a64-instructions.txt        |  2 +-
 .../TargetParser/TargetParserTest.cpp         | 94 +++++++++++--------
 llvm/utils/TableGen/ARMTargetDefEmitter.cpp   | 32 ++++++-
 21 files changed, 144 insertions(+), 78 deletions(-)
 create mode 100644 clang/test/Driver/aarch64-negative-modifiers-for-default-features.c

diff --git a/clang/test/CodeGen/aarch64-targetattr.c b/clang/test/CodeGen/aarch64-targetattr.c
index 4f891f938b61860..d6227be2ebef83d 100644
--- a/clang/test/CodeGen/aarch64-targetattr.c
+++ b/clang/test/CodeGen/aarch64-targetattr.c
@@ -195,19 +195,19 @@ void minusarch() {}
 // CHECK: attributes #[[ATTR0]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+crc,+fp-armv8,+lse,+neon,+ras,+rdm,+v8.1a,+v8.2a,+v8a" }
 // CHECK: attributes #[[ATTR1]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+crc,+fp-armv8,+fullfp16,+lse,+neon,+ras,+rdm,+sve,+v8.1a,+v8.2a,+v8a" }
 // CHECK: attributes #[[ATTR2]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+crc,+fp-armv8,+fullfp16,+lse,+neon,+ras,+rdm,+sve,+sve2,+v8.1a,+v8.2a,+v8a" }
-// CHECK: attributes #[[ATTR3]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+complxnum,+crc,+dotprod,+fp-armv8,+fp16fml,+fullfp16,+i8mm,+jsconv,+lse,+neon,+pauth,+ras,+rcpc,+rdm,+sve,+sve2,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8.5a,+v8.6a,+v8a" }
-// CHECK: attributes #[[ATTR4]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="cortex-a710" "target-features"="+bf16,+complxnum,+crc,+dotprod,+ete,+flagm,+fp-armv8,+fp16fml,+fullfp16,+i8mm,+jsconv,+lse,+mte,+neon,+pauth,+perfmon,+ras,+rcpc,+rdm,+sb,+sve,+sve2,+sve2-bitperm,+trbe,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8.5a,+v8a,+v9a" }
+// CHECK: attributes #[[ATTR3]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+bf16,+bti,+ccidx,+complxnum,+crc,+dit,+dotprod,+flagm,+fp-armv8,+fp16fml,+fullfp16,+i8mm,+jsconv,+lse,+neon,+pauth,+predres,+ras,+rcpc,+rdm,+sb,+ssbs,+sve,+sve2,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8.5a,+v8.6a,+v8a" }
+// CHECK: attributes #[[ATTR4]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="cortex-a710" "target-features"="+bf16,+bti,+ccidx,+complxnum,+crc,+dit,+dotprod,+ete,+flagm,+fp-armv8,+fp16fml,+fullfp16,+i8mm,+jsconv,+lse,+mte,+neon,+pauth,+perfmon,+predres,+ras,+rcpc,+rdm,+sb,+ssbs,+sve,+sve2,+sve2-bitperm,+trbe,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8.5a,+v8a,+v9a" }
 // CHECK: attributes #[[ATTR5]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "tune-cpu"="cortex-a710" }
 // CHECK: attributes #[[ATTR6]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="generic" "target-features"="+ete,+fp-armv8,+neon,+trbe,+v8a" }
 // CHECK: attributes #[[ATTR7]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "tune-cpu"="generic" }
 // CHECK: attributes #[[ATTR8]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="neoverse-n1" "target-features"="+aes,+crc,+dotprod,+fp-armv8,+fullfp16,+lse,+neon,+perfmon,+ras,+rcpc,+rdm,+sha2,+spe,+ssbs,+v8.1a,+v8.2a,+v8a" "tune-cpu"="cortex-a710" }
 // CHECK: attributes #[[ATTR9]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+sve" "tune-cpu"="cortex-a710" }
-// CHECK: attributes #[[ATTR10]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="neoverse-v1" "target-features"="+aes,+bf16,+ccdp,+complxnum,+crc,+dotprod,+fp-armv8,+fp16fml,+fullfp16,+i8mm,+jsconv,+lse,+neon,+pauth,+perfmon,+rand,+ras,+rcpc,+rdm,+sha2,+sha3,+sm4,+spe,+ssbs,+sve,+sve2,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8a" }
-// CHECK: attributes #[[ATTR11]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="neoverse-v1" "target-features"="+aes,+bf16,+ccdp,+complxnum,+crc,+dotprod,+fp-armv8,+fp16fml,+fullfp16,+i8mm,+jsconv,+lse,+neon,+pauth,+perfmon,+rand,+ras,+rcpc,+rdm,+sha2,+sha3,+sm4,+spe,+ssbs,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8a,-sve" }
+// CHECK: attributes #[[ATTR10]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="neoverse-v1" "target-features"="+aes,+bf16,+ccdp,+ccidx,+complxnum,+crc,+dit,+dotprod,+flagm,+fp-armv8,+fp16fml,+fullfp16,+i8mm,+jsconv,+lse,+neon,+pauth,+perfmon,+rand,+ras,+rcpc,+rdm,+sha2,+sha3,+sm4,+spe,+ssbs,+sve,+sve2,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8a" }
+// CHECK: attributes #[[ATTR11]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="neoverse-v1" "target-features"="+aes,+bf16,+ccdp,+ccidx,+complxnum,+crc,+dit,+dotprod,+flagm,+fp-armv8,+fp16fml,+fullfp16,+i8mm,+jsconv,+lse,+neon,+pauth,+perfmon,+rand,+ras,+rcpc,+rdm,+sha2,+sha3,+sm4,+spe,+ssbs,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8a,-sve" }
 // CHECK: attributes #[[ATTR12]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+sve" }
 // CHECK: attributes #[[ATTR13]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16" }
-// CHECK: attributes #[[ATTR14]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="neoverse-n1" "target-features"="+aes,+bf16,+complxnum,+crc,+dotprod,+fp-armv8,+fullfp16,+i8mm,+jsconv,+lse,+neon,+pauth,+perfmon,+ras,+rcpc,+rdm,+sha2,+spe,+ssbs,+sve,+sve2,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8.5a,+v8.6a,+v8a" "tune-cpu"="cortex-a710" }
-// CHECK: attributes #[[ATTR15]] = { noinline nounwind optnone "branch-target-enforcement" "guarded-control-stack" "no-trapping-math"="true" "sign-return-address"="non-leaf" "sign-return-address-key"="a_key" "stack-protector-buffer-size"="8" "target-cpu"="neoverse-n1" "target-features"="+aes,+bf16,+complxnum,+crc,+dotprod,+fp-armv8,+fullfp16,+i8mm,+jsconv,+lse,+neon,+pauth,+perfmon,+ras,+rcpc,+rdm,+sha2,+spe,+ssbs,+sve,+sve2,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8.5a,+v8.6a,+v8a" "tune-cpu"="cortex-a710" }
+// CHECK: attributes #[[ATTR14]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="neoverse-n1" "target-features"="+aes,+bf16,+bti,+ccidx,+complxnum,+crc,+dit,+dotprod,+flagm,+fp-armv8,+fullfp16,+i8mm,+jsconv,+lse,+neon,+pauth,+perfmon,+predres,+ras,+rcpc,+rdm,+sb,+sha2,+spe,+ssbs,+sve,+sve2,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8.5a,+v8.6a,+v8a" "tune-cpu"="cortex-a710" }
+// CHECK: attributes #[[ATTR15]] = { noinline nounwind optnone "branch-target-enforcement" "guarded-control-stack" "no-trapping-math"="true" "sign-return-address"="non-leaf" "sign-return-address-key"="a_key" "stack-protector-buffer-size"="8" "target-cpu"="neoverse-n1" "target-features"="+aes,+bf16,+bti,+ccidx,+complxnum,+crc,+dit,+dotprod,+flagm,+fp-armv8,+fullfp16,+i8mm,+jsconv,+lse,+neon,+pauth,+perfmon,+predres,+ras,+rcpc,+rdm,+sb,+sha2,+spe,+ssbs,+sve,+sve2,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8.5a,+v8.6a,+v8a" "tune-cpu"="cortex-a710" }
 // CHECK: attributes #[[ATTR16]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
 // CHECK: attributes #[[ATTR17]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="-v9.3a" }
 //.
diff --git a/clang/test/Driver/aarch64-negative-modifiers-for-default-features.c b/clang/test/Driver/aarch64-negative-modifiers-for-default-features.c
new file mode 100644
index 000000000000000..03dd8af7588cae1
--- /dev/null
+++ b/clang/test/Driver/aarch64-negative-modifiers-for-default-features.c
@@ -0,0 +1,12 @@
+// Test that default features (e.g. flagm/sb/ssbs for 8.5) can be disabled via -march.
+
+// RUN: %clang --target=aarch64 -march=armv8.5-a+noflagm+nosb+nossbs -c %s -### 2>&1 | FileCheck %s
+// CHECK: "-triple" "aarch64"
+// CHECK-SAME: "-target-feature" "+v8.5a"
+// CHECK-SAME: "-target-feature" "-flagm"
+// CHECK-SAME: "-target-feature" "-sb"
+// CHECK-SAME: "-target-feature" "-ssbs"
+
+// CHECK-NOT: "-target-feature" "+flagm"
+// CHECK-NOT: "-target-feature" "+sb"
+// CHECK-NOT: "-target-feature" "+ssbs"
diff --git a/clang/test/Driver/arm-sb.c b/clang/test/Driver/arm-sb.c
index f2704f33c271113..9c0f381171cb6df 100644
--- a/clang/test/Driver/arm-sb.c
+++ b/clang/test/Driver/arm-sb.c
@@ -11,6 +11,6 @@
 
 // RUN: %clang -### -target arm-none-none-eabi %s 2>&1 | FileCheck %s --check-prefix=ABSENT
 // RUN: %clang -### -target aarch64-none-elf %s 2>&1 | FileCheck %s --check-prefix=ABSENT
-// RUN: %clang -### -target aarch64-none-elf -march=armv8.5a+nosb %s 2>&1 | FileCheck %s --check-prefix=ABSENT
+// RUN: %clang -### -target aarch64-none-elf -march=armv8.5a+nosb %s 2>&1 | FileCheck %s --check-prefix=NOSB
 // ABSENT-NOT: "-target-feature" "+sb"
 // ABSENT-NOT: "-target-feature" "-sb"
diff --git a/clang/test/Driver/print-enabled-extensions/aarch64-apple-a12.c b/clang/test/Driver/print-enabled-extensions/aarch64-apple-a12.c
index 27f066a3107086a..fcc8b674df33f44 100644
--- a/clang/test/Driver/print-enabled-extensions/aarch64-apple-a12.c
+++ b/clang/test/Driver/print-enabled-extensions/aarch64-apple-a12.c
@@ -6,7 +6,6 @@
 // CHECK-NEXT:     Architecture Feature(s)                                Description
 // CHECK-NEXT:     FEAT_AES, FEAT_PMULL                                   Enable AES support
 // CHECK-NEXT:     FEAT_AdvSIMD                                           Enable Advanced SIMD instructions
-// CHECK-NEXT:     FEAT_CCIDX                                             Enable Armv8.3-A Extend of the CCSIDR number of sets
 // CHECK-NEXT:     FEAT_CRC32                                             Enable Armv8.0-A CRC-32 checksum instructions
 // CHECK-NEXT:     FEAT_DPB                                               Enable Armv8.2-A data Cache Clean to Point of Persistence
 // CHECK-NEXT:     FEAT_FCMA                                              Enable Armv8.3-A Floating-point complex number support
diff --git a/clang/test/Driver/print-enabled-extensions/aarch64-apple-a13.c b/clang/test/Driver/print-enabled-extensions/aarch64-apple-a13.c
index 197b21025995106..dae95b1297e1451 100644
--- a/clang/test/Driver/print-enabled-extensions/aarch64-apple-a13.c
+++ b/clang/test/Driver/print-enabled-extensions/aarch64-apple-a13.c
@@ -7,7 +7,6 @@
 // CHECK-NEXT:     FEAT_AES, FEAT_PMULL                                   Enable AES support
 // CHECK-NEXT:     FEAT_AMUv1                                             Enable Armv8.4-A Activity Monitors extension
 // CHECK-NEXT:     FEAT_AdvSIMD                                           Enable Advanced SIMD instructions
-// CHECK-NEXT:     FEAT_CCIDX                                             Enable Armv8.3-A Extend of the CCSIDR number of sets
 // CHECK-NEXT:     FEAT_CRC32                                             Enable Armv8.0-A CRC-32 checksum instructions
 // CHECK-NEXT:     FEAT_DIT                                               Enable Armv8.4-A Data Independent Timing instructions
 // CHECK-NEXT:     FEAT_DPB                                               Enable Armv8.2-A data Cache Clean to Point of Persistence
diff --git a/clang/test/Driver/print-enabled-extensions/aarch64-apple-a14.c b/clang/test/Driver/print-enabled-extensions/aarch64-apple-a14.c
index f1731ef034a0c1f..8ddcddede4110d9 100644
--- a/clang/test/Driver/print-enabled-extensions/aarch64-apple-a14.c
+++ b/clang/test/Driver/print-enabled-extensions/aarch64-apple-a14.c
@@ -7,7 +7,6 @@
 // CHECK-NEXT:     FEAT_AES, FEAT_PMULL                                   Enable AES support
 // CHECK-NEXT:     FEAT_AMUv1                                             Enable Armv8.4-A Activity Monitors extension
 // CHECK-NEXT:     FEAT_AdvSIMD                                           Enable Advanced SIMD instructions
-// CHECK-NEXT:     FEAT_CCIDX                                             Enable Armv8.3-A Extend of the CCSIDR number of sets
 // CHECK-NEXT:     FEAT_CRC32                                             Enable Armv8.0-A CRC-32 checksum instructions
 // CHECK-NEXT:     FEAT_CSV2_2                                            Enable architectural speculation restriction
 // CHECK-NEXT:     FEAT_DIT                                               Enable Armv8.4-A Data Independent Timing instructions
diff --git a/clang/test/Driver/print-enabled-extensions/aarch64-apple-a15.c b/clang/test/Driver/print-enabled-extensions/aarch64-apple-a15.c
index dec48bb70331141..b3f0acefd1e2d48 100644
--- a/clang/test/Driver/print-enabled-extensions/aarch64-apple-a15.c
+++ b/clang/test/Driver/print-enabled-extensions/aarch64-apple-a15.c
@@ -10,7 +10,6 @@
 // CHECK-NEXT:     FEAT_AdvSIMD                                           Enable Advanced SIMD instructions
 // CHECK-NEXT:     FEAT_BF16                                              Enable BFloat16 Extension
 // CHECK-NEXT:     FEAT_BTI                                               Enable Branch Target Identification
-// CHECK-NEXT:     FEAT_CCIDX                                             Enable Armv8.3-A Extend of the CCSIDR number of sets
 // CHECK-NEXT:     FEAT_CRC32                                             Enable Armv8.0-A CRC-32 checksum instructions
 // CHECK-NEXT:     FEAT_CSV2_2                                            Enable architectural speculation restriction
 // CHECK-NEXT:     FEAT_DIT                                               Enable Armv8.4-A Data Independent Timing instructions
diff --git a/clang/test/Driver/print-enabled-extensions/aarch64-apple-a16.c b/clang/test/Driver/print-enabled-extensions/aarch64-apple-a16.c
index 477652d83d82cbb..6f417c1592b06cd 100644
--- a/clang/test/Driver/print-enabled-extensions/aarch64-apple-a16.c
+++ b/clang/test/Driver/print-enabled-extensions/aarch64-apple-a16.c
@@ -10,7 +10,6 @@
 // CHECK-NEXT:     FEAT_AdvSIMD                                           Enable Advanced SIMD instructions
 // CHECK-NEXT:     FEAT_BF16                                              Enable BFloat16 Extension
 // CHECK-NEXT:     FEAT_BTI                                               Enable Branch Target Identification
-// CHECK-NEXT:     FEAT_CCIDX                                             Enable Armv8.3-A Extend of the CCSIDR number of sets
 // CHECK-NEXT:     FEAT_CRC32                                             Enable Armv8.0-A CRC-32 checksum instructions
 // CHECK-NEXT:     FEAT_CSV2_2                                            Enable architectural speculation restriction
 // CHECK-NEXT:     FEAT_DIT                                               Enable Armv8.4-A Data Independent Timing instructions
diff --git a/clang/test/Driver/print-enabled-extensions/aarch64-apple-a17.c b/clang/test/Driver/print-enabled-extensions/aarch64-apple-a17.c
index 311cc94acddc96e..39e5b70c25d8860 100644
--- a/clang/test/Driver/print-enabled-extensions/aarch64-apple-a17.c
+++ b/clang/test/Driver/print-enabled-extensions/aarch64-apple-a17.c
@@ -10,7 +10,6 @@
 // CHECK-NEXT:     FEAT_AdvSIMD                                           Enable Advanced SIMD instructions
 // CHECK-NEXT:     FEAT_BF16                                              Enable BFloat16 Extension
 // CHECK-NEXT:     FEAT_BTI                                               Enable Branch Target Identification
-// CHECK-NEXT:     FEAT_CCIDX                                             Enable Armv8.3-A Extend of the CCSIDR number of sets
 // CHECK-NEXT:     FEAT_CRC32                                             Enable Armv8.0-A CRC-32 checksum instructions
 // CHECK-NEXT:     FEAT_CSV2_2                                            Enable architectural speculation restriction
 // CHECK-NEXT:     FEAT_DIT                                               Enable Armv8.4-A Data Independent Timing instructions
diff --git a/clang/test/Driver/print-enabled-extensions/aarch64-apple-m4.c b/clang/test/Driver/print-enabled-extensions/aarch64-apple-m4.c
index 44d618afef40689..29d66dc8826a77f 100644
--- a/clang/test/Driver/print-enabled-extensions/aarch64-apple-m4.c
+++ b/clang/test/Driver/print-enabled-extensions/aarch64-apple-m4.c
@@ -10,7 +10,6 @@
 // CHECK-NEXT:     FEAT_AdvSIMD                                           Enable Advanced SIMD instructions
 // CHECK-NEXT:     FEAT_BF16                                              Enable BFloat16 Extension
 // CHECK-NEXT:     FEAT_BTI                                               Enable Branch Target Identification
-// CHECK-NEXT:     FEAT_CCIDX                                             Enable Armv8.3-A Extend of the CCSIDR number of sets
 // CHECK-NEXT:     FEAT_CRC32                                             Enable Armv8.0-A CRC-32 checksum instructions
 // CHECK-NEXT:     FEAT_CSV2_2                                            Enable architectural speculation restriction
 // CHECK-NEXT:     FEAT_DIT                                               Enable Armv8.4-A Data Independent Timing instructions
@@ -52,7 +51,6 @@
 // CHECK-NEXT:     FEAT_SME_F64F64                                        Enable Scalable Matrix Extension (SME) F64F64 instructions
 // CHECK-NEXT:     FEAT_SME_I16I64                                        Enable Scalable Matrix Extension (SME) I16I64 instructions
 // CHECK-NEXT:     FEAT_SPECRES                                           Enable Armv8.5-A execution and data prediction invalidation instructions
-// CHECK-NEXT:     FEAT_SSBS, FEAT_SSBS2                                  Enable Speculative Store Bypass Safe bit
 // CHECK-NEXT:     FEAT_TLBIOS, FEAT_TLBIRANGE                            Enable Armv8.4-A TLB Range and Maintenance instructions
 // CHECK-NEXT:     FEAT_TRF                                               Enable Armv8.4-A Trace extension
 // CHECK-NEXT:     FEAT_UAO                                               Enable Armv8.2-A UAO PState
diff --git a/clang/test/Driver/print-enabled-extensions/aarch64-cortex-r82.c b/clang/test/Driver/print-enabled-extensions/aarch64-cortex-r82.c
index 2b85201c2c6fe61..9875c6922d379a9 100644
--- a/clang/test/Driver/print-enabled-extensions/aarch64-cortex-r82.c
+++ b/clang/test/Driver/print-enabled-extensions/aarch64-cortex-r82.c
@@ -5,7 +5,6 @@
 // CHECK-EMPTY:
 // CHECK-NEXT:     Architecture Feature(s)                                Description
 // CHECK-NEXT:     FEAT_AdvSIMD                                           Enable Advanced SIMD instructions
-// CHECK-NEXT:     FEAT_CCIDX                                             Enable Armv8.3-A Extend of the CCSIDR number of sets
 // CHECK-NEXT:     FEAT_CRC32                                             Enable Armv8.0-A CRC-32 checksum instructions
 // CHECK-NEXT:     FEAT_CSV2_2                                            Enable architectural speculation restriction
 // CHECK-NEXT:     FEAT_DIT                                               Enable Armv8.4-A Data Independent Timing instructions
diff --git a/clang/test/Driver/print-enabled-extensions/aarch64-cortex-r82ae.c b/clang/test/Driver/print-enabled-extensions/aarch64-cortex-r82ae.c
index 417687b4af287d4..2db44d7827aadb7 100644
--- a/clang/test/Driver/print-enabled-extensions/aarch64-cortex-r82ae.c
+++ b/clang/test/Driver/print-enabled-extensions/aarch64-cortex-r82ae.c
@@ -5,7 +5,6 @@
 // CHECK-EMPTY:
 // CHECK-NEXT:     Architecture Feature(s)                                Description
 // CHECK-NEXT:     FEAT_AdvSIMD                                           Enable Advanced SIMD instructions
-// CHECK-NEXT:     FEAT_CCIDX                                             Enable Armv8.3-A Extend of the CCSIDR number of sets
 // CHECK-NEXT:     FEAT_CRC32                                             Enable Armv8.0-A CRC-32 checksum instructions
 // CHECK-NEXT:     FEAT_CSV2_2                                            Enable architectural speculation restriction
 // CHECK-NEXT:     FEAT_DIT                                               Enable Armv8.4-A Data Independent Timing instructions
diff --git a/llvm/lib/Target/AArch64/AArch64Features.td b/llvm/lib/Target/AArch64/AArch64Features.td
index f157b1053f59fa0..69d6b02fefffe93 100644
--- a/llvm/lib/Target/AArch64/AArch64Features.td
+++ b/llvm/lib/Target/AArch64/AArch64Features.td
@@ -779,27 +779,26 @@ def HasV8_2aOps : Architecture64<8, 2, "a", "v8.2a",
   [HasV8_1aOps, FeaturePsUAO, FeaturePAN_RWV, FeatureRAS, FeatureCCPP],
   !listconcat(HasV8_1aOps.DefaultExts, [FeatureRAS])>;
 def HasV8_3aOps : Architecture64<8, 3, "a", "v8.3a",
-  [HasV8_2aOps, FeatureRCPC, FeaturePAuth, FeatureJS, FeatureCCIDX,
-    FeatureComplxNum],
+  [HasV8_2aOps, FeatureRCPC, FeaturePAuth, FeatureJS, FeatureComplxNum],
   !listconcat(HasV8_2aOps.DefaultExts, [FeatureComplxNum, FeatureJS,
-    FeaturePAuth, FeatureRCPC])>;
+    FeaturePAuth, FeatureRCPC,  FeatureCCIDX])>;
 def HasV8_4aOps : Architecture64<8, 4, "a", "v8.4a",
   [HasV8_3aOps, FeatureDotProd, FeatureNV, FeatureMPAM, FeatureDIT,
     FeatureTRACEV8_4, FeatureAM, FeatureSEL2, FeatureTLB_RMI, FeatureFlagM,
     FeatureRCPC_IMMO, FeatureLSE2],
-  !listconcat(HasV8_3aOps.DefaultExts, [FeatureDotProd])>;
+  !listconcat(HasV8_3aOps.DefaultExts, [FeatureDotProd, FeatureDIT, FeatureFlagM])>;
 def HasV8_5aOps : Architecture64<8, 5, "a", "v8.5a",
   [HasV8_4aOps, FeatureAltFPCmp, FeatureFRInt3264, FeatureSpecRestrict,
-    FeatureSSBS, FeatureSB, FeaturePredRes, FeatureCacheDeepPersist,
+    FeatureSB, FeaturePredRes, FeatureCacheDeepPersist,
     FeatureBranchTargetId],
-  !listconcat(HasV8_4aOps.DefaultExts, [])>;
+  !listconcat(HasV8_4aOps.DefaultExts, [FeaturePredRes, FeatureSSBS, FeatureBranchTargetId, FeatureSB])>;
 def HasV8_6aOps : Architecture64<8, 6, "a", "v8.6a",
   [HasV8_5aOps, FeatureAMVS, FeatureBF16, FeatureFineGrainedTraps,
     FeatureEnhancedCounterVirtualization, FeatureMatMulInt8],
   !listconcat(HasV8_5aOps.DefaultExts, [FeatureBF16, FeatureMatMulInt8])>;
 def HasV8_7aOps : Architecture64<8, 7, "a", "v8.7a",
   [HasV8_6aOps, FeatureXS, FeatureWFxT, FeatureHCX],
-  !listconcat(HasV8_6aOps.DefaultExts, [])>;
+  !listconcat(HasV8_6aOps.DefaultExts, [FeatureWFxT])>;
 def HasV8_8aOps : Architecture64<8, 8, "a", "v8.8a",
   [HasV8_7aOps, FeatureHBC, FeatureMOPS, FeatureNMI],
   !listconcat(HasV8_7aOps.DefaultExts, [FeatureMOPS, FeatureHBC])>;
@@ -817,7 +816,7 @@ def HasV9_1aOps : Architecture64<9, 1, "a", "v9.1a",
   !listconcat(HasV9_0aOps.DefaultExts, [FeatureBF16, FeatureMatMulInt8, FeatureRME])>;
 def HasV9_2aOps : Architecture64<9, 2, "a", "v9.2a",
   [HasV8_7aOps, HasV9_1aOps],
-  !listconcat(HasV9_1aOps.DefaultExts, [FeatureMEC])>;
+  !listconcat(HasV9_1aOps.DefaultExts, [FeatureMEC, FeatureWFxT])>;
 def HasV9_3aOps : Architecture64<9, 3, "a", "v9.3a",
   [HasV8_8aOps, HasV9_2aOps],
   !listconcat(HasV9_2aOps.DefaultExts, [FeatureMOPS, FeatureHBC])>;
@@ -834,7 +833,7 @@ def HasV8_0rOps : Architecture64<8, 0, "r", "v8r",
     //v8.2
     FeatureRAS, FeaturePsUAO, FeatureCCPP, FeaturePAN_RWV,
     //v8.3
-    FeatureCCIDX, FeaturePAuth, FeatureRCPC,
+    FeaturePAuth, FeatureRCPC,
     //v8.4
     FeatureTRACEV8_4, FeatureTLB_RMI, FeatureFlagM, FeatureDIT, FeatureSEL2,
     FeatureRCPC_IMMO,
@@ -845,7 +844,7 @@ def HasV8_0rOps : Architecture64<8, 0, "r", "v8r",
   // For v8-R, we do not enable crypto and align with GCC that enables a more
   // minimal set of optional architecture extensions.
   !listconcat(
-    !listremove(HasV8_5aOps.DefaultExts, [FeatureLSE]),
+    !listremove(HasV8_5aOps.DefaultExts, [FeatureBranchTargetId, FeaturePredRes]),
     [FeatureSSBS, FeatureFullFP16, FeatureFP16FML, FeatureSB]
   )>;
 
diff --git a/llvm/lib/Target/AArch64/AArch64Processors.td b/llvm/lib/Target/AArch64/AArch64Processors.td
index 83f7fa761e00ea5..84d8cae3a0a5d18 100644
--- a/llvm/lib/Target/AArch64/AArch64Processors.td
+++ b/llvm/lib/Target/AArch64/AArch64Processors.td
@@ -688,6 +688,7 @@ def ProcessorFeatures {
                                  FeatureMatMulInt8, FeatureBF16, FeatureAM,
                                  FeatureMTE, FeatureETE, FeatureSVE2BitPerm,
                                  FeatureFP16FML,
+                                 FeatureCCIDX,
                                  FeatureSB, FeaturePAuth, FeatureSSBS, FeatureSVE, FeatureSVE2,
                                  FeatureComplxNum, FeatureCRC, FeatureDotProd,
                                  FeatureFPARMv8,FeatureFullFP16, FeatureJS, FeatureLSE,
@@ -695,6 +696,7 @@ def ProcessorFeatures {
   list<SubtargetFeature> A520 = [HasV9_2aOps, FeaturePerfMon, FeatureAM,
                                  FeatureMTE, FeatureETE, FeatureSVE2BitPerm,
                                  FeatureFP16FML,
+                                 FeatureCCIDX,
                                  FeatureSB, FeatureSSBS, FeaturePAuth, FeatureFlagM, FeaturePredRes,
                                  FeatureSVE, FeatureSVE2, FeatureBF16, FeatureComplxNum, FeatureCRC,
                                  FeatureFPARMv8, FeatureFullFP16, FeatureMatMulInt8, FeatureJS,
@@ -703,6 +705,7 @@ def ProcessorFeatures {
   list<SubtargetFeature> A520AE = [HasV9_2aOps, FeaturePerfMon, FeatureAM,
                                  FeatureMTE, FeatureETE, FeatureSVE2BitPerm,
                                  FeatureFP16FML,
+                                 FeatureCCIDX,
                                  FeatureSB, FeatureSSBS, FeaturePAuth, FeatureFlagM, FeaturePredRes,
                                  FeatureSVE, FeatureSVE2, FeatureBF16, FeatureComplxNum, FeatureCRC,
                                  FeatureFPARMv8, FeatureFullFP16, FeatureMatMulInt8, FeatureJS,
@@ -734,12 +737,14 @@ def ProcessorFeatures {
                                  FeaturePerfMon, FeatureRCPC, FeatureSPE,
                                  FeatureSSBS, FeatureCRC, FeatureLSE, FeatureRAS, FeatureRDM];
   list<SubtargetFeature> A710 = [HasV9_0aOps, FeatureNEON, FeaturePerfMon,
+                                 FeatureCCIDX, FeatureSSBS,
                                  FeatureETE, FeatureMTE, FeatureFP16FML,
                                  FeatureSVE2BitPerm, FeatureBF16, FeatureMatMulInt8,
                                  FeaturePAuth, FeatureFlagM, FeatureSB, FeatureSVE, FeatureSVE2,
                                  FeatureComplxNum, FeatureCRC, FeatureDotProd, FeatureFPARMv8,
                                  FeatureFullFP16, FeatureJS, FeatureLSE, FeatureRAS, FeatureRCPC, FeatureRDM];
   list<SubtargetFeature> A715 = [HasV9_0aOps, FeatureNEON, FeatureMTE,
+                                 FeatureCCIDX,
                                  FeatureFP16FML, FeatureSVE, FeatureTRBE,
                                  FeatureSVE2BitPerm, FeatureBF16, FeatureETE,
                                  FeaturePerfMon, FeatureMatMulInt8, FeatureSPE,
@@ -749,6 +754,7 @@ def ProcessorFeatures {
                                  FeatureJS, FeatureLSE, FeatureRAS,
                                  FeatureRCPC, FeatureRDM];
   list<SubtargetFeature> A720 = [HasV9_2aOps, FeatureMTE, FeatureFP16FML,
+                                 FeatureCCIDX,
                                  FeatureTRBE, FeatureSVE2BitPerm, FeatureETE,
                                  FeaturePerfMon, FeatureSPE, FeatureSPE_EEF,
                                  FeatureSB, FeatureSSBS, FeaturePAuth, FeatureFlagM, FeaturePredRes,
@@ -757,6 +763,7 @@ def ProcessorFeatures {
                                  FeatureJS, FeatureLSE, FeatureNEON, FeatureRAS,
                                  FeatureRCPC, FeatureRDM];
   list<SubtargetFeature> A720AE = [HasV9_2aOps, FeatureMTE, FeatureFP16FML,
+                                 FeatureCCIDX,
                                  FeatureTRBE, FeatureSVE2BitPerm, FeatureETE,
                                  FeaturePerfMon, FeatureSPE, FeatureSPE_EEF,
                                  FeatureSB, FeatureSSBS, FeaturePAuth, FeatureFlagM, FeaturePredRes,
@@ -765,6 +772,7 @@ def ProcessorFeatures {
                                  FeatureJS, FeatureLSE, FeatureNEON, FeatureRAS,
                                  FeatureRCPC, FeatureRDM];
   list<SubtargetFeature> A725 = [HasV9_2aOps, FeatureMTE, FeatureFP16FML,
+                                 FeatureCCIDX,
                                  FeatureETE, FeaturePerfMon, FeatureSPE,
                                  FeatureSVE2BitPerm, FeatureSPE_EEF, FeatureTRBE,
                                  FeatureFlagM, FeaturePredRes, FeatureSB, FeatureSSBS,
@@ -800,6 +808,7 @@ def ProcessorFeatures {
                                  FeatureMatMulInt8, FeatureBF16, FeatureAM,
                                  FeatureMTE, FeatureETE, FeatureSVE2BitPerm,
                                  FeatureFP16FML,
+                                 FeatureCCIDX,
                                  FeaturePAuth, FeatureSSBS, FeatureSB, FeatureSVE, FeatureSVE2, FeatureFlagM,
                                  FeatureComplxNum, FeatureCRC, FeatureDotProd, FeatureFPARMv8, FeatureFullFP16,
                                  FeatureJS, FeatureLSE, FeatureRAS, FeatureRCPC, FeatureRDM];
@@ -808,6 +817,7 @@ def ProcessorFeatures {
                                  FeatureSPE, FeatureBF16, FeatureMatMulInt8,
                                  FeatureMTE, FeatureSVE2BitPerm, FeatureFullFP16,
                                  FeatureFP16FML,
+                                 FeatureCCIDX,
                                  FeatureSB, FeaturePAuth, FeaturePredRes, FeatureFlagM, FeatureSSBS,
                                  FeatureSVE2, FeatureComplxNum, FeatureCRC, FeatureFPARMv8, FeatureJS,
                                  FeatureLSE, FeatureRAS, FeatureRCPC, FeatureRDM, FeatureDotProd];
@@ -815,11 +825,13 @@ def ProcessorFeatures {
                                  FeaturePerfMon, FeatureETE, FeatureTRBE,
                                  FeatureSPE, FeatureMTE, FeatureSVE2BitPerm,
                                  FeatureFP16FML, FeatureSPE_EEF,
+                                 FeatureCCIDX,
                                  FeatureSB, FeatureSSBS, FeaturePAuth, FeatureFlagM, FeaturePredRes,
                                  FeatureSVE, FeatureSVE2, FeatureComplxNum, FeatureCRC, FeatureDotProd,
                                  FeatureFPARMv8, FeatureFullFP16, FeatureMatMulInt8, FeatureJS, FeatureLSE,
                                  FeatureNEON, FeatureRAS, FeatureRCPC, FeatureRDM, FeatureBF16];
   list<SubtargetFeature> X925 = [HasV9_2aOps, FeatureMTE, FeatureFP16FML,
+                                 FeatureCCIDX,
                                  FeatureETE, FeaturePerfMon, FeatureSPE,
                                  FeatureSVE2BitPerm, FeatureSPE_EEF, FeatureTRBE,
                                  FeatureFlagM, FeaturePredRes, FeatureSB, FeatureSSBS,
@@ -866,7 +878,7 @@ def ProcessorFeatures {
                                      FeatureComplxNum, FeatureCRC, FeatureJS,
                                      FeatureLSE, FeaturePAuth, FeatureFPAC,
                                      FeatureRAS, FeatureRCPC, FeatureRDM,
-                                     FeatureBF16, FeatureDotProd, FeatureMatMulInt8];
+                                     FeatureBF16, FeatureDotProd, FeatureMatMulInt8, FeatureSSBS];
   list<SubtargetFeature> AppleA16 = [HasV8_6aOps, FeatureSHA2, FeatureAES, FeatureFPARMv8,
                                      FeatureNEON, FeaturePerfMon, FeatureSHA3,
                                      FeatureFullFP16, FeatureFP16FML,
@@ -874,7 +886,7 @@ def ProcessorFeatures {
                                      FeatureComplxNum, FeatureCRC, FeatureJS,
                                      FeatureLSE, FeaturePAuth, FeatureFPAC,
                                      FeatureRAS, FeatureRCPC, FeatureRDM,
-                                     FeatureBF16, FeatureDotProd, FeatureMatMulInt8];
+                                     FeatureBF16, FeatureDotProd, FeatureMatMulInt8, FeatureSSBS];
   list<SubtargetFeature> AppleA17 = [HasV8_6aOps, FeatureSHA2, FeatureAES, FeatureFPARMv8,
                                      FeatureNEON, FeaturePerfMon, FeatureSHA3,
                                      FeatureFullFP16, FeatureFP16FML,
@@ -882,7 +894,7 @@ def ProcessorFeatures {
                                      FeatureComplxNum, FeatureCRC, FeatureJS,
                                      FeatureLSE, FeaturePAuth, FeatureFPAC,
                                      FeatureRAS, FeatureRCPC, FeatureRDM,
-                                     FeatureBF16, FeatureDotProd, FeatureMatMulInt8];
+                                     FeatureBF16, FeatureDotProd, FeatureMatMulInt8, FeatureSSBS];
   list<SubtargetFeature> AppleM4 = [HasV9_2aOps, FeatureSHA2, FeatureFPARMv8,
                                     FeatureNEON, FeaturePerfMon, FeatureSHA3,
                                     FeatureFullFP16, FeatureFP16FML,
@@ -913,6 +925,7 @@ def ProcessorFeatures {
                                        FeatureMatMulInt8, FeatureMTE, FeatureSVE2,
                                        FeatureSVE2BitPerm, FeatureTRBE,
                                        FeaturePerfMon,
+                                       FeatureCCIDX,
                                        FeatureDotProd, FeatureFullFP16, FeatureSB, FeatureSSBS, FeatureSVE,
                                        FeatureComplxNum, FeatureCRC, FeatureFPARMv8, FeatureJS, FeatureLSE,
                                        FeatureNEON, FeaturePAuth, FeatureRAS, FeatureRCPC, FeatureRDM];
@@ -920,6 +933,7 @@ def ProcessorFeatures {
                                       FeatureFullFP16, FeatureMTE, FeaturePerfMon,
                                       FeatureRandGen, FeatureSPE, FeatureSPE_EEF,
                                       FeatureSVE2BitPerm,
+                                      FeatureCCIDX,
                                       FeatureSSBS, FeatureSB, FeaturePredRes, FeaturePAuth, FeatureFlagM,
                                       FeatureSVE, FeatureSVE2, FeatureBF16, FeatureComplxNum,
                                       FeatureCRC, FeatureDotProd, FeatureFPARMv8, FeatureMatMulInt8,
@@ -930,6 +944,7 @@ def ProcessorFeatures {
                                            FeatureFullFP16, FeatureMatMulInt8, FeatureNEON,
                                            FeaturePerfMon, FeatureRandGen, FeatureSPE,
                                            FeatureSSBS, FeatureSVE,
+                                           FeatureCCIDX,
                                            FeatureSHA3, FeatureSM4, FeatureDotProd, FeatureComplxNum,
                                            FeatureCRC, FeatureJS, FeatureLSE, FeaturePAuth, FeatureRAS,
                                            FeatureRCPC, FeatureRDM];
@@ -938,6 +953,7 @@ def ProcessorFeatures {
                                        FeatureFullFP16, FeatureMatMulInt8, FeatureNEON,
                                        FeaturePerfMon, FeatureRandGen, FeatureSPE,
                                        FeatureSSBS, FeatureSVE,
+                                       FeatureCCIDX,
                                        FeatureSHA3, FeatureSM4, FeatureDotProd, FeatureComplxNum,
                                        FeatureCRC, FeatureJS, FeatureLSE, FeaturePAuth, FeatureRAS,
                                        FeatureRCPC, FeatureRDM];
@@ -945,12 +961,14 @@ def ProcessorFeatures {
                                        FeaturePerfMon, FeatureETE, FeatureMatMulInt8,
                                        FeatureNEON, FeatureSVE2BitPerm, FeatureFP16FML,
                                        FeatureMTE, FeatureRandGen,
+                                       FeatureCCIDX,
                                        FeatureSVE, FeatureSVE2, FeatureSSBS, FeatureFullFP16, FeatureDotProd,
                                        FeatureComplxNum, FeatureCRC, FeatureFPARMv8, FeatureJS, FeatureLSE,
                                        FeaturePAuth, FeatureRAS, FeatureRCPC, FeatureRDM];
   list<SubtargetFeature> NeoverseV3 = [HasV9_2aOps, FeatureETE, FeatureFP16FML,
                                       FeatureFullFP16, FeatureLS64, FeatureMTE,
                                       FeaturePerfMon, FeatureRandGen, FeatureSPE,
+                                      FeatureCCIDX,
                                       FeatureSPE_EEF, FeatureSVE2BitPerm, FeatureBRBE,
                                       FeatureSSBS, FeatureSB, FeaturePredRes, FeaturePAuth, FeatureFlagM,
                                       FeatureSVE, FeatureSVE2, FeatureBF16, FeatureComplxNum, FeatureCRC,
@@ -961,12 +979,14 @@ def ProcessorFeatures {
                                       FeaturePerfMon, FeatureRandGen, FeatureSPE,
                                       FeatureSPE_EEF, FeatureSVE2BitPerm, FeatureBRBE,
                                       FeatureSSBS, FeatureSB, FeaturePredRes, FeaturePAuth, FeatureFlagM,
+                                      FeatureCCIDX,
                                       FeatureSVE, FeatureSVE2, FeatureBF16, FeatureComplxNum, FeatureCRC,
                                       FeatureDotProd, FeatureFPARMv8, FeatureMatMulInt8, FeatureJS,
                                       FeatureLSE, FeatureNEON, FeatureRAS, FeatureRCPC, FeatureRDM,
                                       FeatureRME];
   list<SubtargetFeature> Saphira    = [HasV8_4aOps, FeatureSHA2, FeatureAES, FeatureFPARMv8,
                                        FeatureNEON, FeatureSPE, FeaturePerfMon, FeatureCRC,
+                                       FeatureCCIDX,
                                        FeatureLSE, FeatureRDM, FeatureRAS, FeatureRCPC];
   list<SubtargetFeature> ThunderX   = [HasV8_0aOps, FeatureCRC, FeatureSHA2, FeatureAES,
                                        FeatureFPARMv8, FeaturePerfMon, FeatureNEON];
@@ -975,6 +995,7 @@ def ProcessorFeatures {
                                           FeatureRDM];
   list<SubtargetFeature> ThunderX3T110 = [HasV8_3aOps, FeatureCRC, FeatureSHA2, FeatureAES,
                                           FeatureFPARMv8, FeatureNEON, FeatureLSE,
+                                          FeatureCCIDX,
                                           FeaturePAuth, FeaturePerfMon, FeatureComplxNum,
                                           FeatureJS, FeatureRAS, FeatureRCPC, FeatureRDM];
   list<SubtargetFeature> TSV110 = [HasV8_2aOps, FeatureSHA2, FeatureAES, FeatureFPARMv8,
@@ -987,6 +1008,7 @@ def ProcessorFeatures {
                                     FeatureSHA2, FeatureSHA3, FeatureAES,
                                     FeatureFullFP16, FeatureBF16, FeatureComplxNum, FeatureCRC,
                                     FeatureDotProd, FeatureFPARMv8, FeatureMatMulInt8, FeatureJS,
+                                    FeatureCCIDX,
                                     FeatureLSE, FeaturePAuth, FeatureRAS, FeatureRCPC, FeatureRDM];
   list<SubtargetFeature> Ampere1A = [HasV8_6aOps, FeatureNEON, FeaturePerfMon,
                                      FeatureMTE, FeatureSSBS, FeatureRandGen,
@@ -995,6 +1017,7 @@ def ProcessorFeatures {
                                      FeatureFullFP16, FeatureBF16, FeatureComplxNum,
                                      FeatureCRC, FeatureDotProd, FeatureFPARMv8, FeatureMatMulInt8,
                                      FeatureJS, FeatureLSE, FeaturePAuth, FeatureRAS, FeatureRCPC,
+                                     FeatureCCIDX,
                                      FeatureRDM];
   list<SubtargetFeature> Ampere1B = [HasV8_7aOps, FeatureNEON, FeaturePerfMon,
                                      FeatureMTE, FeatureSSBS, FeatureRandGen,
@@ -1003,6 +1026,7 @@ def ProcessorFeatures {
                                      FeatureWFxT, FeatureFullFP16, FeatureBF16, FeatureComplxNum,
                                      FeatureCRC, FeatureDotProd, FeatureFPARMv8, FeatureMatMulInt8,
                                      FeatureJS, FeatureLSE, FeaturePAuth, FeatureRAS, FeatureRCPC,
+                                     FeatureCCIDX,
                                      FeatureRDM];
 
   list<SubtargetFeature> Oryon = [HasV8_6aOps, FeatureNEON, FeaturePerfMon,
@@ -1011,6 +1035,7 @@ def ProcessorFeatures {
                                      FeatureSHA3, FeatureAES,
                                      FeatureSPE, FeatureBF16, FeatureComplxNum, FeatureCRC,
                                      FeatureDotProd, FeatureFPARMv8, FeatureMatMulInt8,
+                                     FeatureSSBS, FeatureCCIDX,
                                      FeatureJS, FeatureLSE, FeatureRAS, FeatureRCPC, FeatureRDM];
 
   // ETE and TRBE are future architecture extensions. We temporarily enable them
diff --git a/llvm/test/MC/AArch64/arm64-system-encoding.s b/llvm/test/MC/AArch64/arm64-system-encoding.s
index 313ec9117746035..c58a8f0cb841cb8 100644
--- a/llvm/test/MC/AArch64/arm64-system-encoding.s
+++ b/llvm/test/MC/AArch64/arm64-system-encoding.s
@@ -1,5 +1,5 @@
 ; RUN: not llvm-mc -triple arm64-apple-darwin -show-encoding < %s 2> %t | FileCheck %s
-; RUN: not llvm-mc -triple arm64-apple-darwin -mattr=+v8.3a -show-encoding < %s 2> %t | FileCheck %s --check-prefix=CHECK-V83
+; RUN: not llvm-mc -triple arm64-apple-darwin -mattr=+ccidx -show-encoding < %s 2> %t | FileCheck %s --check-prefix=CHECK-V83
 ; RUN: FileCheck --check-prefix=CHECK-ERRORS < %t %s
 
 foo:
diff --git a/llvm/test/MC/AArch64/armv8.5a-ssbs-error.s b/llvm/test/MC/AArch64/armv8.5a-ssbs-error.s
index a7c9f4c4fbb5c3f..cd5ab43046c798a 100644
--- a/llvm/test/MC/AArch64/armv8.5a-ssbs-error.s
+++ b/llvm/test/MC/AArch64/armv8.5a-ssbs-error.s
@@ -1,5 +1,5 @@
 // RUN: not llvm-mc -triple aarch64 -show-encoding -mattr=+ssbs  < %s 2>&1 | FileCheck %s
-// RUN: not llvm-mc -triple aarch64 -show-encoding -mattr=+v8.5a < %s 2>&1 | FileCheck %s
+// RUN: not llvm-mc -triple aarch64 -show-encoding -mattr=+v8.5a < %s 2>&1 | FileCheck %s --check-prefix=NOSPECID
 // RUN: not llvm-mc -triple aarch64 -show-encoding -mattr=-ssbs  < %s 2>&1 | FileCheck %s --check-prefix=NOSPECID
 
 msr SSBS, #16
diff --git a/llvm/test/MC/AArch64/armv8.5a-ssbs.s b/llvm/test/MC/AArch64/armv8.5a-ssbs.s
index ec6670f8ecc34c3..2a8b7b000646ad4 100644
--- a/llvm/test/MC/AArch64/armv8.5a-ssbs.s
+++ b/llvm/test/MC/AArch64/armv8.5a-ssbs.s
@@ -1,5 +1,5 @@
 // RUN:     llvm-mc -triple aarch64 -show-encoding -mattr=+ssbs  < %s      | FileCheck %s
-// RUN:     llvm-mc -triple aarch64 -show-encoding -mattr=+v8.5a < %s      | FileCheck %s
+// RUN: not llvm-mc -triple aarch64 -show-encoding -mattr=+v8.5a < %s 2>&1 | FileCheck %s --check-prefix=NOSPECID
 // RUN:     llvm-mc -triple aarch64 -show-encoding -mcpu=cortex-a65 < %s   | FileCheck %s
 // RUN:     llvm-mc -triple aarch64 -show-encoding -mcpu=cortex-a65ae < %s | FileCheck %s
 // RUN:     llvm-mc -triple aarch64 -show-encoding -mcpu=cortex-a76 < %s   | FileCheck %s
diff --git a/llvm/test/MC/Disassembler/AArch64/armv8.5a-ssbs.txt b/llvm/test/MC/Disassembler/AArch64/armv8.5a-ssbs.txt
index 7698751c88076b0..84d4fa6accccf48 100644
--- a/llvm/test/MC/Disassembler/AArch64/armv8.5a-ssbs.txt
+++ b/llvm/test/MC/Disassembler/AArch64/armv8.5a-ssbs.txt
@@ -1,5 +1,5 @@
 # RUN: llvm-mc -triple=aarch64 -mattr=+ssbs  -disassemble < %s | FileCheck %s
-# RUN: llvm-mc -triple=aarch64 -mattr=+v8.5a -disassemble < %s | FileCheck %s
+# RUN: llvm-mc -triple=aarch64 -mattr=+v8.5a -disassemble < %s | FileCheck %s --check-prefix=NOSPECID
 # RUN: llvm-mc -triple=aarch64 -mcpu=cortex-a76 -disassemble < %s | FileCheck %s
 # RUN: llvm-mc -triple=aarch64 -mcpu=cortex-a76ae -disassemble < %s | FileCheck %s
 # RUN: llvm-mc -triple=aarch64 -mattr=+v8r -disassemble < %s | FileCheck %s --check-prefix=NOSPECID
diff --git a/llvm/test/MC/Disassembler/AArch64/basic-a64-instructions.txt b/llvm/test/MC/Disassembler/AArch64/basic-a64-instructions.txt
index c76bb0b9020967b..f46301e8c1c15b7 100644
--- a/llvm/test/MC/Disassembler/AArch64/basic-a64-instructions.txt
+++ b/llvm/test/MC/Disassembler/AArch64/basic-a64-instructions.txt
@@ -2,7 +2,7 @@
 # RUN: llvm-mc -triple=arm64 -mattr=+v8a,+fp-armv8 -disassemble < %s | FileCheck %s
 # RUN: llvm-mc -triple=arm64 -mattr=+v8a,+fp-armv8,+fullfp16 -disassemble < %s | FileCheck %s --check-prefix=CHECK --check-prefix=FP16
 # RUN: llvm-mc -triple=arm64 -mattr=+v8.2a -disassemble < %s | FileCheck %s --check-prefix=CHECK-V82
-# RUN: llvm-mc -triple=arm64 -mattr=+v8.3a -disassemble < %s | FileCheck %s --check-prefix=CHECK-V83
+# RUN: llvm-mc -triple=arm64 -mattr=+ccidx -disassemble < %s | FileCheck %s --check-prefix=CHECK-V83
 
 #------------------------------------------------------------------------------
 # Add/sub (immediate)
diff --git a/llvm/unittests/TargetParser/TargetParserTest.cpp b/llvm/unittests/TargetParser/TargetParserTest.cpp
index 960a9892202b35e..3d99ea8d9196288 100644
--- a/llvm/unittests/TargetParser/TargetParserTest.cpp
+++ b/llvm/unittests/TargetParser/TargetParserTest.cpp
@@ -1141,7 +1141,8 @@ INSTANTIATE_TEST_SUITE_P(
                               AArch64::AEK_FP16,        AArch64::AEK_FP16FML,
                               AArch64::AEK_SB,          AArch64::AEK_JSCVT,
                               AArch64::AEK_FCMA,        AArch64::AEK_PERFMON,
-                              AArch64::AEK_ETE,         AArch64::AEK_AM}),
+                              AArch64::AEK_ETE,         AArch64::AEK_AM,
+                              AArch64::AEK_CCIDX}),
         AArch64CPUTestParams("cortex-a520", "armv9.2-a",
                              {AArch64::AEK_BF16,        AArch64::AEK_I8MM,
                               AArch64::AEK_SVE,         AArch64::AEK_SVE2,
@@ -1156,7 +1157,7 @@ INSTANTIATE_TEST_SUITE_P(
                               AArch64::AEK_PERFMON,     AArch64::AEK_PREDRES,
                               AArch64::AEK_JSCVT,       AArch64::AEK_FCMA,
                               AArch64::AEK_PERFMON,     AArch64::AEK_AM,
-                              AArch64::AEK_ETE}),
+                              AArch64::AEK_ETE,         AArch64::AEK_CCIDX}),
         AArch64CPUTestParams("cortex-a520ae", "armv9.2-a",
                              {AArch64::AEK_BF16,        AArch64::AEK_I8MM,
                               AArch64::AEK_SVE,         AArch64::AEK_SVE2,
@@ -1171,7 +1172,7 @@ INSTANTIATE_TEST_SUITE_P(
                               AArch64::AEK_PERFMON,     AArch64::AEK_PREDRES,
                               AArch64::AEK_JSCVT,       AArch64::AEK_FCMA,
                               AArch64::AEK_PERFMON,     AArch64::AEK_AM,
-                              AArch64::AEK_ETE}),
+                              AArch64::AEK_ETE,         AArch64::AEK_CCIDX}),
         AArch64CPUTestParams("cortex-a57", "armv8-a",
                              {AArch64::AEK_CRC, AArch64::AEK_AES,
                               AArch64::AEK_SHA2, AArch64::AEK_FP,
@@ -1256,7 +1257,8 @@ INSTANTIATE_TEST_SUITE_P(
              AArch64::AEK_SVE2,    AArch64::AEK_SVE2BITPERM, AArch64::AEK_PAUTH,
              AArch64::AEK_FLAGM,   AArch64::AEK_SB,          AArch64::AEK_I8MM,
              AArch64::AEK_BF16,    AArch64::AEK_JSCVT,       AArch64::AEK_FCMA,
-             AArch64::AEK_PERFMON, AArch64::AEK_ETE}),
+             AArch64::AEK_PERFMON, AArch64::AEK_ETE,         AArch64::AEK_CCIDX,
+             AArch64::AEK_SSBS}),
         AArch64CPUTestParams("cortex-a715", "armv9-a",
                              {AArch64::AEK_CRC,     AArch64::AEK_FP,
                               AArch64::AEK_BF16,    AArch64::AEK_SIMD,
@@ -1271,7 +1273,8 @@ INSTANTIATE_TEST_SUITE_P(
                               AArch64::AEK_FP16FML, AArch64::AEK_FP16,
                               AArch64::AEK_FLAGM,   AArch64::AEK_JSCVT,
                               AArch64::AEK_FCMA,    AArch64::AEK_PERFMON,
-                              AArch64::AEK_ETE,     AArch64::AEK_TRBE}),
+                              AArch64::AEK_ETE,     AArch64::AEK_TRBE,
+                              AArch64::AEK_CCIDX}),
         AArch64CPUTestParams("cortex-a720", "armv9.2-a",
                              {AArch64::AEK_BF16,        AArch64::AEK_I8MM,
                               AArch64::AEK_SVE,         AArch64::AEK_SVE2,
@@ -1287,7 +1290,7 @@ INSTANTIATE_TEST_SUITE_P(
                               AArch64::AEK_PROFILE,     AArch64::AEK_JSCVT,
                               AArch64::AEK_FCMA,        AArch64::AEK_PERFMON,
                               AArch64::AEK_ETE,         AArch64::AEK_SPE_EEF,
-                              AArch64::AEK_TRBE}),
+                              AArch64::AEK_TRBE,        AArch64::AEK_CCIDX}),
         AArch64CPUTestParams("cortex-a720ae", "armv9.2-a",
                              {AArch64::AEK_BF16,        AArch64::AEK_I8MM,
                               AArch64::AEK_SVE,         AArch64::AEK_SVE2,
@@ -1303,7 +1306,7 @@ INSTANTIATE_TEST_SUITE_P(
                               AArch64::AEK_PROFILE,     AArch64::AEK_JSCVT,
                               AArch64::AEK_FCMA,        AArch64::AEK_PERFMON,
                               AArch64::AEK_ETE,         AArch64::AEK_SPE_EEF,
-                              AArch64::AEK_TRBE}),
+                              AArch64::AEK_TRBE,        AArch64::AEK_CCIDX}),
         AArch64CPUTestParams("cortex-a725", "armv9.2-a",
                              {AArch64::AEK_BF16,        AArch64::AEK_I8MM,
                               AArch64::AEK_SVE,         AArch64::AEK_SVE2,
@@ -1318,7 +1321,8 @@ INSTANTIATE_TEST_SUITE_P(
                               AArch64::AEK_PERFMON,     AArch64::AEK_PREDRES,
                               AArch64::AEK_PROFILE,     AArch64::AEK_JSCVT,
                               AArch64::AEK_FCMA,        AArch64::AEK_ETE,
-                              AArch64::AEK_SPE_EEF,     AArch64::AEK_TRBE}),
+                              AArch64::AEK_SPE_EEF,     AArch64::AEK_TRBE,
+                              AArch64::AEK_CCIDX}),
         AArch64CPUTestParams(
             "neoverse-v1", "armv8.4-a",
             {AArch64::AEK_RAS,     AArch64::AEK_SVE,     AArch64::AEK_SSBS,
@@ -1329,7 +1333,8 @@ INSTANTIATE_TEST_SUITE_P(
              AArch64::AEK_SM4,     AArch64::AEK_FP16,    AArch64::AEK_BF16,
              AArch64::AEK_PROFILE, AArch64::AEK_RAND,    AArch64::AEK_FP16FML,
              AArch64::AEK_I8MM,    AArch64::AEK_JSCVT,   AArch64::AEK_FCMA,
-             AArch64::AEK_PAUTH,   AArch64::AEK_PERFMON, AArch64::AEK_CCDP}),
+             AArch64::AEK_PAUTH,   AArch64::AEK_PERFMON, AArch64::AEK_CCDP,
+             AArch64::AEK_CCIDX}),
         AArch64CPUTestParams("neoverse-v2", "armv9-a",
                              {AArch64::AEK_RAS,         AArch64::AEK_SVE,
                               AArch64::AEK_SSBS,        AArch64::AEK_RCPC,
@@ -1343,7 +1348,7 @@ INSTANTIATE_TEST_SUITE_P(
                               AArch64::AEK_SVE2BITPERM, AArch64::AEK_RAND,
                               AArch64::AEK_JSCVT,       AArch64::AEK_FCMA,
                               AArch64::AEK_PAUTH,       AArch64::AEK_PERFMON,
-                              AArch64::AEK_ETE}),
+                              AArch64::AEK_ETE,         AArch64::AEK_CCIDX}),
         AArch64CPUTestParams("neoverse-v3", "armv9.2-a",
                              {AArch64::AEK_BF16,        AArch64::AEK_I8MM,
                               AArch64::AEK_SVE,         AArch64::AEK_SVE2,
@@ -1361,7 +1366,7 @@ INSTANTIATE_TEST_SUITE_P(
                               AArch64::AEK_PROFILE,     AArch64::AEK_JSCVT,
                               AArch64::AEK_FCMA,        AArch64::AEK_PERFMON,
                               AArch64::AEK_ETE,         AArch64::AEK_SPE_EEF,
-                              AArch64::AEK_RME}),
+                              AArch64::AEK_RME,         AArch64::AEK_CCIDX}),
         AArch64CPUTestParams("neoverse-v3ae", "armv9.2-a",
                              {AArch64::AEK_BF16,        AArch64::AEK_I8MM,
                               AArch64::AEK_SVE,         AArch64::AEK_SVE2,
@@ -1379,7 +1384,7 @@ INSTANTIATE_TEST_SUITE_P(
                               AArch64::AEK_PROFILE,     AArch64::AEK_JSCVT,
                               AArch64::AEK_FCMA,        AArch64::AEK_PERFMON,
                               AArch64::AEK_ETE,         AArch64::AEK_SPE_EEF,
-                              AArch64::AEK_RME}),
+                              AArch64::AEK_RME,         AArch64::AEK_CCIDX}),
         AArch64CPUTestParams(
             "cortex-r82", "armv8-r",
             {AArch64::AEK_CRC, AArch64::AEK_RDM, AArch64::AEK_SSBS,
@@ -1427,7 +1432,7 @@ INSTANTIATE_TEST_SUITE_P(
                               AArch64::AEK_FP16FML,     AArch64::AEK_FLAGM,
                               AArch64::AEK_JSCVT,       AArch64::AEK_FCMA,
                               AArch64::AEK_PERFMON,     AArch64::AEK_AM,
-                              AArch64::AEK_ETE}),
+                              AArch64::AEK_ETE,         AArch64::AEK_CCIDX}),
         AArch64CPUTestParams("cortex-x3", "armv9-a",
                              {AArch64::AEK_CRC,     AArch64::AEK_FP,
                               AArch64::AEK_BF16,    AArch64::AEK_SIMD,
@@ -1442,7 +1447,8 @@ INSTANTIATE_TEST_SUITE_P(
                               AArch64::AEK_PREDRES, AArch64::AEK_FLAGM,
                               AArch64::AEK_SSBS,    AArch64::AEK_JSCVT,
                               AArch64::AEK_FCMA,    AArch64::AEK_PERFMON,
-                              AArch64::AEK_ETE,     AArch64::AEK_TRBE}),
+                              AArch64::AEK_ETE,     AArch64::AEK_TRBE,
+                              AArch64::AEK_CCIDX}),
         AArch64CPUTestParams("cortex-x4", "armv9.2-a",
                              {AArch64::AEK_BF16,        AArch64::AEK_I8MM,
                               AArch64::AEK_SVE,         AArch64::AEK_SVE2,
@@ -1458,7 +1464,7 @@ INSTANTIATE_TEST_SUITE_P(
                               AArch64::AEK_PROFILE,     AArch64::AEK_JSCVT,
                               AArch64::AEK_FCMA,        AArch64::AEK_PERFMON,
                               AArch64::AEK_ETE,         AArch64::AEK_SPE_EEF,
-                              AArch64::AEK_TRBE}),
+                              AArch64::AEK_TRBE,        AArch64::AEK_CCIDX}),
         AArch64CPUTestParams("cortex-x925", "armv9.2-a",
                              {AArch64::AEK_BF16,        AArch64::AEK_I8MM,
                               AArch64::AEK_SVE,         AArch64::AEK_SVE2,
@@ -1473,7 +1479,8 @@ INSTANTIATE_TEST_SUITE_P(
                               AArch64::AEK_PERFMON,     AArch64::AEK_PREDRES,
                               AArch64::AEK_PROFILE,     AArch64::AEK_JSCVT,
                               AArch64::AEK_FCMA,        AArch64::AEK_ETE,
-                              AArch64::AEK_SPE_EEF,     AArch64::AEK_TRBE}),
+                              AArch64::AEK_SPE_EEF,     AArch64::AEK_TRBE,
+                              AArch64::AEK_CCIDX}),
         AArch64CPUTestParams("cyclone", "armv8-a",
                              {AArch64::AEK_AES, AArch64::AEK_SHA2,
                               AArch64::AEK_FP, AArch64::AEK_SIMD,
@@ -1591,7 +1598,8 @@ INSTANTIATE_TEST_SUITE_P(
              AArch64::AEK_RCPC,    AArch64::AEK_DOTPROD, AArch64::AEK_FP16,
              AArch64::AEK_FP16FML, AArch64::AEK_SHA3,    AArch64::AEK_BF16,
              AArch64::AEK_I8MM,    AArch64::AEK_JSCVT,   AArch64::AEK_FCMA,
-             AArch64::AEK_PAUTH,   AArch64::AEK_FPAC,    AArch64::AEK_PERFMON}),
+             AArch64::AEK_PAUTH,   AArch64::AEK_FPAC,    AArch64::AEK_PERFMON,
+             AArch64::AEK_SSBS}),
         AArch64CPUTestParams(
             "apple-m2", "armv8.6-a",
             {AArch64::AEK_CRC,     AArch64::AEK_AES,     AArch64::AEK_SHA2,
@@ -1600,7 +1608,8 @@ INSTANTIATE_TEST_SUITE_P(
              AArch64::AEK_RCPC,    AArch64::AEK_DOTPROD, AArch64::AEK_FP16,
              AArch64::AEK_FP16FML, AArch64::AEK_SHA3,    AArch64::AEK_BF16,
              AArch64::AEK_I8MM,    AArch64::AEK_JSCVT,   AArch64::AEK_FCMA,
-             AArch64::AEK_PAUTH,   AArch64::AEK_FPAC,    AArch64::AEK_PERFMON}),
+             AArch64::AEK_PAUTH,   AArch64::AEK_FPAC,    AArch64::AEK_PERFMON,
+             AArch64::AEK_SSBS}),
         AArch64CPUTestParams(
             "apple-a16", "armv8.6-a",
             {AArch64::AEK_CRC,     AArch64::AEK_AES,     AArch64::AEK_SHA2,
@@ -1610,7 +1619,7 @@ INSTANTIATE_TEST_SUITE_P(
              AArch64::AEK_FP16FML, AArch64::AEK_SHA3,    AArch64::AEK_BF16,
              AArch64::AEK_I8MM,    AArch64::AEK_JSCVT,   AArch64::AEK_FCMA,
              AArch64::AEK_PAUTH,   AArch64::AEK_FPAC,    AArch64::AEK_PERFMON,
-             AArch64::AEK_HCX}),
+             AArch64::AEK_HCX,     AArch64::AEK_SSBS}),
         AArch64CPUTestParams(
             "apple-m3", "armv8.6-a",
             {AArch64::AEK_CRC,     AArch64::AEK_AES,     AArch64::AEK_SHA2,
@@ -1620,7 +1629,7 @@ INSTANTIATE_TEST_SUITE_P(
              AArch64::AEK_FP16FML, AArch64::AEK_SHA3,    AArch64::AEK_BF16,
              AArch64::AEK_I8MM,    AArch64::AEK_JSCVT,   AArch64::AEK_FCMA,
              AArch64::AEK_PAUTH,   AArch64::AEK_FPAC,    AArch64::AEK_PERFMON,
-             AArch64::AEK_HCX}),
+             AArch64::AEK_HCX,     AArch64::AEK_SSBS}),
         AArch64CPUTestParams(
             "apple-a17", "armv8.6-a",
             {AArch64::AEK_CRC,     AArch64::AEK_AES,     AArch64::AEK_SHA2,
@@ -1630,7 +1639,7 @@ INSTANTIATE_TEST_SUITE_P(
              AArch64::AEK_FP16FML, AArch64::AEK_SHA3,    AArch64::AEK_BF16,
              AArch64::AEK_I8MM,    AArch64::AEK_JSCVT,   AArch64::AEK_FCMA,
              AArch64::AEK_PAUTH,   AArch64::AEK_FPAC,    AArch64::AEK_PERFMON,
-             AArch64::AEK_HCX}),
+             AArch64::AEK_HCX,     AArch64::AEK_SSBS}),
         AArch64CPUTestParams("apple-m4", "armv9.2-a",
                              {AArch64::AEK_CRC,       AArch64::AEK_AES,
                               AArch64::AEK_SHA2,      AArch64::AEK_SHA3,
@@ -1695,7 +1704,8 @@ INSTANTIATE_TEST_SUITE_P(
                               AArch64::AEK_I8MM,        AArch64::AEK_JSCVT,
                               AArch64::AEK_FCMA,        AArch64::AEK_PAUTH,
                               AArch64::AEK_FP16FML,     AArch64::AEK_PERFMON,
-                              AArch64::AEK_ETE,         AArch64::AEK_TRBE}),
+                              AArch64::AEK_ETE,         AArch64::AEK_TRBE,
+                              AArch64::AEK_CCIDX}),
         AArch64CPUTestParams("neoverse-n3", "armv9.2-a",
                              {AArch64::AEK_BF16,    AArch64::AEK_I8MM,
                               AArch64::AEK_SVE,     AArch64::AEK_SVE2,
@@ -1711,7 +1721,8 @@ INSTANTIATE_TEST_SUITE_P(
                               AArch64::AEK_RAND,    AArch64::AEK_SVE2BITPERM,
                               AArch64::AEK_FP16FML, AArch64::AEK_PROFILE,
                               AArch64::AEK_JSCVT,   AArch64::AEK_PERFMON,
-                              AArch64::AEK_ETE,     AArch64::AEK_SPE_EEF}),
+                              AArch64::AEK_ETE,     AArch64::AEK_SPE_EEF,
+                              AArch64::AEK_CCIDX}),
         AArch64CPUTestParams(
             "ampere1", "armv8.6-a",
             {AArch64::AEK_CRC,  AArch64::AEK_FP,    AArch64::AEK_FP16,
@@ -1720,17 +1731,18 @@ INSTANTIATE_TEST_SUITE_P(
              AArch64::AEK_SHA3, AArch64::AEK_BF16,  AArch64::AEK_SHA2,
              AArch64::AEK_AES,  AArch64::AEK_I8MM,  AArch64::AEK_SSBS,
              AArch64::AEK_SB,   AArch64::AEK_RAND,  AArch64::AEK_JSCVT,
-             AArch64::AEK_FCMA, AArch64::AEK_PAUTH, AArch64::AEK_PERFMON}),
+             AArch64::AEK_FCMA, AArch64::AEK_PAUTH, AArch64::AEK_PERFMON,
+             AArch64::AEK_CCIDX}),
         AArch64CPUTestParams(
             "ampere1a", "armv8.6-a",
-            {AArch64::AEK_CRC,   AArch64::AEK_FP,     AArch64::AEK_FP16,
-             AArch64::AEK_SIMD,  AArch64::AEK_RAS,    AArch64::AEK_LSE,
-             AArch64::AEK_RDM,   AArch64::AEK_RCPC,   AArch64::AEK_DOTPROD,
-             AArch64::AEK_SM4,   AArch64::AEK_SHA3,   AArch64::AEK_BF16,
-             AArch64::AEK_SHA2,  AArch64::AEK_AES,    AArch64::AEK_I8MM,
-             AArch64::AEK_SSBS,  AArch64::AEK_SB,     AArch64::AEK_RAND,
-             AArch64::AEK_MTE,   AArch64::AEK_JSCVT,  AArch64::AEK_FCMA,
-             AArch64::AEK_PAUTH, AArch64::AEK_PERFMON}),
+            {AArch64::AEK_CRC,   AArch64::AEK_FP,      AArch64::AEK_FP16,
+             AArch64::AEK_SIMD,  AArch64::AEK_RAS,     AArch64::AEK_LSE,
+             AArch64::AEK_RDM,   AArch64::AEK_RCPC,    AArch64::AEK_DOTPROD,
+             AArch64::AEK_SM4,   AArch64::AEK_SHA3,    AArch64::AEK_BF16,
+             AArch64::AEK_SHA2,  AArch64::AEK_AES,     AArch64::AEK_I8MM,
+             AArch64::AEK_SSBS,  AArch64::AEK_SB,      AArch64::AEK_RAND,
+             AArch64::AEK_MTE,   AArch64::AEK_JSCVT,   AArch64::AEK_FCMA,
+             AArch64::AEK_PAUTH, AArch64::AEK_PERFMON, AArch64::AEK_CCIDX}),
         AArch64CPUTestParams(
             "ampere1b", "armv8.7-a",
             {AArch64::AEK_CRC,   AArch64::AEK_FP,    AArch64::AEK_FP16,
@@ -1741,7 +1753,7 @@ INSTANTIATE_TEST_SUITE_P(
              AArch64::AEK_SSBS,  AArch64::AEK_SB,    AArch64::AEK_RAND,
              AArch64::AEK_MTE,   AArch64::AEK_JSCVT, AArch64::AEK_FCMA,
              AArch64::AEK_PAUTH, AArch64::AEK_CSSC,  AArch64::AEK_PERFMON,
-             AArch64::AEK_WFXT}),
+             AArch64::AEK_WFXT,  AArch64::AEK_CCIDX}),
         AArch64CPUTestParams(
             "neoverse-512tvb", "armv8.4-a",
             {AArch64::AEK_RAS,     AArch64::AEK_SVE,     AArch64::AEK_SSBS,
@@ -1752,7 +1764,8 @@ INSTANTIATE_TEST_SUITE_P(
              AArch64::AEK_SM4,     AArch64::AEK_FP16,    AArch64::AEK_BF16,
              AArch64::AEK_PROFILE, AArch64::AEK_RAND,    AArch64::AEK_FP16FML,
              AArch64::AEK_I8MM,    AArch64::AEK_JSCVT,   AArch64::AEK_FCMA,
-             AArch64::AEK_PAUTH,   AArch64::AEK_PERFMON, AArch64::AEK_CCDP}),
+             AArch64::AEK_PAUTH,   AArch64::AEK_PERFMON, AArch64::AEK_CCDP,
+             AArch64::AEK_CCIDX}),
         AArch64CPUTestParams("thunderx2t99", "armv8.1-a",
                              {AArch64::AEK_CRC, AArch64::AEK_AES,
                               AArch64::AEK_SHA2, AArch64::AEK_LSE,
@@ -1765,7 +1778,7 @@ INSTANTIATE_TEST_SUITE_P(
                               AArch64::AEK_SIMD, AArch64::AEK_RAS,
                               AArch64::AEK_RCPC, AArch64::AEK_JSCVT,
                               AArch64::AEK_FCMA, AArch64::AEK_PAUTH,
-                              AArch64::AEK_PERFMON}),
+                              AArch64::AEK_PERFMON, AArch64::AEK_CCIDX}),
         AArch64CPUTestParams("thunderx", "armv8-a",
                              {AArch64::AEK_CRC, AArch64::AEK_AES,
                               AArch64::AEK_SHA2, AArch64::AEK_SIMD,
@@ -1808,7 +1821,7 @@ INSTANTIATE_TEST_SUITE_P(
             {AArch64::AEK_AES, AArch64::AEK_FP, AArch64::AEK_SIMD,
              AArch64::AEK_PERFMON, AArch64::AEK_SHA2, AArch64::AEK_PROFILE,
              AArch64::AEK_CRC, AArch64::AEK_LSE, AArch64::AEK_RDM,
-             AArch64::AEK_RAS, AArch64::AEK_RCPC}),
+             AArch64::AEK_RAS, AArch64::AEK_RCPC, AArch64::AEK_CCIDX}),
         AArch64CPUTestParams(
             "oryon-1", "armv8.6-a",
             {AArch64::AEK_CRC,     AArch64::AEK_FP,      AArch64::AEK_PAUTH,
@@ -1817,7 +1830,8 @@ INSTANTIATE_TEST_SUITE_P(
              AArch64::AEK_RCPC,    AArch64::AEK_DOTPROD, AArch64::AEK_SM4,
              AArch64::AEK_SHA3,    AArch64::AEK_BF16,    AArch64::AEK_SHA2,
              AArch64::AEK_AES,     AArch64::AEK_I8MM,    AArch64::AEK_RAND,
-             AArch64::AEK_PROFILE, AArch64::AEK_PERFMON})),
+             AArch64::AEK_PROFILE, AArch64::AEK_PERFMON, AArch64::AEK_CCIDX,
+             AArch64::AEK_SSBS})),
 
     AArch64CPUTestParams::PrintToStringParamName);
 
@@ -2248,9 +2262,9 @@ TEST(TargetParserTest, AArch64ArchExtFeature) {
 }
 
 TEST(TargetParserTest, AArch64PrintSupportedExtensions) {
-  std::string expected =
-      "All available -march extensions for AArch64\n\n"
-      "    Name                Architecture Feature(s)                                Description\n";
+  std::string expected = "All available -march extensions for AArch64\n\n"
+                         "    Name                Architecture Feature(s)      "
+                         "                          Description\n";
 
   outs().flush();
   testing::internal::CaptureStdout();
diff --git a/llvm/utils/TableGen/ARMTargetDefEmitter.cpp b/llvm/utils/TableGen/ARMTargetDefEmitter.cpp
index a4b25025b3c618a..71ca331461c00c8 100644
--- a/llvm/utils/TableGen/ARMTargetDefEmitter.cpp
+++ b/llvm/utils/TableGen/ARMTargetDefEmitter.cpp
@@ -19,10 +19,38 @@
 #include "llvm/TableGen/Record.h"
 #include "llvm/TableGen/TableGenBackend.h"
 #include <cstdint>
+#include <set>
 #include <string>
 
 using namespace llvm;
 
+/// Collect the full set of implied features for a SubtargetFeature.
+static void CollectImpliedFeatures(std::set<Record *> &SeenFeats, Record *Rec) {
+  assert(Rec->isSubClassOf("SubtargetFeature") &&
+         "Rec is not a SubtargetFeature");
+
+  SeenFeats.insert(Rec);
+  for (Record *Implied : Rec->getValueAsListOfDefs("Implies"))
+    CollectImpliedFeatures(SeenFeats, Implied);
+}
+
+static void CheckFeatureTree(Record *Root) {
+  std::set<Record *> SeenFeats;
+  CollectImpliedFeatures(SeenFeats, Root);
+
+  // Check that each of the mandatory (implied) features which is an
+  // ExtensionWithMArch is also enabled by default.
+  auto DefaultExtsVec = Root->getValueAsListOfDefs("DefaultExts");
+  std::set<Record *> DefaultExts{DefaultExtsVec.begin(), DefaultExtsVec.end()};
+  for (auto *Feat : SeenFeats) {
+    if (Feat->isSubClassOf("ExtensionWithMArch") && !DefaultExts.count(Feat))
+      PrintFatalError(Root->getLoc(),
+                      "ExtensionWithMArch " + Feat->getName() +
+                          " is implied (mandatory) as a SubtargetFeature, but "
+                          "is not present in DefaultExts");
+  }
+}
+
 static void EmitARMTargetDef(RecordKeeper &RK, raw_ostream &OS) {
   OS << "// Autogenerated by ARMTargetDefEmitter.cpp\n\n";
 
@@ -283,9 +311,7 @@ static void EmitARMTargetDef(RecordKeeper &RK, raw_ostream &OS) {
     auto Profile = Arch->getValueAsString("Profile");
     auto ArchInfo = ArchInfoName(Major, Minor, Profile);
 
-    // The apple-latest alias is backend only, do not expose it to -mcpu.
-    if (Name == "apple-latest")
-      continue;
+    CheckFeatureTree(Arch);
 
     OS << "  {\n"
        << "    \"" << Name << "\",\n"

From 76161451f5ca837b25769c7d142f8271b45d9b28 Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Sat, 17 Aug 2024 09:49:05 -0500
Subject: [PATCH 269/441] [libc] Fix type signature for strlcpy and strlcat
 (#104643)

Summary:
These should not be const on the destination pointer.
---
 libc/newhdrgen/yaml/string.yaml | 4 ++--
 libc/spec/bsd_ext.td            | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/libc/newhdrgen/yaml/string.yaml b/libc/newhdrgen/yaml/string.yaml
index 387ea9ec4e1422a..1d6e64bfb9cf60a 100644
--- a/libc/newhdrgen/yaml/string.yaml
+++ b/libc/newhdrgen/yaml/string.yaml
@@ -181,7 +181,7 @@ functions:
       - BSDExtensions
     return_type: size_t
     arguments:
-      - type: const char *__restrict
+      - type: char *__restrict
       - type: const char *__restrict
       - type: size_t
   - name: strlcpy
@@ -189,7 +189,7 @@ functions:
       - BSDExtensions
     return_type: size_t
     arguments:
-      - type: const char *__restrict
+      - type: char *__restrict
       - type: const char *__restrict
       - type: size_t
   - name: strlen
diff --git a/libc/spec/bsd_ext.td b/libc/spec/bsd_ext.td
index 4d33313521735e7..2b91324e36db9aa 100644
--- a/libc/spec/bsd_ext.td
+++ b/libc/spec/bsd_ext.td
@@ -20,12 +20,12 @@ def BsdExtensions : StandardSpec<"BSDExtensions"> {
         FunctionSpec<
             "strlcat",
             RetValSpec<SizeTType>,
-            [ArgSpec<ConstCharRestrictedPtr>, ArgSpec<ConstCharRestrictedPtr>, ArgSpec<SizeTType>]
+            [ArgSpec<CharRestrictedPtr>, ArgSpec<ConstCharRestrictedPtr>, ArgSpec<SizeTType>]
         >,
         FunctionSpec<
             "strlcpy",
             RetValSpec<SizeTType>,
-            [ArgSpec<ConstCharRestrictedPtr>, ArgSpec<ConstCharRestrictedPtr>, ArgSpec<SizeTType>]
+            [ArgSpec<CharRestrictedPtr>, ArgSpec<ConstCharRestrictedPtr>, ArgSpec<SizeTType>]
         >,
         FunctionSpec<
             "strsep",

From 16929219b056fdbf9e0c93092904f2c5cac8a66f Mon Sep 17 00:00:00 2001
From: Changpeng Fang <changpeng.fang@amd.com>
Date: Sat, 17 Aug 2024 11:22:47 -0700
Subject: [PATCH 270/441] AMDGPU: Add tonearest and towardzero roundings for
 intrinsic llvm.fptrunc.round (#104486)

This work simplifies and generalizes the instruction definition for
intrinsic llvm.fptrunc.round. We no longer name the instruction with the
rounding mode. Instead, we introduce an immediate operand for the
rounding mode for the pseudo instruction. This immediate will be used to
set up the hardware mode register at the time the real instruction is
generated. We name the pseudo instruction as FPTRUNC_ROUND_F16_F32 (for
f32 -> f16), which is easy to generalize for other types.

"round.towardzero" and "round.tonearest" are added for f32 -> f16
truncating, in addition to the existing "round.upward" and
"round.downward". Other rounding modes are not supported by hardware at
this moment.
---
 llvm/lib/Target/AMDGPU/AMDGPUGISel.td         |  3 +-
 llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp |  3 +-
 llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h   |  3 +-
 .../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 28 ++++++---
 .../Target/AMDGPU/AMDGPURegisterBankInfo.cpp  |  3 +-
 llvm/lib/Target/AMDGPU/SIISelLowering.cpp     | 44 +++++++------
 llvm/lib/Target/AMDGPU/SIISelLowering.h       |  1 +
 llvm/lib/Target/AMDGPU/SIInstrInfo.td         | 10 ++-
 llvm/lib/Target/AMDGPU/SIInstructions.td      | 28 +++------
 llvm/lib/Target/AMDGPU/SIModeRegister.cpp     | 29 +++------
 .../CodeGen/AMDGPU/fail.llvm.fptrunc.round.ll |  9 +--
 .../test/CodeGen/AMDGPU/llvm.fptrunc.round.ll | 63 +++++++++++++++++++
 .../CodeGen/AMDGPU/mode-register-fptrunc.mir  | 52 ++++++++++++++-
 13 files changed, 185 insertions(+), 91 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUGISel.td b/llvm/lib/Target/AMDGPU/AMDGPUGISel.td
index 825c1de6c91f12b..8bee84b8a87f276 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUGISel.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUGISel.td
@@ -297,8 +297,7 @@ def : GINodeEquiv<G_AMDGPU_S_BUFFER_LOAD_UBYTE, SIsbuffer_load_ubyte>;
 def : GINodeEquiv<G_AMDGPU_S_BUFFER_LOAD_SSHORT, SIsbuffer_load_short>;
 def : GINodeEquiv<G_AMDGPU_S_BUFFER_LOAD_USHORT, SIsbuffer_load_ushort>;
 
-def : GINodeEquiv<G_FPTRUNC_ROUND_UPWARD, SIfptrunc_round_upward>;
-def : GINodeEquiv<G_FPTRUNC_ROUND_DOWNWARD, SIfptrunc_round_downward>;
+def : GINodeEquiv<G_FPTRUNC_ROUND, SIfptrunc_round>;
 
 class GISelSop2Pat <
   SDPatternOperator node,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index 8c8950957dd8197..e57c8f8b7b4835c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -5500,8 +5500,7 @@ const char* AMDGPUTargetLowering::getTargetNodeName(unsigned Opcode) const {
   NODE_NAME_CASE(CONST_DATA_PTR)
   NODE_NAME_CASE(PC_ADD_REL_OFFSET)
   NODE_NAME_CASE(LDS)
-  NODE_NAME_CASE(FPTRUNC_ROUND_UPWARD)
-  NODE_NAME_CASE(FPTRUNC_ROUND_DOWNWARD)
+  NODE_NAME_CASE(FPTRUNC_ROUND)
   NODE_NAME_CASE(DUMMY_CHAIN)
   case AMDGPUISD::FIRST_MEM_OPCODE_NUMBER: break;
   NODE_NAME_CASE(LOAD_D16_HI)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
index 1a5244f7ec809b8..59f640ea99de3ea 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
@@ -553,8 +553,7 @@ enum NodeType : unsigned {
   CONST_DATA_PTR,
   PC_ADD_REL_OFFSET,
   LDS,
-  FPTRUNC_ROUND_UPWARD,
-  FPTRUNC_ROUND_DOWNWARD,
+  FPTRUNC_ROUND,
 
   DUMMY_CHAIN,
   FIRST_MEM_OPCODE_NUMBER = ISD::FIRST_TARGET_MEMORY_OPCODE,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index c6c4b8f93064711..65198b4285a1e44 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -7090,22 +7090,30 @@ bool AMDGPULegalizerInfo::legalizeBVHIntrinsic(MachineInstr &MI,
 
 bool AMDGPULegalizerInfo::legalizeFPTruncRound(MachineInstr &MI,
                                                MachineIRBuilder &B) const {
-  unsigned Opc;
-  int RoundMode = MI.getOperand(2).getImm();
+  MachineRegisterInfo &MRI = *B.getMRI();
+  Register Src = MI.getOperand(1).getReg();
+  if (MRI.getType(Src) != LLT::scalar(32))
+    return false;
 
-  if (RoundMode == (int)RoundingMode::TowardPositive)
-    Opc = AMDGPU::G_FPTRUNC_ROUND_UPWARD;
-  else if (RoundMode == (int)RoundingMode::TowardNegative)
-    Opc = AMDGPU::G_FPTRUNC_ROUND_DOWNWARD;
-  else
+  // Only support towardzero, tonearest, upward and downward.
+  int RoundMode = MI.getOperand(2).getImm();
+  if (RoundMode != (int)RoundingMode::TowardZero &&
+      RoundMode != (int)RoundingMode::NearestTiesToEven &&
+      RoundMode != (int)RoundingMode::TowardPositive &&
+      RoundMode != (int)RoundingMode::TowardNegative)
     return false;
 
-  B.buildInstr(Opc)
+  // "round.towardzero" -> TowardZero 0        -> FP_ROUND_ROUND_TO_ZERO 3
+  // "round.tonearest"  -> NearestTiesToEven 1 -> FP_ROUND_ROUND_TO_NEAREST 0
+  // "round.upward"     -> TowardPositive 2    -> FP_ROUND_ROUND_TO_INF 1
+  // "round.downward    -> TowardNegative 3    -> FP_ROUND_ROUND_TO_NEGINF 2
+  unsigned HW_Mode = (RoundMode + 3) % 4;
+  B.buildInstr(AMDGPU::G_FPTRUNC_ROUND)
       .addDef(MI.getOperand(0).getReg())
-      .addUse(MI.getOperand(1).getReg());
+      .addUse(Src)
+      .addImm(HW_Mode);
 
   MI.eraseFromParent();
-
   return true;
 }
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
index 17067ddd93ff085..5add368c05646a1 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
@@ -5259,8 +5259,7 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
     OpdsMapping[0] = AMDGPU::getValueMapping(Bank, 1);
     break;
   }
-  case AMDGPU::G_FPTRUNC_ROUND_UPWARD:
-  case AMDGPU::G_FPTRUNC_ROUND_DOWNWARD:
+  case AMDGPU::G_FPTRUNC_ROUND:
     return getDefaultMappingVOP(MI);
   case AMDGPU::G_PREFETCH:
     OpdsMapping[0] = getSGPROpMapping(MI.getOperand(0).getReg(), MRI, *TRI);
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index d9387b2dda30ed6..09fe4ac33c3267b 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -5804,24 +5804,8 @@ SDValue SITargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
   case ISD::FP_ROUND:
   case ISD::STRICT_FP_ROUND:
     return lowerFP_ROUND(Op, DAG);
-  case ISD::FPTRUNC_ROUND: {
-    unsigned Opc;
-    SDLoc DL(Op);
-
-    if (Op.getOperand(0)->getValueType(0) != MVT::f32)
-      return SDValue();
-
-    // Get the rounding mode from the last operand
-    int RoundMode = Op.getConstantOperandVal(1);
-    if (RoundMode == (int)RoundingMode::TowardPositive)
-      Opc = AMDGPUISD::FPTRUNC_ROUND_UPWARD;
-    else if (RoundMode == (int)RoundingMode::TowardNegative)
-      Opc = AMDGPUISD::FPTRUNC_ROUND_DOWNWARD;
-    else
-      return SDValue();
-
-    return DAG.getNode(Opc, DL, Op.getNode()->getVTList(), Op->getOperand(0));
-  }
+  case ISD::FPTRUNC_ROUND:
+    return lowerFPTRUNC_ROUND(Op, DAG);
   case ISD::TRAP:
     return lowerTRAP(Op, DAG);
   case ISD::DEBUGTRAP:
@@ -6671,6 +6655,30 @@ SDValue SITargetLowering::getFPExtOrFPRound(SelectionDAG &DAG,
                 DAG.getTargetConstant(0, DL, MVT::i32));
 }
 
+SDValue SITargetLowering::lowerFPTRUNC_ROUND(SDValue Op,
+                                             SelectionDAG &DAG) const {
+  if (Op.getOperand(0)->getValueType(0) != MVT::f32)
+    return SDValue();
+
+  // Only support towardzero, tonearest, upward and downward.
+  int RoundMode = Op.getConstantOperandVal(1);
+  if (RoundMode != (int)RoundingMode::TowardZero &&
+      RoundMode != (int)RoundingMode::NearestTiesToEven &&
+      RoundMode != (int)RoundingMode::TowardPositive &&
+      RoundMode != (int)RoundingMode::TowardNegative)
+    return SDValue();
+
+  // "round.towardzero" -> TowardZero 0        -> FP_ROUND_ROUND_TO_ZERO 3
+  // "round.tonearest"  -> NearestTiesToEven 1 -> FP_ROUND_ROUND_TO_NEAREST 0
+  // "round.upward"     -> TowardPositive 2    -> FP_ROUND_ROUND_TO_INF 1
+  // "round.downward    -> TowardNegative 3    -> FP_ROUND_ROUND_TO_NEGINF 2
+  unsigned HW_Mode = (RoundMode + 3) % 4;
+  SDLoc DL(Op);
+  SDValue RoundFlag = DAG.getTargetConstant(HW_Mode, DL, MVT::i32);
+  return DAG.getNode(AMDGPUISD::FPTRUNC_ROUND, DL, Op.getNode()->getVTList(),
+                     Op->getOperand(0), RoundFlag);
+}
+
 SDValue SITargetLowering::lowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const {
   assert(Op.getValueType() == MVT::f16 &&
          "Do not know how to custom lower FP_ROUND for non-f16 type");
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h
index 1f198a92c0fa6a3..eed4b3e79cdeee8 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h
@@ -145,6 +145,7 @@ class SITargetLowering final : public AMDGPUTargetLowering {
 
   /// Custom lowering for ISD::FP_ROUND for MVT::f16.
   SDValue lowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const;
+  SDValue lowerFPTRUNC_ROUND(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerFMINNUM_FMAXNUM(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerFLDEXP(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerMUL(SDValue Op, SelectionDAG &DAG) const;
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.td b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
index e99b43afd1c3a28..85281713e22b1f8 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.td
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
@@ -304,13 +304,11 @@ def SIdenorm_mode : SDNode<"AMDGPUISD::DENORM_MODE",
   [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue]
 >;
 
-def SIfptrunc_round_upward : SDNode<"AMDGPUISD::FPTRUNC_ROUND_UPWARD",
-  SDTFPRoundOp
->;
+def SDTFPRoundModeOp  : SDTypeProfile<1, 2, [
+  SDTCisFP<0>, SDTCisFP<1>, SDTCisInt<2>, SDTCisOpSmallerThanOp<0, 1>, SDTCisSameNumEltsAs<0, 1>
+]>;
 
-def SIfptrunc_round_downward : SDNode<"AMDGPUISD::FPTRUNC_ROUND_DOWNWARD",
-  SDTFPRoundOp
->;
+def SIfptrunc_round : SDNode<"AMDGPUISD::FPTRUNC_ROUND", SDTFPRoundModeOp>;
 
 //===----------------------------------------------------------------------===//
 // ValueType helpers
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index 5a139d1cf8d8255..e7831d00a3a4a89 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -225,20 +225,12 @@ def S_INVERSE_BALLOT_U64 : SPseudoInstSI<
 >;
 } // End usesCustomInserter = 1
 
-// Pseudo instructions used for @llvm.fptrunc.round upward
-// and @llvm.fptrunc.round downward.
-// These intrinsics will be legalized to G_FPTRUNC_ROUND_UPWARD
-// and G_FPTRUNC_ROUND_DOWNWARD before being lowered to
-// FPTRUNC_UPWARD_PSEUDO and FPTRUNC_DOWNWARD_PSEUDO.
-// The final codegen is done in the ModeRegister pass.
+// Pseudo instructions used for @llvm.fptrunc.round. The final codegen is done
+// in the ModeRegister pass.
 let Uses = [MODE, EXEC] in {
-def FPTRUNC_UPWARD_PSEUDO : VPseudoInstSI <(outs VGPR_32:$vdst),
-  (ins VGPR_32:$src0),
-  [(set f16:$vdst, (SIfptrunc_round_upward f32:$src0))]>;
-
-def FPTRUNC_DOWNWARD_PSEUDO : VPseudoInstSI <(outs VGPR_32:$vdst),
-  (ins VGPR_32:$src0),
-  [(set f16:$vdst, (SIfptrunc_round_downward f32:$src0))]>;
+def FPTRUNC_ROUND_F16_F32_PSEUDO : VPseudoInstSI <(outs VGPR_32:$vdst),
+  (ins VGPR_32:$src0, i32imm:$round),
+  [(set f16:$vdst, (SIfptrunc_round f32:$src0, i32:$round))]>;
 } // End Uses = [MODE, EXEC]
 
 // Invert the exec mask and overwrite the inactive lanes of dst with inactive,
@@ -4072,15 +4064,9 @@ def G_SI_CALL : AMDGPUGenericInstruction {
   let isConvergent = 1;
 }
 
-def G_FPTRUNC_ROUND_UPWARD : AMDGPUGenericInstruction {
-  let OutOperandList = (outs type0:$vdst);
-  let InOperandList = (ins type1:$src0);
-  let hasSideEffects = 0;
-}
-
-def G_FPTRUNC_ROUND_DOWNWARD : AMDGPUGenericInstruction {
+def G_FPTRUNC_ROUND : AMDGPUGenericInstruction {
   let OutOperandList = (outs type0:$vdst);
-  let InOperandList = (ins type1:$src0);
+  let InOperandList = (ins type1:$src0, untyped_imm_0:$round);
   let hasSideEffects = 0;
 }
 
diff --git a/llvm/lib/Target/AMDGPU/SIModeRegister.cpp b/llvm/lib/Target/AMDGPU/SIModeRegister.cpp
index e7f448233ca347d..a590c6560942cf4 100644
--- a/llvm/lib/Target/AMDGPU/SIModeRegister.cpp
+++ b/llvm/lib/Target/AMDGPU/SIModeRegister.cpp
@@ -163,17 +163,19 @@ FunctionPass *llvm::createSIModeRegisterPass() { return new SIModeRegister(); }
 // double precision setting.
 Status SIModeRegister::getInstructionMode(MachineInstr &MI,
                                           const SIInstrInfo *TII) {
+  unsigned Opcode = MI.getOpcode();
   if (TII->usesFPDPRounding(MI) ||
-      MI.getOpcode() == AMDGPU::FPTRUNC_UPWARD_PSEUDO ||
-      MI.getOpcode() == AMDGPU::FPTRUNC_DOWNWARD_PSEUDO) {
-    switch (MI.getOpcode()) {
+      Opcode == AMDGPU::FPTRUNC_ROUND_F16_F32_PSEUDO) {
+    switch (Opcode) {
     case AMDGPU::V_INTERP_P1LL_F16:
     case AMDGPU::V_INTERP_P1LV_F16:
     case AMDGPU::V_INTERP_P2_F16:
       // f16 interpolation instructions need double precision round to zero
       return Status(FP_ROUND_MODE_DP(3),
                     FP_ROUND_MODE_DP(FP_ROUND_ROUND_TO_ZERO));
-    case AMDGPU::FPTRUNC_UPWARD_PSEUDO: {
+    case AMDGPU::FPTRUNC_ROUND_F16_F32_PSEUDO: {
+      unsigned Mode = MI.getOperand(2).getImm();
+      MI.removeOperand(2);
       // Replacing the pseudo by a real instruction in place
       if (TII->getSubtarget().hasTrue16BitInsts()) {
         MachineBasicBlock &MBB = *MI.getParent();
@@ -188,24 +190,7 @@ Status SIModeRegister::getInstructionMode(MachineInstr &MI,
       } else
         MI.setDesc(TII->get(AMDGPU::V_CVT_F16_F32_e32));
       return Status(FP_ROUND_MODE_DP(3),
-                    FP_ROUND_MODE_DP(FP_ROUND_ROUND_TO_INF));
-    }
-    case AMDGPU::FPTRUNC_DOWNWARD_PSEUDO: {
-      // Replacing the pseudo by a real instruction in place
-      if (TII->getSubtarget().hasTrue16BitInsts()) {
-        MachineBasicBlock &MBB = *MI.getParent();
-        MachineInstrBuilder B(*MBB.getParent(), MI);
-        MI.setDesc(TII->get(AMDGPU::V_CVT_F16_F32_t16_e64));
-        MachineOperand Src0 = MI.getOperand(1);
-        MI.removeOperand(1);
-        B.addImm(0); // src0_modifiers
-        B.add(Src0); // re-add src0 operand
-        B.addImm(0); // clamp
-        B.addImm(0); // omod
-      } else
-        MI.setDesc(TII->get(AMDGPU::V_CVT_F16_F32_e32));
-      return Status(FP_ROUND_MODE_DP(3),
-                    FP_ROUND_MODE_DP(FP_ROUND_ROUND_TO_NEGINF));
+                    FP_ROUND_MODE_DP(Mode));
     }
     default:
       return DefaultStatus;
diff --git a/llvm/test/CodeGen/AMDGPU/fail.llvm.fptrunc.round.ll b/llvm/test/CodeGen/AMDGPU/fail.llvm.fptrunc.round.ll
index f312f3138c08d34..9fa3eb22a554a80 100644
--- a/llvm/test/CodeGen/AMDGPU/fail.llvm.fptrunc.round.ll
+++ b/llvm/test/CodeGen/AMDGPU/fail.llvm.fptrunc.round.ll
@@ -1,8 +1,9 @@
-; RUN: not --crash llc -mtriple=amdgcn -mcpu=gfx1030 -verify-machineinstrs -o /dev/null %s 2>&1 | FileCheck %s --ignore-case --check-prefix=FAIL
-; RUN: not --crash llc -global-isel -mtriple=amdgcn -mcpu=gfx1030 -verify-machineinstrs -o /dev/null %s 2>&1 | FileCheck %s --ignore-case --check-prefix=FAIL
+; RUN: not --crash llc -mtriple=amdgcn -mcpu=gfx1030 -o /dev/null %s 2>&1 | FileCheck %s --ignore-case --check-prefixes=SDAG-FAIL
+; RUN: not --crash llc -global-isel -mtriple=amdgcn -mcpu=gfx1030 -o /dev/null %s 2>&1 | FileCheck %s --ignore-case --check-prefix=GISEL-FAIL
 
-define amdgpu_gs void @test_fptrunc_round_legalization(double %a, i32 %data0, <4 x i32> %data1, ptr addrspace(1) %out) {
-; FAIL: LLVM ERROR: Cannot select
+define amdgpu_gs void @test_fptrunc_round_f64(double %a, ptr addrspace(1) %out) {
+; SDAG-FAIL: LLVM ERROR: Cannot select
+; GISEL-FAIL: unable to legalize instruction
   %res = call half @llvm.fptrunc.round.f64(double %a, metadata !"round.upward")
   store half %res, ptr addrspace(1) %out, align 4
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.fptrunc.round.ll b/llvm/test/CodeGen/AMDGPU/llvm.fptrunc.round.ll
index b8c16d2ed3b2f1f..71d0ee524bab73b 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.fptrunc.round.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.fptrunc.round.ll
@@ -3,6 +3,15 @@
 ; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1010 < %s | FileCheck -check-prefixes=CHECK,SDAG %s
 ; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1030 < %s | FileCheck -check-prefixes=CHECK,GISEL %s
 
+define amdgpu_gs half @v_fptrunc_round_f32_to_f16_tonearest(float %a) {
+; CHECK-LABEL: v_fptrunc_round_f32_to_f16_tonearest:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; CHECK-NEXT:    ; return to shader part epilog
+  %res = call half @llvm.fptrunc.round.f16.f32(float %a, metadata !"round.tonearest")
+  ret half %res
+}
+
 define amdgpu_gs half @v_fptrunc_round_f32_to_f16_upward(float %a) {
 ; CHECK-LABEL: v_fptrunc_round_f32_to_f16_upward:
 ; CHECK:       ; %bb.0:
@@ -23,6 +32,16 @@ define amdgpu_gs half @v_fptrunc_round_f32_to_f16_downward(float %a) {
   ret half %res
 }
 
+define amdgpu_gs half @v_fptrunc_round_f32_to_f16_towardzero(float %a) {
+; CHECK-LABEL: v_fptrunc_round_f32_to_f16_towardzero:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 3
+; CHECK-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; CHECK-NEXT:    ; return to shader part epilog
+  %res = call half @llvm.fptrunc.round.f16.f32(float %a, metadata !"round.towardzero")
+  ret half %res
+}
+
 define amdgpu_gs void @v_fptrunc_round_f32_to_f16_upward_multiple_calls(float %a, float %b, ptr addrspace(1) %out) {
 ; CHECK-LABEL: v_fptrunc_round_f32_to_f16_upward_multiple_calls:
 ; CHECK:       ; %bb.0:
@@ -45,6 +64,50 @@ define amdgpu_gs void @v_fptrunc_round_f32_to_f16_upward_multiple_calls(float %a
   ret void
 }
 
+define amdgpu_gs void @v_fptrunc_round_f32_to_f16_downward_multiple_calls(float %a, float %b, ptr addrspace(1) %out) {
+; CHECK-LABEL: v_fptrunc_round_f32_to_f16_downward_multiple_calls:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 1
+; CHECK-NEXT:    v_cvt_f16_f32_e32 v4, v0
+; CHECK-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 2
+; CHECK-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; CHECK-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; CHECK-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 3, 1), 0
+; CHECK-NEXT:    v_add_f16_e32 v0, v4, v0
+; CHECK-NEXT:    v_add_f16_e32 v0, v1, v0
+; CHECK-NEXT:    global_store_short v[2:3], v0, off
+; CHECK-NEXT:    s_endpgm
+  %res1 = call half @llvm.fptrunc.round.f16.f32(float %a, metadata !"round.upward")
+  %res2 = call half @llvm.fptrunc.round.f16.f32(float %a, metadata !"round.downward")
+  %res3 = call half @llvm.fptrunc.round.f16.f32(float %b, metadata !"round.downward")
+  %res4 = fadd half %res1, %res2
+  %res5 = fadd half %res3, %res4
+  store half %res5, ptr addrspace(1) %out, align 4
+  ret void
+}
+
+define amdgpu_gs void @v_fptrunc_round_f32_to_f16_towardzero_multiple_calls(float %a, float %b, ptr addrspace(1) %out) {
+; CHECK-LABEL: v_fptrunc_round_f32_to_f16_towardzero_multiple_calls:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 3
+; CHECK-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; CHECK-NEXT:    v_cvt_f16_f32_e32 v4, v1
+; CHECK-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 1
+; CHECK-NEXT:    v_cvt_f16_f32_e32 v1, v1
+; CHECK-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 0
+; CHECK-NEXT:    v_add_f16_e32 v0, v0, v4
+; CHECK-NEXT:    v_add_f16_e32 v0, v1, v0
+; CHECK-NEXT:    global_store_short v[2:3], v0, off
+; CHECK-NEXT:    s_endpgm
+  %res1 = call half @llvm.fptrunc.round.f16.f32(float %a, metadata !"round.towardzero")
+  %res2 = call half @llvm.fptrunc.round.f16.f32(float %b, metadata !"round.towardzero")
+  %res3 = call half @llvm.fptrunc.round.f16.f32(float %b, metadata !"round.upward")
+  %res4 = fadd half %res1, %res2
+  %res5 = fadd half %res3, %res4
+  store half %res5, ptr addrspace(1) %out, align 4
+  ret void
+}
+
 define amdgpu_gs i32 @s_fptrunc_round_f32_to_f16_upward(float inreg %a, ptr addrspace(1) %out) {
 ; CHECK-LABEL: s_fptrunc_round_f32_to_f16_upward:
 ; CHECK:       ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/mode-register-fptrunc.mir b/llvm/test/CodeGen/AMDGPU/mode-register-fptrunc.mir
index fda33f52bf06d2b..3706c535224438e 100644
--- a/llvm/test/CodeGen/AMDGPU/mode-register-fptrunc.mir
+++ b/llvm/test/CodeGen/AMDGPU/mode-register-fptrunc.mir
@@ -3,6 +3,29 @@
 # RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -run-pass si-mode-register  %s -o - | FileCheck %s --check-prefixes=CHECK
 # RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -run-pass si-mode-register  %s -o - | FileCheck %s --check-prefixes=GFX11
 
+---
+name: ftrunc_tonearest
+
+body: |
+  bb.0:
+    liveins: $sgpr0
+    ; CHECK-LABEL: name: ftrunc_tonearest
+    ; CHECK: liveins: $sgpr0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; CHECK-NEXT: $vgpr1 = V_CVT_F16_F32_e32 $vgpr0, implicit $mode, implicit $exec
+    ; CHECK-NEXT: S_ENDPGM 0
+    ;
+    ; GFX11-LABEL: name: ftrunc_tonearest
+    ; GFX11: liveins: $sgpr0
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GFX11-NEXT: $vgpr1 = V_CVT_F16_F32_t16_e64 0, $vgpr0, 0, 0, implicit $mode, implicit $exec
+    ; GFX11-NEXT: S_ENDPGM 0
+    $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    $vgpr1 = FPTRUNC_ROUND_F16_F32_PSEUDO $vgpr0, 0, implicit $mode, implicit $exec
+    S_ENDPGM 0
+...
 ---
 name: ftrunc_upward
 
@@ -25,7 +48,7 @@ body: |
     ; GFX11-NEXT: $vgpr1 = V_CVT_F16_F32_t16_e64 0, $vgpr0, 0, 0, implicit $mode, implicit $exec
     ; GFX11-NEXT: S_ENDPGM 0
     $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
-    $vgpr1 = FPTRUNC_UPWARD_PSEUDO $vgpr0, implicit $mode, implicit $exec
+    $vgpr1 = FPTRUNC_ROUND_F16_F32_PSEUDO $vgpr0, 1, implicit $mode, implicit $exec
     S_ENDPGM 0
 ...
 ---
@@ -50,6 +73,31 @@ body: |
     ; GFX11-NEXT: $vgpr0 = V_CVT_F16_F32_t16_e64 0, $vgpr1, 0, 0, implicit $mode, implicit $exec
     ; GFX11-NEXT: S_ENDPGM 0
     $vgpr1 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
-    $vgpr0 = FPTRUNC_DOWNWARD_PSEUDO $vgpr1, implicit $mode, implicit $exec
+    $vgpr0 = FPTRUNC_ROUND_F16_F32_PSEUDO $vgpr1, 2, implicit $mode, implicit $exec
+    S_ENDPGM 0
+...
+---
+name: ftrunc_towardzero
+
+body: |
+  bb.0:
+    liveins: $sgpr0
+    ; CHECK-LABEL: name: ftrunc_towardzero
+    ; CHECK: liveins: $sgpr0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; CHECK-NEXT: S_SETREG_IMM32_B32 3, 2177, implicit-def $mode, implicit $mode
+    ; CHECK-NEXT: $vgpr1 = V_CVT_F16_F32_e32 $vgpr0, implicit $mode, implicit $exec
+    ; CHECK-NEXT: S_ENDPGM 0
+    ;
+    ; GFX11-LABEL: name: ftrunc_towardzero
+    ; GFX11: liveins: $sgpr0
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    ; GFX11-NEXT: S_SETREG_IMM32_B32 3, 2177, implicit-def $mode, implicit $mode
+    ; GFX11-NEXT: $vgpr1 = V_CVT_F16_F32_t16_e64 0, $vgpr0, 0, 0, implicit $mode, implicit $exec
+    ; GFX11-NEXT: S_ENDPGM 0
+    $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+    $vgpr1 = FPTRUNC_ROUND_F16_F32_PSEUDO $vgpr0, 3, implicit $mode, implicit $exec
     S_ENDPGM 0
 ...

From 0b092dfee3c5d9d7e4bde636330ce39a656ee676 Mon Sep 17 00:00:00 2001
From: weiguozhi <57237827+weiguozhi@users.noreply.github.com>
Date: Sat, 17 Aug 2024 11:51:05 -0700
Subject: [PATCH 271/441] [X86] Don't save/restore fp around longjmp
 instructions (#102556)

Longjmp instructions can also modify fp register, it is expected
behavior. We should not save/restore fp around these instructions.
---
 llvm/lib/Target/X86/X86ISelLowering.cpp    |  2 +
 llvm/test/CodeGen/X86/clobber_frame_ptr.ll | 43 ++++++++++++++++++++++
 2 files changed, 45 insertions(+)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index e0face6b6a79042..2fa416f2e331c43 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -36208,6 +36208,7 @@ X86TargetLowering::emitEHSjLjLongJmp(MachineInstr &MI,
       MIB.add(MO);
   }
   MIB.setMemRefs(MMOs);
+  MIB.setMIFlag(MachineInstr::FrameDestroy);
 
   // Reload IP
   MIB = BuildMI(*thisMBB, MI, MIMD, TII->get(PtrLoadOpc), Tmp);
@@ -36233,6 +36234,7 @@ X86TargetLowering::emitEHSjLjLongJmp(MachineInstr &MI,
                                  // the last instruction of the expansion.
   }
   MIB.setMemRefs(MMOs);
+  MIB.setMIFlag(MachineInstr::FrameDestroy);
 
   // Jump
   BuildMI(*thisMBB, MI, MIMD, TII->get(IJmpOpc)).addReg(Tmp);
diff --git a/llvm/test/CodeGen/X86/clobber_frame_ptr.ll b/llvm/test/CodeGen/X86/clobber_frame_ptr.ll
index 6209e1a85e9e197..fd8ba7feb9f48fb 100644
--- a/llvm/test/CodeGen/X86/clobber_frame_ptr.ll
+++ b/llvm/test/CodeGen/X86/clobber_frame_ptr.ll
@@ -157,3 +157,46 @@ define i64 @test3(i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, i64 %a6,
   %x = call cc 11 i64 @hipe2(i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, i64 %a6, i64 %a7)
   ret i64 %x
 }
+
+@buf = dso_local global [20 x ptr] zeroinitializer, align 16
+
+; longjmp modifies fp, it is expected behavior, wo should not save/restore fp
+; around it.
+define void @test4() {
+; CHECK-LABEL: test4:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq %rsp, %rbp
+; CHECK-NEXT:    .cfi_def_cfa_register %rbp
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    andq $-16, %rsp
+; CHECK-NEXT:    subq $16, %rsp
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_remember_state
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x06, 0x77, 0x08, 0x06, 0x11, 0x10, 0x22 #
+; CHECK-NEXT:    xorl %r13d, %r13d
+; CHECK-NEXT:    callq external@PLT
+; CHECK-NEXT:    addq $8, %rsp
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_restore_state
+; CHECK-NEXT:    movq buf(%rip), %rbp
+; CHECK-NEXT:    movq buf+8(%rip), %rax
+; CHECK-NEXT:    movq buf+16(%rip), %rsp
+; CHECK-NEXT:    jmpq *%rax
+entry:
+  %x = call ghccc i32 @external(i32 0)
+  call void @llvm.eh.sjlj.longjmp(ptr @buf)
+  unreachable
+}

From 107f3efdbedeecf4768fc0387db929e51ce96f85 Mon Sep 17 00:00:00 2001
From: Jesse Huang <jesse.huang@sifive.com>
Date: Sun, 18 Aug 2024 02:59:46 +0800
Subject: [PATCH 272/441] [RISCV][MC] Make error message of CSR with wrong
 extension more detailed (#104424)

Make the error message of lacking an extension for a CSR more
detailed and update related tests
---
 .../Target/RISCV/AsmParser/RISCVAsmParser.cpp |  22 +-
 .../test/MC/RISCV/machine-csr-names-invalid.s |  66 ++--
 llvm/test/MC/RISCV/rv32-only-csr-names.s      | 286 +++++++++---------
 3 files changed, 195 insertions(+), 179 deletions(-)

diff --git a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
index a288e7d884d311d..5ade54dc3fccd49 100644
--- a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
+++ b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
@@ -1913,7 +1913,7 @@ ParseStatus RISCVAsmParser::parseCSRSystemRegister(OperandVector &Operands) {
     if (getParser().parseIdentifier(Identifier))
       return ParseStatus::Failure;
 
-    auto SysReg = RISCVSysReg::lookupSysRegByName(Identifier);
+    const auto *SysReg = RISCVSysReg::lookupSysRegByName(Identifier);
     if (!SysReg)
       SysReg = RISCVSysReg::lookupSysRegByAltName(Identifier);
     if (!SysReg)
@@ -1923,8 +1923,24 @@ ParseStatus RISCVAsmParser::parseCSRSystemRegister(OperandVector &Operands) {
 
     // Accept a named Sys Reg if the required features are present.
     if (SysReg) {
-      if (!SysReg->haveRequiredFeatures(getSTI().getFeatureBits()))
-        return Error(S, "system register use requires an option to be enabled");
+      const auto &FeatureBits = getSTI().getFeatureBits();
+      if (!SysReg->haveRequiredFeatures(FeatureBits)) {
+        const auto *Feature = llvm::find_if(RISCVFeatureKV, [&](auto Feature) {
+          return SysReg->FeaturesRequired[Feature.Value];
+        });
+        auto ErrorMsg = std::string("system register '") + SysReg->Name + "' ";
+        if (SysReg->isRV32Only && FeatureBits[RISCV::Feature64Bit]) {
+          ErrorMsg += "is RV32 only";
+          if (Feature != std::end(RISCVFeatureKV))
+            ErrorMsg += " and ";
+        }
+        if (Feature != std::end(RISCVFeatureKV)) {
+          ErrorMsg +=
+              "requires '" + std::string(Feature->Key) + "' to be enabled";
+        }
+
+        return Error(S, ErrorMsg);
+      }
       Operands.push_back(
           RISCVOperand::createSysReg(Identifier, S, SysReg->Encoding));
       return ParseStatus::Success;
diff --git a/llvm/test/MC/RISCV/machine-csr-names-invalid.s b/llvm/test/MC/RISCV/machine-csr-names-invalid.s
index 9afb6d7744a8df1..526c41773484ca0 100644
--- a/llvm/test/MC/RISCV/machine-csr-names-invalid.s
+++ b/llvm/test/MC/RISCV/machine-csr-names-invalid.s
@@ -3,38 +3,38 @@
 
 # These machine mode CSR register names are RV32 only.
 
-csrrs t1, pmpcfg1, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, pmpcfg3, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
+csrrs t1, pmpcfg1, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'pmpcfg1' is RV32 only
+csrrs t1, pmpcfg3, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'pmpcfg3' is RV32 only
 
-csrrs t1, mcycleh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, minstreth, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
+csrrs t1, mcycleh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mcycleh' is RV32 only
+csrrs t1, minstreth, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'minstreth' is RV32 only
 
-csrrs t1, mhpmcounter3h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter4h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter5h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter6h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter7h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter8h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter9h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter10h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter11h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter12h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter13h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter14h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter15h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter16h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter17h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter18h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter19h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter20h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter21h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter22h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter23h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter24h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter25h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter26h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter27h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter28h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter29h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter30h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter31h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
+csrrs t1, mhpmcounter3h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter3h' is RV32 only
+csrrs t1, mhpmcounter4h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter4h' is RV32 only
+csrrs t1, mhpmcounter5h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter5h' is RV32 only
+csrrs t1, mhpmcounter6h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter6h' is RV32 only
+csrrs t1, mhpmcounter7h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter7h' is RV32 only
+csrrs t1, mhpmcounter8h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter8h' is RV32 only
+csrrs t1, mhpmcounter9h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter9h' is RV32 only
+csrrs t1, mhpmcounter10h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter10h' is RV32 only
+csrrs t1, mhpmcounter11h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter11h' is RV32 only
+csrrs t1, mhpmcounter12h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter12h' is RV32 only
+csrrs t1, mhpmcounter13h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter13h' is RV32 only
+csrrs t1, mhpmcounter14h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter14h' is RV32 only
+csrrs t1, mhpmcounter15h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter15h' is RV32 only
+csrrs t1, mhpmcounter16h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter16h' is RV32 only
+csrrs t1, mhpmcounter17h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter17h' is RV32 only
+csrrs t1, mhpmcounter18h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter18h' is RV32 only
+csrrs t1, mhpmcounter19h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter19h' is RV32 only
+csrrs t1, mhpmcounter20h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter20h' is RV32 only
+csrrs t1, mhpmcounter21h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter21h' is RV32 only
+csrrs t1, mhpmcounter22h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter22h' is RV32 only
+csrrs t1, mhpmcounter23h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter23h' is RV32 only
+csrrs t1, mhpmcounter24h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter24h' is RV32 only
+csrrs t1, mhpmcounter25h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter25h' is RV32 only
+csrrs t1, mhpmcounter26h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter26h' is RV32 only
+csrrs t1, mhpmcounter27h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter27h' is RV32 only
+csrrs t1, mhpmcounter28h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter28h' is RV32 only
+csrrs t1, mhpmcounter29h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter29h' is RV32 only
+csrrs t1, mhpmcounter30h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter30h' is RV32 only
+csrrs t1, mhpmcounter31h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter31h' is RV32 only
diff --git a/llvm/test/MC/RISCV/rv32-only-csr-names.s b/llvm/test/MC/RISCV/rv32-only-csr-names.s
index 5a91aea4c791b20..db88eacf9396bb3 100644
--- a/llvm/test/MC/RISCV/rv32-only-csr-names.s
+++ b/llvm/test/MC/RISCV/rv32-only-csr-names.s
@@ -3,146 +3,146 @@
 
 # The following CSR register names are all RV32 only.
 
-csrrs t1, cycleh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, timeh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, instreth, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-
-csrrs t1, hpmcounter3h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hpmcounter4h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hpmcounter5h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hpmcounter6h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hpmcounter7h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hpmcounter8h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hpmcounter9h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hpmcounter10h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hpmcounter11h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hpmcounter12h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hpmcounter13h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hpmcounter14h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hpmcounter15h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hpmcounter16h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hpmcounter17h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hpmcounter18h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hpmcounter19h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hpmcounter20h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hpmcounter21h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hpmcounter22h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hpmcounter23h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hpmcounter24h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hpmcounter25h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hpmcounter26h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hpmcounter27h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hpmcounter28h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hpmcounter29h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hpmcounter30h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hpmcounter31h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-
-csrrs t1, henvcfgh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-
-csrrs t1, htimedeltah, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-
-csrrs t1, mstatush, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-
-csrrs t1, menvcfgh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-
-csrrs t1, mseccfgh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-
-csrrs t1, pmpcfg1, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, pmpcfg3, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, pmpcfg5, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, pmpcfg7, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, pmpcfg9, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, pmpcfg11, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, pmpcfg13, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, pmpcfg15, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-
-csrrs t1, mcycleh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, minstreth, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-
-csrrs t1, mhpmcounter3h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter4h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter5h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter6h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter7h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter8h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter9h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter10h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter11h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter12h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter13h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter14h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter15h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter16h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter17h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter18h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter19h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter20h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter21h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter22h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter23h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter24h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter25h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter26h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter27h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter28h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter29h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter30h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmcounter31h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-
-csrrs t1, mhpmevent3h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmevent4h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmevent5h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmevent6h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmevent7h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmevent8h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmevent9h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmevent10h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmevent11h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmevent12h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmevent13h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmevent14h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmevent15h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmevent16h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmevent17h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmevent18h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmevent19h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmevent20h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmevent21h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmevent22h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmevent23h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmevent24h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmevent25h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmevent26h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmevent27h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmevent28h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmevent29h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmevent30h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mhpmevent31h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-
-csrrs t1, mstateen0h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mstateen1h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mstateen3h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mstateen3h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-
-csrrs t1, hstateen0h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hstateen1h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hstateen3h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hstateen3h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-
-csrrs t1, stimecmph, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, vstimecmph, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-
-csrrs t1, midelegh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mieh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mvienh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, mviph, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, miph, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, sieh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, siph, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hidelegh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hvienh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hviph, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hviprio1h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, hviprio2h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, vsieh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
-csrrs t1, vsiph, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register use requires an option to be enabled
+csrrs t1, cycleh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'cycleh' is RV32 only
+csrrs t1, timeh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'timeh' is RV32 only
+csrrs t1, instreth, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'instreth' is RV32 only
+
+csrrs t1, hpmcounter3h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hpmcounter3h' is RV32 only
+csrrs t1, hpmcounter4h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hpmcounter4h' is RV32 only
+csrrs t1, hpmcounter5h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hpmcounter5h' is RV32 only
+csrrs t1, hpmcounter6h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hpmcounter6h' is RV32 only
+csrrs t1, hpmcounter7h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hpmcounter7h' is RV32 only
+csrrs t1, hpmcounter8h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hpmcounter8h' is RV32 only
+csrrs t1, hpmcounter9h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hpmcounter9h' is RV32 only
+csrrs t1, hpmcounter10h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hpmcounter10h' is RV32 only
+csrrs t1, hpmcounter11h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hpmcounter11h' is RV32 only
+csrrs t1, hpmcounter12h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hpmcounter12h' is RV32 only
+csrrs t1, hpmcounter13h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hpmcounter13h' is RV32 only
+csrrs t1, hpmcounter14h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hpmcounter14h' is RV32 only
+csrrs t1, hpmcounter15h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hpmcounter15h' is RV32 only
+csrrs t1, hpmcounter16h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hpmcounter16h' is RV32 only
+csrrs t1, hpmcounter17h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hpmcounter17h' is RV32 only
+csrrs t1, hpmcounter18h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hpmcounter18h' is RV32 only
+csrrs t1, hpmcounter19h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hpmcounter19h' is RV32 only
+csrrs t1, hpmcounter20h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hpmcounter20h' is RV32 only
+csrrs t1, hpmcounter21h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hpmcounter21h' is RV32 only
+csrrs t1, hpmcounter22h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hpmcounter22h' is RV32 only
+csrrs t1, hpmcounter23h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hpmcounter23h' is RV32 only
+csrrs t1, hpmcounter24h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hpmcounter24h' is RV32 only
+csrrs t1, hpmcounter25h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hpmcounter25h' is RV32 only
+csrrs t1, hpmcounter26h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hpmcounter26h' is RV32 only
+csrrs t1, hpmcounter27h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hpmcounter27h' is RV32 only
+csrrs t1, hpmcounter28h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hpmcounter28h' is RV32 only
+csrrs t1, hpmcounter29h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hpmcounter29h' is RV32 only
+csrrs t1, hpmcounter30h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hpmcounter30h' is RV32 only
+csrrs t1, hpmcounter31h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hpmcounter31h' is RV32 only
+
+csrrs t1, henvcfgh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'henvcfgh' is RV32 only
+
+csrrs t1, htimedeltah, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'htimedeltah' is RV32 only
+
+csrrs t1, mstatush, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mstatush' is RV32 only
+
+csrrs t1, menvcfgh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'menvcfgh' is RV32 only
+
+csrrs t1, mseccfgh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mseccfgh' is RV32 only
+
+csrrs t1, pmpcfg1, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'pmpcfg1' is RV32 only
+csrrs t1, pmpcfg3, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'pmpcfg3' is RV32 only
+csrrs t1, pmpcfg5, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'pmpcfg5' is RV32 only
+csrrs t1, pmpcfg7, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'pmpcfg7' is RV32 only
+csrrs t1, pmpcfg9, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'pmpcfg9' is RV32 only
+csrrs t1, pmpcfg11, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'pmpcfg11' is RV32 only
+csrrs t1, pmpcfg13, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'pmpcfg13' is RV32 only
+csrrs t1, pmpcfg15, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'pmpcfg15' is RV32 only
+
+csrrs t1, mcycleh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mcycleh' is RV32 only
+csrrs t1, minstreth, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'minstreth' is RV32 only
+
+csrrs t1, mhpmcounter3h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter3h' is RV32 only
+csrrs t1, mhpmcounter4h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter4h' is RV32 only
+csrrs t1, mhpmcounter5h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter5h' is RV32 only
+csrrs t1, mhpmcounter6h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter6h' is RV32 only
+csrrs t1, mhpmcounter7h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter7h' is RV32 only
+csrrs t1, mhpmcounter8h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter8h' is RV32 only
+csrrs t1, mhpmcounter9h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter9h' is RV32 only
+csrrs t1, mhpmcounter10h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter10h' is RV32 only
+csrrs t1, mhpmcounter11h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter11h' is RV32 only
+csrrs t1, mhpmcounter12h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter12h' is RV32 only
+csrrs t1, mhpmcounter13h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter13h' is RV32 only
+csrrs t1, mhpmcounter14h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter14h' is RV32 only
+csrrs t1, mhpmcounter15h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter15h' is RV32 only
+csrrs t1, mhpmcounter16h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter16h' is RV32 only
+csrrs t1, mhpmcounter17h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter17h' is RV32 only
+csrrs t1, mhpmcounter18h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter18h' is RV32 only
+csrrs t1, mhpmcounter19h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter19h' is RV32 only
+csrrs t1, mhpmcounter20h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter20h' is RV32 only
+csrrs t1, mhpmcounter21h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter21h' is RV32 only
+csrrs t1, mhpmcounter22h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter22h' is RV32 only
+csrrs t1, mhpmcounter23h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter23h' is RV32 only
+csrrs t1, mhpmcounter24h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter24h' is RV32 only
+csrrs t1, mhpmcounter25h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter25h' is RV32 only
+csrrs t1, mhpmcounter26h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter26h' is RV32 only
+csrrs t1, mhpmcounter27h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter27h' is RV32 only
+csrrs t1, mhpmcounter28h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter28h' is RV32 only
+csrrs t1, mhpmcounter29h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter29h' is RV32 only
+csrrs t1, mhpmcounter30h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter30h' is RV32 only
+csrrs t1, mhpmcounter31h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmcounter31h' is RV32 only
+
+csrrs t1, mhpmevent3h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmevent3h' is RV32 only
+csrrs t1, mhpmevent4h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmevent4h' is RV32 only
+csrrs t1, mhpmevent5h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmevent5h' is RV32 only
+csrrs t1, mhpmevent6h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmevent6h' is RV32 only
+csrrs t1, mhpmevent7h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmevent7h' is RV32 only
+csrrs t1, mhpmevent8h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmevent8h' is RV32 only
+csrrs t1, mhpmevent9h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmevent9h' is RV32 only
+csrrs t1, mhpmevent10h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmevent10h' is RV32 only
+csrrs t1, mhpmevent11h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmevent11h' is RV32 only
+csrrs t1, mhpmevent12h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmevent12h' is RV32 only
+csrrs t1, mhpmevent13h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmevent13h' is RV32 only
+csrrs t1, mhpmevent14h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmevent14h' is RV32 only
+csrrs t1, mhpmevent15h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmevent15h' is RV32 only
+csrrs t1, mhpmevent16h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmevent16h' is RV32 only
+csrrs t1, mhpmevent17h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmevent17h' is RV32 only
+csrrs t1, mhpmevent18h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmevent18h' is RV32 only
+csrrs t1, mhpmevent19h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmevent19h' is RV32 only
+csrrs t1, mhpmevent20h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmevent20h' is RV32 only
+csrrs t1, mhpmevent21h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmevent21h' is RV32 only
+csrrs t1, mhpmevent22h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmevent22h' is RV32 only
+csrrs t1, mhpmevent23h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmevent23h' is RV32 only
+csrrs t1, mhpmevent24h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmevent24h' is RV32 only
+csrrs t1, mhpmevent25h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmevent25h' is RV32 only
+csrrs t1, mhpmevent26h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmevent26h' is RV32 only
+csrrs t1, mhpmevent27h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmevent27h' is RV32 only
+csrrs t1, mhpmevent28h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmevent28h' is RV32 only
+csrrs t1, mhpmevent29h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmevent29h' is RV32 only
+csrrs t1, mhpmevent30h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmevent30h' is RV32 only
+csrrs t1, mhpmevent31h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mhpmevent31h' is RV32 only
+
+csrrs t1, mstateen0h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mstateen0h' is RV32 only
+csrrs t1, mstateen1h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mstateen1h' is RV32 only
+csrrs t1, mstateen3h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mstateen3h' is RV32 only
+csrrs t1, mstateen3h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mstateen3h' is RV32 only
+
+csrrs t1, hstateen0h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hstateen0h' is RV32 only
+csrrs t1, hstateen1h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hstateen1h' is RV32 only
+csrrs t1, hstateen3h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hstateen3h' is RV32 only
+csrrs t1, hstateen3h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hstateen3h' is RV32 only
+
+csrrs t1, stimecmph, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'stimecmph' is RV32 only
+csrrs t1, vstimecmph, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'vstimecmph' is RV32 only
+
+csrrs t1, midelegh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'midelegh' is RV32 only
+csrrs t1, mieh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mieh' is RV32 only
+csrrs t1, mvienh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mvienh' is RV32 only
+csrrs t1, mviph, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'mviph' is RV32 only
+csrrs t1, miph, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'miph' is RV32 only
+csrrs t1, sieh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'sieh' is RV32 only
+csrrs t1, siph, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'siph' is RV32 only
+csrrs t1, hidelegh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hidelegh' is RV32 only
+csrrs t1, hvienh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hvienh' is RV32 only
+csrrs t1, hviph, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hviph' is RV32 only
+csrrs t1, hviprio1h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hviprio1h' is RV32 only
+csrrs t1, hviprio2h, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'hviprio2h' is RV32 only
+csrrs t1, vsieh, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'vsieh' is RV32 only
+csrrs t1, vsiph, zero # CHECK-NEED-RV32: :[[@LINE]]:11: error: system register 'vsiph' is RV32 only

From f620c5b692dd0bc08692979236073db0db27f0fc Mon Sep 17 00:00:00 2001
From: Timm Baeder <tbaeder@redhat.com>
Date: Sat, 17 Aug 2024 22:06:46 +0200
Subject: [PATCH 273/441] [clang][bytecode] Classify 1-bit unsigned integers as
 bool (#104662)

This happens for enum types with bool parent types. isBooleanType()
returns false for them however.

The previous version did the same thing by re-classifying the enum
integer type, but that breaks with forward-declared enums.
---
 clang/lib/AST/ByteCode/Context.cpp | 6 +++---
 clang/test/AST/ByteCode/c.c        | 3 ++-
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/clang/lib/AST/ByteCode/Context.cpp b/clang/lib/AST/ByteCode/Context.cpp
index e9c1fd1b8dc9f96..48ae363f651f5a4 100644
--- a/clang/lib/AST/ByteCode/Context.cpp
+++ b/clang/lib/AST/ByteCode/Context.cpp
@@ -135,9 +135,6 @@ std::optional<PrimType> Context::classify(QualType T) const {
   if (T->isAnyComplexType() || T->isVectorType())
     return std::nullopt;
 
-  if (const auto *ET = T->getAs<EnumType>())
-    return classify(ET->getDecl()->getIntegerType());
-
   if (T->isSignedIntegerOrEnumerationType()) {
     switch (Ctx.getIntWidth(T)) {
     case 64:
@@ -163,6 +160,9 @@ std::optional<PrimType> Context::classify(QualType T) const {
       return PT_Uint16;
     case 8:
       return PT_Uint8;
+    case 1:
+      // Might happen for enum types.
+      return PT_Bool;
     default:
       return PT_IntAP;
     }
diff --git a/clang/test/AST/ByteCode/c.c b/clang/test/AST/ByteCode/c.c
index 13a5e082a125f20..b38259d41130eb1 100644
--- a/clang/test/AST/ByteCode/c.c
+++ b/clang/test/AST/ByteCode/c.c
@@ -295,4 +295,5 @@ void T1(void) {
                                                            // pedantic-warning {{use of GNU statement expression extension}}
 }
 
-
+enum teste1 test1f(void), (*test1)(void) = test1f; // pedantic-warning {{ISO C forbids forward references to 'enum' types}}
+enum teste1 { TEST1 };

From b21b907aeb36a4ce6b403944e9c440a3bba5c3e0 Mon Sep 17 00:00:00 2001
From: Kelvin Li <kkwli@users.noreply.github.com>
Date: Sat, 17 Aug 2024 16:53:11 -0400
Subject: [PATCH 274/441] [flang] Handle Hollerith in data statement
 initialization in big endian (#103451)

Update to read the Hollerith literal in reverse order for big endian
environment.
---
 flang/lib/Evaluate/tools.cpp    |  4 ++--
 flang/test/Semantics/data08.f90 | 17 ++++++++++++-----
 2 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/flang/lib/Evaluate/tools.cpp b/flang/lib/Evaluate/tools.cpp
index 4d78f814f8ef2c9..6b3db619c1e2f1a 100644
--- a/flang/lib/Evaluate/tools.cpp
+++ b/flang/lib/Evaluate/tools.cpp
@@ -1300,12 +1300,12 @@ std::optional<Expr<SomeType>> HollerithToBOZ(FoldingContext &context,
     const Expr<SomeType> &expr, const DynamicType &type) {
   if (std::optional<std::string> chValue{GetScalarConstantValue<Ascii>(expr)}) {
     // Pad on the right with spaces when short, truncate the right if long.
-    // TODO: big-endian targets
     auto bytes{static_cast<std::size_t>(
         ToInt64(type.MeasureSizeInBytes(context, false)).value())};
     BOZLiteralConstant bits{0};
     for (std::size_t j{0}; j < bytes; ++j) {
-      char ch{j >= chValue->size() ? ' ' : chValue->at(j)};
+      auto idx{isHostLittleEndian ? j : bytes - j - 1};
+      char ch{idx >= chValue->size() ? ' ' : chValue->at(idx)};
       BOZLiteralConstant chBOZ{static_cast<unsigned char>(ch)};
       bits = bits.IOR(chBOZ.SHIFTL(8 * j));
     }
diff --git a/flang/test/Semantics/data08.f90 b/flang/test/Semantics/data08.f90
index 7bd70637aabda78..7e12a71d117728e 100644
--- a/flang/test/Semantics/data08.f90
+++ b/flang/test/Semantics/data08.f90
@@ -1,12 +1,19 @@
-! RUN: %flang_fc1 -fdebug-dump-symbols -pedantic %s 2>&1 | FileCheck %s
+! RUN: %flang_fc1 -fdebug-dump-symbols -pedantic %s 2>&1 | FileCheck %s \
+! RUN:   --check-prefixes=%if system-aix %{"CHECK","BE"%} \
+! RUN:                    %else %{"CHECK","LE"%}
+
 ! CHECK: DATA statement value initializes 'jx' of type 'INTEGER(4)' with CHARACTER
 ! CHECK: DATA statement value initializes 'jy' of type 'INTEGER(4)' with CHARACTER
 ! CHECK: DATA statement value initializes 'jz' of type 'INTEGER(4)' with CHARACTER
 ! CHECK: DATA statement value initializes 'kx' of type 'INTEGER(8)' with CHARACTER
-! CHECK: jx (InDataStmt) size=4 offset=0: ObjectEntity type: INTEGER(4) init:1684234849_4
-! CHECK: jy (InDataStmt) size=4 offset=4: ObjectEntity type: INTEGER(4) init:543384161_4
-! CHECK: jz (InDataStmt) size=4 offset=8: ObjectEntity type: INTEGER(4) init:1684234849_4
-! CHECK: kx (InDataStmt) size=8 offset=16: ObjectEntity type: INTEGER(8) init:7523094288207667809_8
+! LE: jx (InDataStmt) size=4 offset=0: ObjectEntity type: INTEGER(4) init:1684234849_4
+! BE: jx (InDataStmt) size=4 offset=0: ObjectEntity type: INTEGER(4) init:1633837924_4
+! LE: jy (InDataStmt) size=4 offset=4: ObjectEntity type: INTEGER(4) init:543384161_4
+! BE: jy (InDataStmt) size=4 offset=4: ObjectEntity type: INTEGER(4) init:1633837856_4
+! LE: jz (InDataStmt) size=4 offset=8: ObjectEntity type: INTEGER(4) init:1684234849_4
+! BE: jz (InDataStmt) size=4 offset=8: ObjectEntity type: INTEGER(4) init:1633837924_4
+! LE: kx (InDataStmt) size=8 offset=16: ObjectEntity type: INTEGER(8) init:7523094288207667809_8
+! BE: kx (InDataStmt) size=8 offset=16: ObjectEntity type: INTEGER(8) init:7017280452245743464_8
 
 integer :: jx, jy, jz
 integer(8) :: kx

From 555399d562201f3114fb22f883cd1b6710af1a0d Mon Sep 17 00:00:00 2001
From: Timm Baeder <tbaeder@redhat.com>
Date: Sat, 17 Aug 2024 22:57:19 +0200
Subject: [PATCH 275/441] [clang][bytecode] Fix shifting negative values
 (#104663)

---
 clang/lib/AST/ByteCode/Interp.h   | 11 +++++++++++
 clang/test/AST/ByteCode/cxx23.cpp | 14 ++++++++++++++
 2 files changed, 25 insertions(+)

diff --git a/clang/lib/AST/ByteCode/Interp.h b/clang/lib/AST/ByteCode/Interp.h
index c3ce21cb13770ed..9891e3dba0d309e 100644
--- a/clang/lib/AST/ByteCode/Interp.h
+++ b/clang/lib/AST/ByteCode/Interp.h
@@ -2354,6 +2354,17 @@ inline bool DoShift(InterpState &S, CodePtr OpPC, LT &LHS, RT &RHS) {
                                  LT::AsUnsigned::from(RHS, Bits), Bits, &R);
   }
 
+  // We did the shift above as unsigned. Restore the sign bit if we need to.
+  if constexpr (Dir == ShiftDir::Right) {
+    if (LHS.isSigned() && LHS.isNegative()) {
+      typename LT::AsUnsigned SignBit;
+      LT::AsUnsigned::shiftLeft(LT::AsUnsigned::from(1, Bits),
+                                LT::AsUnsigned::from(Bits - 1, Bits), Bits,
+                                &SignBit);
+      LT::AsUnsigned::bitOr(R, SignBit, Bits, &R);
+    }
+  }
+
   S.Stk.push<LT>(LT::from(R));
   return true;
 }
diff --git a/clang/test/AST/ByteCode/cxx23.cpp b/clang/test/AST/ByteCode/cxx23.cpp
index eb05a9fda0dfb13..756eec5b8256053 100644
--- a/clang/test/AST/ByteCode/cxx23.cpp
+++ b/clang/test/AST/ByteCode/cxx23.cpp
@@ -224,3 +224,17 @@ namespace ExplicitLambdaInstancePointer {
       static_assert(fp(1) == 1, "");
   }
 }
+
+namespace TwosComplementShifts {
+  using uint32 = __UINT32_TYPE__;
+  using int32 = __INT32_TYPE__;
+  static_assert(uint32(int32(0x1234) << 16) == 0x12340000);
+  static_assert(uint32(int32(0x1234) << 19) == 0x91a00000);
+  static_assert(uint32(int32(0x1234) << 20) == 0x23400000);
+  static_assert(uint32(int32(0x1234) << 24) == 0x34000000);
+  static_assert(uint32(int32(-1) << 31) == 0x80000000);
+
+  static_assert(-2 >> 1 == -1);
+  static_assert(-3 >> 1 == -2);
+  static_assert(-7 >> 1 == -4);
+}

From 69f76c782b554a004078af6909c19a11e3846415 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Martin=20Storsj=C3=B6?= <martin@martin.st>
Date: Sun, 18 Aug 2024 00:44:16 +0300
Subject: [PATCH 276/441] [LLD] [MinGW] Recognize the -rpath option (#102886)

GNU ld silently accepts the -rpath option for Windows targets, as a
no-op.

This has lead to some build systems (and users) passing this option
while building for Windows/MinGW, even if Windows doesn't have any
concept like rpath.

Older versions of Conan did include -rpath in the pkg-config files it
generated, see e.g.

https://github.com/conan-io/conan/blob/17c58f0c61931f9de218ac571cd97a8e0befa68e/conans/client/generators/pkg_config.py#L104-L114
and
https://github.com/conan-io/conan/blob/17c58f0c61931f9de218ac571cd97a8e0befa68e/conans/client/build/compiler_flags.py#L26-L34
- and see https://github.com/mstorsjo/llvm-mingw/issues/300 for user
reports about this issue.

Recognize the option in LLD for MinGW targets, to improve drop-in
compatibility compared to GNU ld, but produce a warning to alert users
that the option really has no effect for these targets.
---
 lld/MinGW/Driver.cpp       | 3 +++
 lld/MinGW/Options.td       | 3 +++
 lld/test/MinGW/driver.test | 6 ++++++
 3 files changed, 12 insertions(+)

diff --git a/lld/MinGW/Driver.cpp b/lld/MinGW/Driver.cpp
index 35fd478a21905e7..c7d7b9cfca386f5 100644
--- a/lld/MinGW/Driver.cpp
+++ b/lld/MinGW/Driver.cpp
@@ -448,6 +448,9 @@ bool link(ArrayRef<const char *> argsArr, llvm::raw_ostream &stdoutOS,
       add("-errorlimit:" + s);
   }
 
+  if (auto *a = args.getLastArg(OPT_rpath))
+    warn("parameter " + a->getSpelling() + " has no effect on PE/COFF targets");
+
   for (auto *a : args.filtered(OPT_mllvm))
     add("-mllvm:" + StringRef(a->getValue()));
 
diff --git a/lld/MinGW/Options.td b/lld/MinGW/Options.td
index 56f67e3dd96c426..7bd5fb80749da2c 100644
--- a/lld/MinGW/Options.td
+++ b/lld/MinGW/Options.td
@@ -243,6 +243,9 @@ defm: EqNoHelp<"sysroot">;
 def: F<"sort-common">;
 def: F<"start-group">;
 
+// Ignored options, that produce warnings
+defm rpath: EqNoHelp<"rpath">;
+
 // Ignore GCC collect2 LTO plugin related options. Note that we don't support
 // GCC LTO, but GCC collect2 passes these options even in non-LTO mode.
 def: J<"plugin-opt=-fresolution=">;
diff --git a/lld/test/MinGW/driver.test b/lld/test/MinGW/driver.test
index cbccd09793c2f0d..0dab66b613c774f 100644
--- a/lld/test/MinGW/driver.test
+++ b/lld/test/MinGW/driver.test
@@ -446,3 +446,9 @@ RUN: ld.lld -### foo.o -m i386pep --build-id=none 2>&1 | FileCheck -check-prefix
 RUN: ld.lld -### foo.o -m i386pep -s 2>&1 | FileCheck -check-prefix=NO_BUILD_ID %s
 RUN: ld.lld -### foo.o -m i386pep -S 2>&1 | FileCheck -check-prefix=NO_BUILD_ID %s
 NO_BUILD_ID: -build-id:no
+
+RUN: ld.lld -### foo.o -m i386pep -rpath foo 2>&1 | FileCheck -check-prefix=WARN_RPATH %s
+RUN: ld.lld -### foo.o -m i386pep --rpath foo 2>&1 | FileCheck -check-prefix=WARN_RPATH %s
+RUN: ld.lld -### foo.o -m i386pep -rpath=foo 2>&1 | FileCheck -check-prefix=WARN_RPATH %s
+RUN: ld.lld -### foo.o -m i386pep --rpath=foo 2>&1 | FileCheck -check-prefix=WARN_RPATH %s
+WARN_RPATH: warning: parameter -{{-?}}rpath has no effect on PE/COFF targets

From f7d94b783f62f61cb9085adaa71e1ffa95918bcc Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Sat, 17 Aug 2024 17:34:53 -0700
Subject: [PATCH 277/441] [SelectionDAG] Use getAllOnesConstant.

---
 llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp    | 5 +++--
 llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp | 2 +-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index debecc40ebb823c..74e3a898569beaf 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -5042,7 +5042,8 @@ void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
   case ISD::RESET_FPENV: {
     // It is legalized to call 'fesetenv(FE_DFL_ENV)'. On most targets
     // FE_DFL_ENV is defined as '((const fenv_t *) -1)' in glibc.
-    SDValue Ptr = DAG.getIntPtrConstant(-1LL, dl);
+    EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
+    SDValue Ptr = DAG.getAllOnesConstant(dl, PtrTy);
     SDValue Chain = Node->getOperand(0);
     Results.push_back(
         DAG.makeStateFunctionCall(RTLIB::FESETENV, Ptr, Chain, dl));
@@ -5097,7 +5098,7 @@ void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
     // target must provide custom lowering.
     const DataLayout &DL = DAG.getDataLayout();
     EVT PtrTy = TLI.getPointerTy(DL);
-    SDValue Mode = DAG.getConstant(-1LL, dl, PtrTy);
+    SDValue Mode = DAG.getAllOnesConstant(dl, PtrTy);
     Results.push_back(DAG.makeStateFunctionCall(RTLIB::FESETMODE, Mode,
                                                 Node->getOperand(0), dl));
     break;
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 0bf84ebb90db599..ae9f11d02a8ac49 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -10658,7 +10658,7 @@ SDValue TargetLowering::expandCMP(SDNode *Node, SelectionDAG &DAG) const {
     SDValue SelectZeroOrOne =
         DAG.getSelect(dl, ResVT, IsGT, DAG.getConstant(1, dl, ResVT),
                       DAG.getConstant(0, dl, ResVT));
-    return DAG.getSelect(dl, ResVT, IsLT, DAG.getConstant(-1, dl, ResVT),
+    return DAG.getSelect(dl, ResVT, IsLT, DAG.getAllOnesConstant(dl, ResVT),
                          SelectZeroOrOne);
   }
 

From efa859cd3e195bc4c377e8acf21d45fa410c63b6 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Sat, 17 Aug 2024 17:35:38 -0700
Subject: [PATCH 278/441] [ARM] Use SelectonDAG::getSignedConstant.

---
 llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp     | 51 ++++++++++++---------
 llvm/lib/Target/ARM/ARMISelLowering.cpp     | 10 ++--
 llvm/lib/Target/ARM/ARMInstrInfo.td         |  6 ++-
 llvm/lib/Target/X86/X86ISelLoweringCall.cpp |  3 +-
 4 files changed, 42 insertions(+), 28 deletions(-)

diff --git a/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp
index 7ffc64378ef4baf..f1b579ce7d2d9c9 100644
--- a/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp
+++ b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp
@@ -712,7 +712,8 @@ bool ARMDAGToDAGISel::SelectAddrModeImm12(SDValue N,
         Base = CurDAG->getTargetFrameIndex(
             FI, TLI->getPointerTy(CurDAG->getDataLayout()));
       }
-      OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32);
+      OffImm = CurDAG->getSignedConstant(RHSC, SDLoc(N), MVT::i32,
+                                         /*isTarget=*/true);
       return true;
     }
   }
@@ -881,7 +882,8 @@ bool ARMDAGToDAGISel::SelectAddrMode2OffsetImmPre(SDNode *Op, SDValue N,
   if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val)) { // 12 bits.
     if (AddSub == ARM_AM::sub) Val *= -1;
     Offset = CurDAG->getRegister(0, MVT::i32);
-    Opc = CurDAG->getTargetConstant(Val, SDLoc(Op), MVT::i32);
+    Opc =
+        CurDAG->getSignedConstant(Val, SDLoc(Op), MVT::i32, /*isTarget*/ true);
     return true;
   }
 
@@ -1185,7 +1187,8 @@ ARMDAGToDAGISel::SelectThumbAddrModeImm5S(SDValue N, unsigned Scale,
   int RHSC;
   if (isScaledConstantInRange(N.getOperand(1), Scale, 0, 32, RHSC)) {
     Base = N.getOperand(0);
-    OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32);
+    OffImm =
+        CurDAG->getSignedConstant(RHSC, SDLoc(N), MVT::i32, /*isTarget=*/true);
     return true;
   }
 
@@ -1247,7 +1250,8 @@ bool ARMDAGToDAGISel::SelectThumbAddrModeSP(SDValue N,
         if (MFI.getObjectAlign(FI) >= Align(4)) {
           Base = CurDAG->getTargetFrameIndex(
               FI, TLI->getPointerTy(CurDAG->getDataLayout()));
-          OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32);
+          OffImm = CurDAG->getSignedConstant(RHSC, SDLoc(N), MVT::i32,
+                                             /*isTarget=*/true);
           return true;
         }
       }
@@ -1267,8 +1271,8 @@ bool ARMDAGToDAGISel::SelectTAddrModeImm7(SDValue N, SDValue &Base,
       Base = N.getOperand(0);
       if (N.getOpcode() == ISD::SUB)
         RHSC = -RHSC;
-      OffImm =
-          CurDAG->getTargetConstant(RHSC * (1 << Shift), SDLoc(N), MVT::i32);
+      OffImm = CurDAG->getSignedConstant(RHSC * (1 << Shift), SDLoc(N),
+                                         MVT::i32, /*isTarget=*/true);
       return true;
     }
   }
@@ -1330,7 +1334,8 @@ bool ARMDAGToDAGISel::SelectT2AddrModeImm12(SDValue N,
         Base = CurDAG->getTargetFrameIndex(
             FI, TLI->getPointerTy(CurDAG->getDataLayout()));
       }
-      OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32);
+      OffImm = CurDAG->getSignedConstant(RHSC, SDLoc(N), MVT::i32,
+                                         /*isTarget=*/true);
       return true;
     }
   }
@@ -1357,7 +1362,8 @@ bool ARMDAGToDAGISel::SelectT2AddrModeImm8(SDValue N, SDValue &Base,
       if (N.getOpcode() == ISD::SUB)
         RHSC = -RHSC;
       OffImm =
-          CurDAG->getTargetConstant(RHSC * (1 << Shift), SDLoc(N), MVT::i32);
+          CurDAG->getSignedConstant(RHSC * (1 << Shift), SDLoc(N), MVT::i32,
+                                    /*isTarget=*/true);
       return true;
     }
   }
@@ -1387,7 +1393,8 @@ bool ARMDAGToDAGISel::SelectT2AddrModeImm8(SDValue N,
         Base = CurDAG->getTargetFrameIndex(
             FI, TLI->getPointerTy(CurDAG->getDataLayout()));
       }
-      OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32);
+      OffImm = CurDAG->getSignedConstant(RHSC, SDLoc(N), MVT::i32,
+                                         /*isTarget=*/true);
       return true;
     }
   }
@@ -1404,8 +1411,10 @@ bool ARMDAGToDAGISel::SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N,
   int RHSC;
   if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x100, RHSC)) { // 8 bits.
     OffImm = ((AM == ISD::PRE_INC) || (AM == ISD::POST_INC))
-      ? CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32)
-      : CurDAG->getTargetConstant(-RHSC, SDLoc(N), MVT::i32);
+                 ? CurDAG->getSignedConstant(RHSC, SDLoc(N), MVT::i32,
+                                             /*isTarget=*/true)
+                 : CurDAG->getSignedConstant(-RHSC, SDLoc(N), MVT::i32,
+                                             /*isTarget=*/true);
     return true;
   }
 
@@ -1428,8 +1437,8 @@ bool ARMDAGToDAGISel::SelectT2AddrModeImm7(SDValue N, SDValue &Base,
 
       if (N.getOpcode() == ISD::SUB)
         RHSC = -RHSC;
-      OffImm =
-          CurDAG->getTargetConstant(RHSC * (1 << Shift), SDLoc(N), MVT::i32);
+      OffImm = CurDAG->getSignedConstant(RHSC * (1 << Shift), SDLoc(N),
+                                         MVT::i32, /*isTarget=*/true);
       return true;
     }
   }
@@ -1471,11 +1480,11 @@ bool ARMDAGToDAGISel::SelectT2AddrModeImm7Offset(SDNode *Op, SDValue N,
   int RHSC;
   // 7 bit constant, shifted by Shift.
   if (isScaledConstantInRange(N, 1 << Shift, 0, 0x80, RHSC)) {
-    OffImm =
-        ((AM == ISD::PRE_INC) || (AM == ISD::POST_INC))
-            ? CurDAG->getTargetConstant(RHSC * (1 << Shift), SDLoc(N), MVT::i32)
-            : CurDAG->getTargetConstant(-RHSC * (1 << Shift), SDLoc(N),
-                                        MVT::i32);
+    OffImm = ((AM == ISD::PRE_INC) || (AM == ISD::POST_INC))
+                 ? CurDAG->getSignedConstant(RHSC * (1 << Shift), SDLoc(N),
+                                             MVT::i32, /*isTarget=*/true)
+                 : CurDAG->getSignedConstant(-RHSC * (1 << Shift), SDLoc(N),
+                                             MVT::i32, /*isTarget=*/true);
     return true;
   }
   return false;
@@ -1485,7 +1494,8 @@ template <int Min, int Max>
 bool ARMDAGToDAGISel::SelectImmediateInRange(SDValue N, SDValue &OffImm) {
   int Val;
   if (isScaledConstantInRange(N, 1, Min, Max, Val)) {
-    OffImm = CurDAG->getTargetConstant(Val, SDLoc(N), MVT::i32);
+    OffImm =
+        CurDAG->getSignedConstant(Val, SDLoc(N), MVT::i32, /*isTarget=*/true);
     return true;
   }
   return false;
@@ -3861,8 +3871,7 @@ void ARMDAGToDAGISel::Select(SDNode *N) {
               ConstantMaterializationCost(~Imm, Subtarget)) {
         // The current immediate costs more to materialize than a negated
         // immediate, so negate the immediate and use a BIC.
-        SDValue NewImm =
-          CurDAG->getConstant(~N1C->getZExtValue(), dl, MVT::i32);
+        SDValue NewImm = CurDAG->getConstant(~Imm, dl, MVT::i32);
         // If the new constant didn't exist before, reposition it in the topological
         // ordering so it is just before N. Otherwise, don't touch its location.
         if (NewImm->getNodeId() == -1)
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp
index 1fab30a0b855088..1e8bb8a495e68bb 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -2841,7 +2841,8 @@ ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
   Ops.push_back(Callee);
 
   if (isTailCall) {
-    Ops.push_back(DAG.getTargetConstant(SPDiff, dl, MVT::i32));
+    Ops.push_back(
+        DAG.getSignedConstant(SPDiff, dl, MVT::i32, /*isTarget=*/true));
   }
 
   // Add argument registers to the end of the list so that they are known live
@@ -2892,7 +2893,7 @@ ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
   // we need to undo that after it returns to restore the status-quo.
   bool TailCallOpt = getTargetMachine().Options.GuaranteedTailCallOpt;
   uint64_t CalleePopBytes =
-      canGuaranteeTCO(CallConv, TailCallOpt) ? alignTo(NumBytes, 16) : -1ULL;
+      canGuaranteeTCO(CallConv, TailCallOpt) ? alignTo(NumBytes, 16) : -1U;
 
   Chain = DAG.getCALLSEQ_END(Chain, NumBytes, CalleePopBytes, InGlue, dl);
   if (!Ins.empty())
@@ -8551,7 +8552,7 @@ static SDValue LowerVECTOR_SHUFFLEv8i8(SDValue Op,
 
   SmallVector<SDValue, 8> VTBLMask;
   for (int I : ShuffleMask)
-    VTBLMask.push_back(DAG.getConstant(I, DL, MVT::i32));
+    VTBLMask.push_back(DAG.getSignedConstant(I, DL, MVT::i32));
 
   if (V2.getNode()->isUndef())
     return DAG.getNode(ARMISD::VTBL1, DL, MVT::v8i8, V1,
@@ -20671,7 +20672,8 @@ void ARMTargetLowering::LowerAsmOperandForConstraint(SDValue Op,
         }
         return;
     }
-    Result = DAG.getTargetConstant(CVal, SDLoc(Op), Op.getValueType());
+    Result = DAG.getSignedConstant(CVal, SDLoc(Op), Op.getValueType(),
+                                   /*isTarget=*/true);
     break;
   }
 
diff --git a/llvm/lib/Target/ARM/ARMInstrInfo.td b/llvm/lib/Target/ARM/ARMInstrInfo.td
index c6bcad8e2a82c52..26f7d70b43b2623 100644
--- a/llvm/lib/Target/ARM/ARMInstrInfo.td
+++ b/llvm/lib/Target/ARM/ARMInstrInfo.td
@@ -371,12 +371,14 @@ def ARMVCCElse : PatLeaf<(i32 2)>;
 
 // imm_neg_XFORM - Return the negation of an i32 immediate value.
 def imm_neg_XFORM : SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant(-(int)N->getZExtValue(), SDLoc(N), MVT::i32);
+  return CurDAG->getSignedConstant(-(int)N->getZExtValue(), SDLoc(N), MVT::i32,
+                                   /*isTarget=*/true);
 }]>;
 
 // imm_not_XFORM - Return the complement of a i32 immediate value.
 def imm_not_XFORM : SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant(~(int)N->getZExtValue(), SDLoc(N), MVT::i32);
+  return CurDAG->getSignedConstant(~(int)N->getZExtValue(), SDLoc(N), MVT::i32,
+                                   /*isTarget=*/true);
 }]>;
 def gi_imm_not_XFORM : GICustomOperandRenderer<"renderInvertedImm">,
   GISDNodeXFormEquiv<imm_not_XFORM>;
diff --git a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp
index ce6b0f9c07dc8df..b0158c64c1f383c 100644
--- a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp
+++ b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp
@@ -2426,7 +2426,8 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
   Ops.push_back(Callee);
 
   if (isTailCall)
-    Ops.push_back(DAG.getTargetConstant(FPDiff, dl, MVT::i32));
+    Ops.push_back(
+        DAG.getSignedConstant(FPDiff, dl, MVT::i32, /*isTarget=*/true));
 
   // Add argument registers to the end of the list so that they are known live
   // into the call.

From 69115cce29d5dbbdacaa05aaedc89810aa58ef0c Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Sat, 17 Aug 2024 17:56:33 -0700
Subject: [PATCH 279/441] [AArch64] Use
 SelectionDAG::getSignedConstant/getAllOnesConstant.

---
 .../Target/AArch64/AArch64ISelDAGToDAG.cpp    |  5 ++--
 .../Target/AArch64/AArch64ISelLowering.cpp    | 25 ++++++++++---------
 llvm/lib/Target/AArch64/AArch64InstrInfo.td   |  2 +-
 3 files changed, 17 insertions(+), 15 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
index d9b0f083d469cc3..f5f9a62faa0f534 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
@@ -913,7 +913,8 @@ bool AArch64DAGToDAGISel::SelectRDVLImm(SDValue N, SDValue &Imm) {
   if ((MulImm % std::abs(Scale)) == 0) {
     int64_t RDVLImm = MulImm / Scale;
     if ((RDVLImm >= Low) && (RDVLImm <= High)) {
-      Imm = CurDAG->getTargetConstant(RDVLImm, SDLoc(N), MVT::i32);
+      Imm = CurDAG->getSignedConstant(RDVLImm, SDLoc(N), MVT::i32,
+                                      /*isTarget=*/true);
       return true;
     }
   }
@@ -4245,7 +4246,7 @@ bool AArch64DAGToDAGISel::SelectSVESignedArithImm(SDValue N, SDValue &Imm) {
     int64_t ImmVal = CNode->getSExtValue();
     SDLoc DL(N);
     if (ImmVal >= -128 && ImmVal < 128) {
-      Imm = CurDAG->getTargetConstant(ImmVal, DL, MVT::i32);
+      Imm = CurDAG->getSignedConstant(ImmVal, DL, MVT::i32, /*isTarget=*/true);
       return true;
     }
   }
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 97fb2c5f552731f..e1d265fdf0d1a8e 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -3970,9 +3970,9 @@ static SDValue getAArch64Cmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
         SDValue SExt =
             DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, LHS.getValueType(), LHS,
                         DAG.getValueType(MVT::i16));
-        Cmp = emitComparison(SExt, DAG.getConstant(ValueofRHS, dl,
-                                                   RHS.getValueType()),
-                             CC, dl, DAG);
+        Cmp = emitComparison(
+            SExt, DAG.getSignedConstant(ValueofRHS, dl, RHS.getValueType()), CC,
+            dl, DAG);
         AArch64CC = changeIntCCToAArch64CC(CC);
       }
     }
@@ -4158,7 +4158,7 @@ SDValue AArch64TargetLowering::LowerXOR(SDValue Op, SelectionDAG &DAG) const {
 
     FVal = Other;
     TVal = DAG.getNode(ISD::XOR, dl, Other.getValueType(), Other,
-                       DAG.getConstant(-1ULL, dl, Other.getValueType()));
+                       DAG.getAllOnesConstant(dl, Other.getValueType()));
 
     return DAG.getNode(AArch64ISD::CSEL, dl, Sel.getValueType(), FVal, TVal,
                        CCVal, Cmp);
@@ -8966,7 +8966,8 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI,
     // Each tail call may have to adjust the stack by a different amount, so
     // this information must travel along with the operation for eventual
     // consumption by emitEpilogue.
-    Ops.push_back(DAG.getTargetConstant(FPDiff, DL, MVT::i32));
+    Ops.push_back(
+        DAG.getSignedConstant(FPDiff, DL, MVT::i32, /*isTarget=*/true));
   }
 
   if (CLI.PAI) {
@@ -11144,7 +11145,7 @@ SDValue AArch64TargetLowering::LowerAAPCS_VASTART(SDValue Op,
 
     GRTop = DAG.getFrameIndex(FuncInfo->getVarArgsGPRIndex(), PtrVT);
     GRTop = DAG.getNode(ISD::ADD, DL, PtrVT, GRTop,
-                        DAG.getConstant(GPRSize, DL, PtrVT));
+                        DAG.getSignedConstant(GPRSize, DL, PtrVT));
     GRTop = DAG.getZExtOrTrunc(GRTop, DL, PtrMemVT);
 
     MemOps.push_back(DAG.getStore(Chain, DL, GRTop, GRTopAddr,
@@ -11162,7 +11163,7 @@ SDValue AArch64TargetLowering::LowerAAPCS_VASTART(SDValue Op,
 
     VRTop = DAG.getFrameIndex(FuncInfo->getVarArgsFPRIndex(), PtrVT);
     VRTop = DAG.getNode(ISD::ADD, DL, PtrVT, VRTop,
-                        DAG.getConstant(FPRSize, DL, PtrVT));
+                        DAG.getSignedConstant(FPRSize, DL, PtrVT));
     VRTop = DAG.getZExtOrTrunc(VRTop, DL, PtrMemVT);
 
     MemOps.push_back(DAG.getStore(Chain, DL, VRTop, VRTopAddr,
@@ -11175,7 +11176,7 @@ SDValue AArch64TargetLowering::LowerAAPCS_VASTART(SDValue Op,
   SDValue GROffsAddr = DAG.getNode(ISD::ADD, DL, PtrVT, VAList,
                                    DAG.getConstant(Offset, DL, PtrVT));
   MemOps.push_back(
-      DAG.getStore(Chain, DL, DAG.getConstant(-GPRSize, DL, MVT::i32),
+      DAG.getStore(Chain, DL, DAG.getSignedConstant(-GPRSize, DL, MVT::i32),
                    GROffsAddr, MachinePointerInfo(SV, Offset), Align(4)));
 
   // int __vr_offs at offset 28 (16 on ILP32)
@@ -11183,7 +11184,7 @@ SDValue AArch64TargetLowering::LowerAAPCS_VASTART(SDValue Op,
   SDValue VROffsAddr = DAG.getNode(ISD::ADD, DL, PtrVT, VAList,
                                    DAG.getConstant(Offset, DL, PtrVT));
   MemOps.push_back(
-      DAG.getStore(Chain, DL, DAG.getConstant(-FPRSize, DL, MVT::i32),
+      DAG.getStore(Chain, DL, DAG.getSignedConstant(-FPRSize, DL, MVT::i32),
                    VROffsAddr, MachinePointerInfo(SV, Offset), Align(4)));
 
   return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps);
@@ -15620,7 +15621,7 @@ SDValue AArch64TargetLowering::LowerATOMIC_LOAD_AND(SDValue Op,
   assert(VT != MVT::i128 && "Handled elsewhere, code replicated.");
   SDValue RHS = Op.getOperand(2);
   AtomicSDNode *AN = cast<AtomicSDNode>(Op.getNode());
-  RHS = DAG.getNode(ISD::XOR, dl, VT, DAG.getConstant(-1ULL, dl, VT), RHS);
+  RHS = DAG.getNode(ISD::XOR, dl, VT, DAG.getAllOnesConstant(dl, VT), RHS);
   return DAG.getAtomic(ISD::ATOMIC_LOAD_CLR, dl, AN->getMemoryVT(),
                        Op.getOperand(0), Op.getOperand(1), RHS,
                        AN->getMemOperand());
@@ -21506,7 +21507,7 @@ static SDValue tryConvertSVEWideCompare(SDNode *N, ISD::CondCode CC,
       if (auto *CN = dyn_cast<ConstantSDNode>(Comparator.getOperand(0))) {
         int64_t ImmVal = CN->getSExtValue();
         if (ImmVal >= -16 && ImmVal <= 15)
-          Imm = DAG.getConstant(ImmVal, DL, MVT::i32);
+          Imm = DAG.getSignedConstant(ImmVal, DL, MVT::i32);
         else
           return SDValue();
       }
@@ -24400,7 +24401,7 @@ static SDValue performSETCCCombine(SDNode *N,
       // this pattern will get better opt in emitComparison
       uint64_t TstImm = -1ULL << LHS->getConstantOperandVal(1);
       SDValue TST = DAG.getNode(ISD::AND, DL, TstVT, LHS->getOperand(0),
-                                DAG.getConstant(TstImm, DL, TstVT));
+                                DAG.getSignedConstant(TstImm, DL, TstVT));
       return DAG.getNode(ISD::SETCC, DL, VT, TST, RHS, N->getOperand(2));
     }
   }
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index a9324af5beb784d..a5fae8029deab35 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -2214,7 +2214,7 @@ def s64imm_32bit : ImmLeaf<i64, [{
 }]>;
 
 def trunc_imm : SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant(N->getZExtValue(), SDLoc(N), MVT::i32);
+  return CurDAG->getTargetConstant((uint32_t)N->getZExtValue(), SDLoc(N), MVT::i32);
 }]>;
 
 def gi_trunc_imm : GICustomOperandRenderer<"renderTruncImm">,

From 27a62ec72aed3faf1388600f485552471b580e3b Mon Sep 17 00:00:00 2001
From: Philip Reames <preames@rivosinc.com>
Date: Sat, 17 Aug 2024 18:34:23 -0700
Subject: [PATCH 280/441] [LSR] Split the -lsr-term-fold transformation into
 it's own pass (#104234)

This transformation doesn't actually use any of the internal state of
LSR and recomputes all information from SCEV.  Splitting it out makes
it easier to test.

Note that long term I would like to write a version of this transform
which *is* integrated with LSR's solver, but if that happens, we'll
just delete the extra pass.

Integration wise, I switched from using TTI to using a pass configuration
variable.  This seems slightly more idiomatic, and means we don't run
the extra logic on any target other than RISCV.
---
 .../llvm/Analysis/TargetTransformInfo.h       |   9 -
 .../llvm/Analysis/TargetTransformInfoImpl.h   |   2 -
 llvm/include/llvm/CodeGen/BasicTTIImpl.h      |   5 -
 llvm/include/llvm/CodeGen/TargetPassConfig.h  |   3 +
 llvm/include/llvm/InitializePasses.h          |   1 +
 llvm/include/llvm/LinkAllPasses.h             |   1 +
 .../llvm/Passes/MachinePassRegistry.def       |   1 +
 llvm/include/llvm/Transforms/Scalar.h         |   8 +
 .../llvm/Transforms/Scalar/LoopTermFold.h     |  30 ++
 llvm/lib/Analysis/TargetTransformInfo.cpp     |   4 -
 llvm/lib/CodeGen/TargetPassConfig.cpp         |   2 +
 llvm/lib/Passes/PassBuilder.cpp               |   1 +
 llvm/lib/Passes/PassRegistry.def              |   1 +
 llvm/lib/Target/RISCV/RISCVTargetMachine.cpp  |   1 +
 .../Target/RISCV/RISCVTargetTransformInfo.h   |   3 -
 llvm/lib/Transforms/Scalar/CMakeLists.txt     |   1 +
 .../Transforms/Scalar/LoopStrengthReduce.cpp  | 262 ------------
 llvm/lib/Transforms/Scalar/LoopTermFold.cpp   | 379 ++++++++++++++++++
 llvm/lib/Transforms/Scalar/Scalar.cpp         |   1 +
 llvm/test/CodeGen/RISCV/O3-pipeline.ll        |   1 +
 .../RISCV/lsr-cost-compare.ll                 |   2 +-
 .../RISCV/term-fold-crash.ll                  |   2 +-
 .../lsr-term-fold-negative-testcase.ll        |   2 +-
 .../LoopStrengthReduce/lsr-term-fold.ll       |   2 +-
 .../lsr-unreachable-bb-phi-node.ll            |   5 +-
 25 files changed, 438 insertions(+), 291 deletions(-)
 create mode 100644 llvm/include/llvm/Transforms/Scalar/LoopTermFold.h
 create mode 100644 llvm/lib/Transforms/Scalar/LoopTermFold.cpp

diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index dde90abc06cd9ce..b2124c6106198ec 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -739,11 +739,6 @@ class TargetTransformInfo {
   /// cost should return false, otherwise return true.
   bool isNumRegsMajorCostOfLSR() const;
 
-  /// Return true if LSR should attempts to replace a use of an otherwise dead
-  /// primary IV in the latch condition with another IV available in the loop.
-  /// When successful, makes the primary IV dead.
-  bool shouldFoldTerminatingConditionAfterLSR() const;
-
   /// Return true if LSR should drop a found solution if it's calculated to be
   /// less profitable than the baseline.
   bool shouldDropLSRSolutionIfLessProfitable() const;
@@ -1888,7 +1883,6 @@ class TargetTransformInfo::Concept {
   virtual bool isLSRCostLess(const TargetTransformInfo::LSRCost &C1,
                              const TargetTransformInfo::LSRCost &C2) = 0;
   virtual bool isNumRegsMajorCostOfLSR() = 0;
-  virtual bool shouldFoldTerminatingConditionAfterLSR() const = 0;
   virtual bool shouldDropLSRSolutionIfLessProfitable() const = 0;
   virtual bool isProfitableLSRChainElement(Instruction *I) = 0;
   virtual bool canMacroFuseCmp() = 0;
@@ -2367,9 +2361,6 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
   bool isNumRegsMajorCostOfLSR() override {
     return Impl.isNumRegsMajorCostOfLSR();
   }
-  bool shouldFoldTerminatingConditionAfterLSR() const override {
-    return Impl.shouldFoldTerminatingConditionAfterLSR();
-  }
   bool shouldDropLSRSolutionIfLessProfitable() const override {
     return Impl.shouldDropLSRSolutionIfLessProfitable();
   }
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index d208a710bb27fda..11b07ac0b7fc47c 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -244,8 +244,6 @@ class TargetTransformInfoImplBase {
 
   bool isNumRegsMajorCostOfLSR() const { return true; }
 
-  bool shouldFoldTerminatingConditionAfterLSR() const { return false; }
-
   bool shouldDropLSRSolutionIfLessProfitable() const { return false; }
 
   bool isProfitableLSRChainElement(Instruction *I) const { return false; }
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index 77ddc10e8a0e76d..217e3f1324f9c97 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -394,11 +394,6 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     return TargetTransformInfoImplBase::isNumRegsMajorCostOfLSR();
   }
 
-  bool shouldFoldTerminatingConditionAfterLSR() const {
-    return TargetTransformInfoImplBase::
-        shouldFoldTerminatingConditionAfterLSR();
-  }
-
   bool shouldDropLSRSolutionIfLessProfitable() const {
     return TargetTransformInfoImplBase::shouldDropLSRSolutionIfLessProfitable();
   }
diff --git a/llvm/include/llvm/CodeGen/TargetPassConfig.h b/llvm/include/llvm/CodeGen/TargetPassConfig.h
index d00e0bed91a4579..2f5951e3ec3bce7 100644
--- a/llvm/include/llvm/CodeGen/TargetPassConfig.h
+++ b/llvm/include/llvm/CodeGen/TargetPassConfig.h
@@ -140,6 +140,9 @@ class TargetPassConfig : public ImmutablePass {
   /// callers.
   bool RequireCodeGenSCCOrder = false;
 
+  /// Enable LoopTermFold immediately after LSR
+  bool EnableLoopTermFold = false;
+
   /// Add the actual instruction selection passes. This does not include
   /// preparation passes on IR.
   bool addCoreISelPasses();
diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h
index a4ac314bb590e57..cc5e93c58f564ac 100644
--- a/llvm/include/llvm/InitializePasses.h
+++ b/llvm/include/llvm/InitializePasses.h
@@ -169,6 +169,7 @@ void initializeLoopInfoWrapperPassPass(PassRegistry &);
 void initializeLoopPassPass(PassRegistry &);
 void initializeLoopSimplifyPass(PassRegistry &);
 void initializeLoopStrengthReducePass(PassRegistry &);
+void initializeLoopTermFoldPass(PassRegistry &);
 void initializeLoopUnrollPass(PassRegistry &);
 void initializeLowerAtomicLegacyPassPass(PassRegistry &);
 void initializeLowerConstantIntrinsicsPass(PassRegistry &);
diff --git a/llvm/include/llvm/LinkAllPasses.h b/llvm/include/llvm/LinkAllPasses.h
index e6a70dfd1ea6f83..1da02153d846f11 100644
--- a/llvm/include/llvm/LinkAllPasses.h
+++ b/llvm/include/llvm/LinkAllPasses.h
@@ -90,6 +90,7 @@ struct ForcePassLinking {
     (void)llvm::createLoopExtractorPass();
     (void)llvm::createLoopSimplifyPass();
     (void)llvm::createLoopStrengthReducePass();
+    (void)llvm::createLoopTermFoldPass();
     (void)llvm::createLoopUnrollPass();
     (void)llvm::createLowerGlobalDtorsLegacyPass();
     (void)llvm::createLowerInvokePass();
diff --git a/llvm/include/llvm/Passes/MachinePassRegistry.def b/llvm/include/llvm/Passes/MachinePassRegistry.def
index 8e669ee5791239d..05baf514fa72104 100644
--- a/llvm/include/llvm/Passes/MachinePassRegistry.def
+++ b/llvm/include/llvm/Passes/MachinePassRegistry.def
@@ -79,6 +79,7 @@ FUNCTION_PASS("win-eh-prepare", WinEHPreparePass())
 #define LOOP_PASS(NAME, CREATE_PASS)
 #endif
 LOOP_PASS("loop-reduce", LoopStrengthReducePass())
+LOOP_PASS("loop-term-fold", LoopTermFoldPass())
 #undef LOOP_PASS
 
 #ifndef MACHINE_MODULE_PASS
diff --git a/llvm/include/llvm/Transforms/Scalar.h b/llvm/include/llvm/Transforms/Scalar.h
index 98d0adca3552149..17f4327eb3e1aba 100644
--- a/llvm/include/llvm/Transforms/Scalar.h
+++ b/llvm/include/llvm/Transforms/Scalar.h
@@ -51,6 +51,14 @@ Pass *createLICMPass();
 //
 Pass *createLoopStrengthReducePass();
 
+//===----------------------------------------------------------------------===//
+//
+// LoopTermFold -  This pass attempts to eliminate the last use of an IV in
+// a loop terminator instruction by rewriting it in terms of another IV.
+// Expected to be run immediately after LSR.
+//
+Pass *createLoopTermFoldPass();
+
 //===----------------------------------------------------------------------===//
 //
 // LoopUnroll - This pass is a simple loop unrolling pass.
diff --git a/llvm/include/llvm/Transforms/Scalar/LoopTermFold.h b/llvm/include/llvm/Transforms/Scalar/LoopTermFold.h
new file mode 100644
index 000000000000000..974024c586aa80b
--- /dev/null
+++ b/llvm/include/llvm/Transforms/Scalar/LoopTermFold.h
@@ -0,0 +1,30 @@
+//===- LoopTermFold.h - Loop Term Fold Pass ---------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TRANSFORMS_SCALAR_LOOPTERMFOLD_H
+#define LLVM_TRANSFORMS_SCALAR_LOOPTERMFOLD_H
+
+#include "llvm/Analysis/LoopAnalysisManager.h"
+#include "llvm/IR/PassManager.h"
+
+namespace llvm {
+
+class Loop;
+class LPMUpdater;
+
+class LoopTermFoldPass : public PassInfoMixin<LoopTermFoldPass> {
+public:
+  PreservedAnalyses run(Loop &L, LoopAnalysisManager &AM,
+                        LoopStandardAnalysisResults &AR, LPMUpdater &U);
+};
+
+} // end namespace llvm
+
+#endif // LLVM_TRANSFORMS_SCALAR_LOOPTERMFOLD_H
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index dcde78925bfa984..2c26493bd3f1ca2 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -427,10 +427,6 @@ bool TargetTransformInfo::isNumRegsMajorCostOfLSR() const {
   return TTIImpl->isNumRegsMajorCostOfLSR();
 }
 
-bool TargetTransformInfo::shouldFoldTerminatingConditionAfterLSR() const {
-  return TTIImpl->shouldFoldTerminatingConditionAfterLSR();
-}
-
 bool TargetTransformInfo::shouldDropLSRSolutionIfLessProfitable() const {
   return TTIImpl->shouldDropLSRSolutionIfLessProfitable();
 }
diff --git a/llvm/lib/CodeGen/TargetPassConfig.cpp b/llvm/lib/CodeGen/TargetPassConfig.cpp
index 1b0012b65b80d48..1d52ebe6717f043 100644
--- a/llvm/lib/CodeGen/TargetPassConfig.cpp
+++ b/llvm/lib/CodeGen/TargetPassConfig.cpp
@@ -828,6 +828,8 @@ void TargetPassConfig::addIRPasses() {
     if (!DisableLSR) {
       addPass(createCanonicalizeFreezeInLoopsPass());
       addPass(createLoopStrengthReducePass());
+      if (EnableLoopTermFold)
+        addPass(createLoopTermFoldPass());
       if (PrintLSR)
         addPass(createPrintFunctionPass(dbgs(),
                                         "\n\n*** Code after LSR ***\n"));
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index 3200767282b226e..17eed97fd950c98 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -249,6 +249,7 @@
 #include "llvm/Transforms/Scalar/LoopSimplifyCFG.h"
 #include "llvm/Transforms/Scalar/LoopSink.h"
 #include "llvm/Transforms/Scalar/LoopStrengthReduce.h"
+#include "llvm/Transforms/Scalar/LoopTermFold.h"
 #include "llvm/Transforms/Scalar/LoopUnrollAndJamPass.h"
 #include "llvm/Transforms/Scalar/LoopUnrollPass.h"
 #include "llvm/Transforms/Scalar/LoopVersioningLICM.h"
diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def
index a11fc3755494abc..6b5e1cf83c4698b 100644
--- a/llvm/lib/Passes/PassRegistry.def
+++ b/llvm/lib/Passes/PassRegistry.def
@@ -646,6 +646,7 @@ LOOP_PASS("loop-idiom-vectorize", LoopIdiomVectorizePass())
 LOOP_PASS("loop-instsimplify", LoopInstSimplifyPass())
 LOOP_PASS("loop-predication", LoopPredicationPass())
 LOOP_PASS("loop-reduce", LoopStrengthReducePass())
+LOOP_PASS("loop-term-fold", LoopTermFoldPass())
 LOOP_PASS("loop-simplifycfg", LoopSimplifyCFGPass())
 LOOP_PASS("loop-unroll-full", LoopFullUnrollPass())
 LOOP_PASS("loop-versioning-licm", LoopVersioningLICMPass())
diff --git a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
index b6884321f084112..794df2212dfa534 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
@@ -336,6 +336,7 @@ class RISCVPassConfig : public TargetPassConfig {
     if (TM.getOptLevel() != CodeGenOptLevel::None)
       substitutePass(&PostRASchedulerID, &PostMachineSchedulerID);
     setEnableSinkAndFold(EnableSinkFold);
+    EnableLoopTermFold = true;
   }
 
   RISCVTargetMachine &getRISCVTargetMachine() const {
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
index f5eca2839acd058..cc69e1d118b5a18 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
@@ -394,9 +394,6 @@ class RISCVTTIImpl : public BasicTTIImplBase<RISCVTTIImpl> {
   bool isLSRCostLess(const TargetTransformInfo::LSRCost &C1,
                      const TargetTransformInfo::LSRCost &C2);
 
-  bool shouldFoldTerminatingConditionAfterLSR() const {
-    return true;
-  }
   bool
   shouldConsiderAddressTypePromotion(const Instruction &I,
                                      bool &AllowPromotionWithoutCommonHeader);
diff --git a/llvm/lib/Transforms/Scalar/CMakeLists.txt b/llvm/lib/Transforms/Scalar/CMakeLists.txt
index ba09ebf8b04c4cd..939a1457239567a 100644
--- a/llvm/lib/Transforms/Scalar/CMakeLists.txt
+++ b/llvm/lib/Transforms/Scalar/CMakeLists.txt
@@ -44,6 +44,7 @@ add_llvm_component_library(LLVMScalarOpts
   LoopRotation.cpp
   LoopSimplifyCFG.cpp
   LoopStrengthReduce.cpp
+  LoopTermFold.cpp
   LoopUnrollPass.cpp
   LoopUnrollAndJamPass.cpp
   LoopVersioningLICM.cpp
diff --git a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
index 91461d1ed27592c..a62b87fe2a53d46 100644
--- a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
@@ -189,10 +189,6 @@ static cl::opt<unsigned> SetupCostDepthLimit(
     "lsr-setupcost-depth-limit", cl::Hidden, cl::init(7),
     cl::desc("The limit on recursion depth for LSRs setup cost"));
 
-static cl::opt<cl::boolOrDefault> AllowTerminatingConditionFoldingAfterLSR(
-    "lsr-term-fold", cl::Hidden,
-    cl::desc("Attempt to replace primary IV with other IV."));
-
 static cl::opt<cl::boolOrDefault> AllowDropSolutionIfLessProfitable(
     "lsr-drop-solution", cl::Hidden,
     cl::desc("Attempt to drop solution if it is less profitable"));
@@ -205,9 +201,6 @@ static cl::opt<bool> DropScaledForVScale(
     "lsr-drop-scaled-reg-for-vscale", cl::Hidden, cl::init(true),
     cl::desc("Avoid using scaled registers with vscale-relative addressing"));
 
-STATISTIC(NumTermFold,
-          "Number of terminating condition fold recognized and performed");
-
 #ifndef NDEBUG
 // Stress test IV chain generation.
 static cl::opt<bool> StressIVChain(
@@ -7062,186 +7055,6 @@ static llvm::PHINode *GetInductionVariable(const Loop &L, ScalarEvolution &SE,
   return nullptr;
 }
 
-static std::optional<std::tuple<PHINode *, PHINode *, const SCEV *, bool>>
-canFoldTermCondOfLoop(Loop *L, ScalarEvolution &SE, DominatorTree &DT,
-                      const LoopInfo &LI, const TargetTransformInfo &TTI) {
-  if (!L->isInnermost()) {
-    LLVM_DEBUG(dbgs() << "Cannot fold on non-innermost loop\n");
-    return std::nullopt;
-  }
-  // Only inspect on simple loop structure
-  if (!L->isLoopSimplifyForm()) {
-    LLVM_DEBUG(dbgs() << "Cannot fold on non-simple loop\n");
-    return std::nullopt;
-  }
-
-  if (!SE.hasLoopInvariantBackedgeTakenCount(L)) {
-    LLVM_DEBUG(dbgs() << "Cannot fold on backedge that is loop variant\n");
-    return std::nullopt;
-  }
-
-  BasicBlock *LoopLatch = L->getLoopLatch();
-  BranchInst *BI = dyn_cast<BranchInst>(LoopLatch->getTerminator());
-  if (!BI || BI->isUnconditional())
-    return std::nullopt;
-  auto *TermCond = dyn_cast<ICmpInst>(BI->getCondition());
-  if (!TermCond) {
-    LLVM_DEBUG(
-        dbgs() << "Cannot fold on branching condition that is not an ICmpInst");
-    return std::nullopt;
-  }
-  if (!TermCond->hasOneUse()) {
-    LLVM_DEBUG(
-        dbgs()
-        << "Cannot replace terminating condition with more than one use\n");
-    return std::nullopt;
-  }
-
-  BinaryOperator *LHS = dyn_cast<BinaryOperator>(TermCond->getOperand(0));
-  Value *RHS = TermCond->getOperand(1);
-  if (!LHS || !L->isLoopInvariant(RHS))
-    // We could pattern match the inverse form of the icmp, but that is
-    // non-canonical, and this pass is running *very* late in the pipeline.
-    return std::nullopt;
-
-  // Find the IV used by the current exit condition.
-  PHINode *ToFold;
-  Value *ToFoldStart, *ToFoldStep;
-  if (!matchSimpleRecurrence(LHS, ToFold, ToFoldStart, ToFoldStep))
-    return std::nullopt;
-
-  // Ensure the simple recurrence is a part of the current loop.
-  if (ToFold->getParent() != L->getHeader())
-    return std::nullopt;
-
-  // If that IV isn't dead after we rewrite the exit condition in terms of
-  // another IV, there's no point in doing the transform.
-  if (!isAlmostDeadIV(ToFold, LoopLatch, TermCond))
-    return std::nullopt;
-
-  // Inserting instructions in the preheader has a runtime cost, scale
-  // the allowed cost with the loops trip count as best we can.
-  const unsigned ExpansionBudget = [&]() {
-    unsigned Budget = 2 * SCEVCheapExpansionBudget;
-    if (unsigned SmallTC = SE.getSmallConstantMaxTripCount(L))
-      return std::min(Budget, SmallTC);
-    if (std::optional<unsigned> SmallTC = getLoopEstimatedTripCount(L))
-      return std::min(Budget, *SmallTC);
-    // Unknown trip count, assume long running by default.
-    return Budget;
-  }();
-
-  const SCEV *BECount = SE.getBackedgeTakenCount(L);
-  const DataLayout &DL = L->getHeader()->getDataLayout();
-  SCEVExpander Expander(SE, DL, "lsr_fold_term_cond");
-
-  PHINode *ToHelpFold = nullptr;
-  const SCEV *TermValueS = nullptr;
-  bool MustDropPoison = false;
-  auto InsertPt = L->getLoopPreheader()->getTerminator();
-  for (PHINode &PN : L->getHeader()->phis()) {
-    if (ToFold == &PN)
-      continue;
-
-    if (!SE.isSCEVable(PN.getType())) {
-      LLVM_DEBUG(dbgs() << "IV of phi '" << PN
-                        << "' is not SCEV-able, not qualified for the "
-                           "terminating condition folding.\n");
-      continue;
-    }
-    const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(SE.getSCEV(&PN));
-    // Only speculate on affine AddRec
-    if (!AddRec || !AddRec->isAffine()) {
-      LLVM_DEBUG(dbgs() << "SCEV of phi '" << PN
-                        << "' is not an affine add recursion, not qualified "
-                           "for the terminating condition folding.\n");
-      continue;
-    }
-
-    // Check that we can compute the value of AddRec on the exiting iteration
-    // without soundness problems.  evaluateAtIteration internally needs
-    // to multiply the stride of the iteration number - which may wrap around.
-    // The issue here is subtle because computing the result accounting for
-    // wrap is insufficient. In order to use the result in an exit test, we
-    // must also know that AddRec doesn't take the same value on any previous
-    // iteration. The simplest case to consider is a candidate IV which is
-    // narrower than the trip count (and thus original IV), but this can
-    // also happen due to non-unit strides on the candidate IVs.
-    if (!AddRec->hasNoSelfWrap() ||
-        !SE.isKnownNonZero(AddRec->getStepRecurrence(SE)))
-      continue;
-
-    const SCEVAddRecExpr *PostInc = AddRec->getPostIncExpr(SE);
-    const SCEV *TermValueSLocal = PostInc->evaluateAtIteration(BECount, SE);
-    if (!Expander.isSafeToExpand(TermValueSLocal)) {
-      LLVM_DEBUG(
-          dbgs() << "Is not safe to expand terminating value for phi node" << PN
-                 << "\n");
-      continue;
-    }
-
-    if (Expander.isHighCostExpansion(TermValueSLocal, L, ExpansionBudget,
-                                     &TTI, InsertPt)) {
-      LLVM_DEBUG(
-          dbgs() << "Is too expensive to expand terminating value for phi node"
-                 << PN << "\n");
-      continue;
-    }
-
-    // The candidate IV may have been otherwise dead and poison from the
-    // very first iteration.  If we can't disprove that, we can't use the IV.
-    if (!mustExecuteUBIfPoisonOnPathTo(&PN, LoopLatch->getTerminator(), &DT)) {
-      LLVM_DEBUG(dbgs() << "Can not prove poison safety for IV "
-                        << PN << "\n");
-      continue;
-    }
-
-    // The candidate IV may become poison on the last iteration.  If this
-    // value is not branched on, this is a well defined program.  We're
-    // about to add a new use to this IV, and we have to ensure we don't
-    // insert UB which didn't previously exist.
-    bool MustDropPoisonLocal = false;
-    Instruction *PostIncV =
-      cast<Instruction>(PN.getIncomingValueForBlock(LoopLatch));
-    if (!mustExecuteUBIfPoisonOnPathTo(PostIncV, LoopLatch->getTerminator(),
-                                       &DT)) {
-      LLVM_DEBUG(dbgs() << "Can not prove poison safety to insert use"
-                        << PN << "\n");
-
-      // If this is a complex recurrance with multiple instructions computing
-      // the backedge value, we might need to strip poison flags from all of
-      // them.
-      if (PostIncV->getOperand(0) != &PN)
-        continue;
-
-      // In order to perform the transform, we need to drop the poison generating
-      // flags on this instruction (if any).
-      MustDropPoisonLocal = PostIncV->hasPoisonGeneratingFlags();
-    }
-
-    // We pick the last legal alternate IV.  We could expore choosing an optimal
-    // alternate IV if we had a decent heuristic to do so.
-    ToHelpFold = &PN;
-    TermValueS = TermValueSLocal;
-    MustDropPoison = MustDropPoisonLocal;
-  }
-
-  LLVM_DEBUG(if (ToFold && !ToHelpFold) dbgs()
-                 << "Cannot find other AddRec IV to help folding\n";);
-
-  LLVM_DEBUG(if (ToFold && ToHelpFold) dbgs()
-             << "\nFound loop that can fold terminating condition\n"
-             << "  BECount (SCEV): " << *SE.getBackedgeTakenCount(L) << "\n"
-             << "  TermCond: " << *TermCond << "\n"
-             << "  BrandInst: " << *BI << "\n"
-             << "  ToFold: " << *ToFold << "\n"
-             << "  ToHelpFold: " << *ToHelpFold << "\n");
-
-  if (!ToFold || !ToHelpFold)
-    return std::nullopt;
-  return std::make_tuple(ToFold, ToHelpFold, TermValueS, MustDropPoison);
-}
-
 static bool ReduceLoopStrength(Loop *L, IVUsers &IU, ScalarEvolution &SE,
                                DominatorTree &DT, LoopInfo &LI,
                                const TargetTransformInfo &TTI,
@@ -7302,81 +7115,6 @@ static bool ReduceLoopStrength(Loop *L, IVUsers &IU, ScalarEvolution &SE,
     }
   }
 
-  const bool EnableFormTerm = [&] {
-    switch (AllowTerminatingConditionFoldingAfterLSR) {
-    case cl::BOU_TRUE:
-      return true;
-    case cl::BOU_FALSE:
-      return false;
-    case cl::BOU_UNSET:
-      return TTI.shouldFoldTerminatingConditionAfterLSR();
-    }
-    llvm_unreachable("Unhandled cl::boolOrDefault enum");
-  }();
-
-  if (EnableFormTerm) {
-    if (auto Opt = canFoldTermCondOfLoop(L, SE, DT, LI, TTI)) {
-      auto [ToFold, ToHelpFold, TermValueS, MustDrop] = *Opt;
-
-      Changed = true;
-      NumTermFold++;
-
-      BasicBlock *LoopPreheader = L->getLoopPreheader();
-      BasicBlock *LoopLatch = L->getLoopLatch();
-
-      (void)ToFold;
-      LLVM_DEBUG(dbgs() << "To fold phi-node:\n"
-                        << *ToFold << "\n"
-                        << "New term-cond phi-node:\n"
-                        << *ToHelpFold << "\n");
-
-      Value *StartValue = ToHelpFold->getIncomingValueForBlock(LoopPreheader);
-      (void)StartValue;
-      Value *LoopValue = ToHelpFold->getIncomingValueForBlock(LoopLatch);
-
-      // See comment in canFoldTermCondOfLoop on why this is sufficient.
-      if (MustDrop)
-        cast<Instruction>(LoopValue)->dropPoisonGeneratingFlags();
-
-      // SCEVExpander for both use in preheader and latch
-      const DataLayout &DL = L->getHeader()->getDataLayout();
-      SCEVExpander Expander(SE, DL, "lsr_fold_term_cond");
-
-      assert(Expander.isSafeToExpand(TermValueS) &&
-             "Terminating value was checked safe in canFoldTerminatingCondition");
-
-      // Create new terminating value at loop preheader
-      Value *TermValue = Expander.expandCodeFor(TermValueS, ToHelpFold->getType(),
-                                                LoopPreheader->getTerminator());
-
-      LLVM_DEBUG(dbgs() << "Start value of new term-cond phi-node:\n"
-                        << *StartValue << "\n"
-                        << "Terminating value of new term-cond phi-node:\n"
-                        << *TermValue << "\n");
-
-      // Create new terminating condition at loop latch
-      BranchInst *BI = cast<BranchInst>(LoopLatch->getTerminator());
-      ICmpInst *OldTermCond = cast<ICmpInst>(BI->getCondition());
-      IRBuilder<> LatchBuilder(LoopLatch->getTerminator());
-      Value *NewTermCond =
-          LatchBuilder.CreateICmp(CmpInst::ICMP_EQ, LoopValue, TermValue,
-                                  "lsr_fold_term_cond.replaced_term_cond");
-      // Swap successors to exit loop body if IV equals to new TermValue
-      if (BI->getSuccessor(0) == L->getHeader())
-        BI->swapSuccessors();
-
-      LLVM_DEBUG(dbgs() << "Old term-cond:\n"
-                        << *OldTermCond << "\n"
-                        << "New term-cond:\n" << *NewTermCond << "\n");
-
-      BI->setCondition(NewTermCond);
-
-      Expander.clear();
-      OldTermCond->eraseFromParent();
-      DeleteDeadPHIs(L->getHeader(), &TLI, MSSAU.get());
-    }
-  }
-
   if (SalvageableDVIRecords.empty())
     return Changed;
 
diff --git a/llvm/lib/Transforms/Scalar/LoopTermFold.cpp b/llvm/lib/Transforms/Scalar/LoopTermFold.cpp
new file mode 100644
index 000000000000000..12ef367adc43e36
--- /dev/null
+++ b/llvm/lib/Transforms/Scalar/LoopTermFold.cpp
@@ -0,0 +1,379 @@
+//===- LoopTermFold.cpp - Eliminate last use of IV in exit branch----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/Scalar/LoopTermFold.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/Analysis/LoopAnalysisManager.h"
+#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/Analysis/LoopPass.h"
+#include "llvm/Analysis/MemorySSA.h"
+#include "llvm/Analysis/MemorySSAUpdater.h"
+#include "llvm/Analysis/ScalarEvolution.h"
+#include "llvm/Analysis/ScalarEvolutionExpressions.h"
+#include "llvm/Analysis/TargetLibraryInfo.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
+#include "llvm/Analysis/ValueTracking.h"
+#include "llvm/Config/llvm-config.h"
+#include "llvm/IR/BasicBlock.h"
+#include "llvm/IR/Dominators.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/InstrTypes.h"
+#include "llvm/IR/Instruction.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/Type.h"
+#include "llvm/IR/Value.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/Pass.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Transforms/Scalar.h"
+#include "llvm/Transforms/Utils.h"
+#include "llvm/Transforms/Utils/BasicBlockUtils.h"
+#include "llvm/Transforms/Utils/Local.h"
+#include "llvm/Transforms/Utils/LoopUtils.h"
+#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
+#include <cassert>
+#include <optional>
+#include <utility>
+
+using namespace llvm;
+
+#define DEBUG_TYPE "loop-term-fold"
+
+STATISTIC(NumTermFold,
+          "Number of terminating condition fold recognized and performed");
+
+static std::optional<std::tuple<PHINode *, PHINode *, const SCEV *, bool>>
+canFoldTermCondOfLoop(Loop *L, ScalarEvolution &SE, DominatorTree &DT,
+                      const LoopInfo &LI, const TargetTransformInfo &TTI) {
+  if (!L->isInnermost()) {
+    LLVM_DEBUG(dbgs() << "Cannot fold on non-innermost loop\n");
+    return std::nullopt;
+  }
+  // Only inspect on simple loop structure
+  if (!L->isLoopSimplifyForm()) {
+    LLVM_DEBUG(dbgs() << "Cannot fold on non-simple loop\n");
+    return std::nullopt;
+  }
+
+  if (!SE.hasLoopInvariantBackedgeTakenCount(L)) {
+    LLVM_DEBUG(dbgs() << "Cannot fold on backedge that is loop variant\n");
+    return std::nullopt;
+  }
+
+  BasicBlock *LoopLatch = L->getLoopLatch();
+  BranchInst *BI = dyn_cast<BranchInst>(LoopLatch->getTerminator());
+  if (!BI || BI->isUnconditional())
+    return std::nullopt;
+  auto *TermCond = dyn_cast<ICmpInst>(BI->getCondition());
+  if (!TermCond) {
+    LLVM_DEBUG(
+        dbgs() << "Cannot fold on branching condition that is not an ICmpInst");
+    return std::nullopt;
+  }
+  if (!TermCond->hasOneUse()) {
+    LLVM_DEBUG(
+        dbgs()
+        << "Cannot replace terminating condition with more than one use\n");
+    return std::nullopt;
+  }
+
+  BinaryOperator *LHS = dyn_cast<BinaryOperator>(TermCond->getOperand(0));
+  Value *RHS = TermCond->getOperand(1);
+  if (!LHS || !L->isLoopInvariant(RHS))
+    // We could pattern match the inverse form of the icmp, but that is
+    // non-canonical, and this pass is running *very* late in the pipeline.
+    return std::nullopt;
+
+  // Find the IV used by the current exit condition.
+  PHINode *ToFold;
+  Value *ToFoldStart, *ToFoldStep;
+  if (!matchSimpleRecurrence(LHS, ToFold, ToFoldStart, ToFoldStep))
+    return std::nullopt;
+
+  // Ensure the simple recurrence is a part of the current loop.
+  if (ToFold->getParent() != L->getHeader())
+    return std::nullopt;
+
+  // If that IV isn't dead after we rewrite the exit condition in terms of
+  // another IV, there's no point in doing the transform.
+  if (!isAlmostDeadIV(ToFold, LoopLatch, TermCond))
+    return std::nullopt;
+
+  // Inserting instructions in the preheader has a runtime cost, scale
+  // the allowed cost with the loops trip count as best we can.
+  const unsigned ExpansionBudget = [&]() {
+    unsigned Budget = 2 * SCEVCheapExpansionBudget;
+    if (unsigned SmallTC = SE.getSmallConstantMaxTripCount(L))
+      return std::min(Budget, SmallTC);
+    if (std::optional<unsigned> SmallTC = getLoopEstimatedTripCount(L))
+      return std::min(Budget, *SmallTC);
+    // Unknown trip count, assume long running by default.
+    return Budget;
+  }();
+
+  const SCEV *BECount = SE.getBackedgeTakenCount(L);
+  const DataLayout &DL = L->getHeader()->getDataLayout();
+  SCEVExpander Expander(SE, DL, "lsr_fold_term_cond");
+
+  PHINode *ToHelpFold = nullptr;
+  const SCEV *TermValueS = nullptr;
+  bool MustDropPoison = false;
+  auto InsertPt = L->getLoopPreheader()->getTerminator();
+  for (PHINode &PN : L->getHeader()->phis()) {
+    if (ToFold == &PN)
+      continue;
+
+    if (!SE.isSCEVable(PN.getType())) {
+      LLVM_DEBUG(dbgs() << "IV of phi '" << PN
+                        << "' is not SCEV-able, not qualified for the "
+                           "terminating condition folding.\n");
+      continue;
+    }
+    const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(SE.getSCEV(&PN));
+    // Only speculate on affine AddRec
+    if (!AddRec || !AddRec->isAffine()) {
+      LLVM_DEBUG(dbgs() << "SCEV of phi '" << PN
+                        << "' is not an affine add recursion, not qualified "
+                           "for the terminating condition folding.\n");
+      continue;
+    }
+
+    // Check that we can compute the value of AddRec on the exiting iteration
+    // without soundness problems.  evaluateAtIteration internally needs
+    // to multiply the stride of the iteration number - which may wrap around.
+    // The issue here is subtle because computing the result accounting for
+    // wrap is insufficient. In order to use the result in an exit test, we
+    // must also know that AddRec doesn't take the same value on any previous
+    // iteration. The simplest case to consider is a candidate IV which is
+    // narrower than the trip count (and thus original IV), but this can
+    // also happen due to non-unit strides on the candidate IVs.
+    if (!AddRec->hasNoSelfWrap() ||
+        !SE.isKnownNonZero(AddRec->getStepRecurrence(SE)))
+      continue;
+
+    const SCEVAddRecExpr *PostInc = AddRec->getPostIncExpr(SE);
+    const SCEV *TermValueSLocal = PostInc->evaluateAtIteration(BECount, SE);
+    if (!Expander.isSafeToExpand(TermValueSLocal)) {
+      LLVM_DEBUG(
+          dbgs() << "Is not safe to expand terminating value for phi node" << PN
+                 << "\n");
+      continue;
+    }
+
+    if (Expander.isHighCostExpansion(TermValueSLocal, L, ExpansionBudget, &TTI,
+                                     InsertPt)) {
+      LLVM_DEBUG(
+          dbgs() << "Is too expensive to expand terminating value for phi node"
+                 << PN << "\n");
+      continue;
+    }
+
+    // The candidate IV may have been otherwise dead and poison from the
+    // very first iteration.  If we can't disprove that, we can't use the IV.
+    if (!mustExecuteUBIfPoisonOnPathTo(&PN, LoopLatch->getTerminator(), &DT)) {
+      LLVM_DEBUG(dbgs() << "Can not prove poison safety for IV " << PN << "\n");
+      continue;
+    }
+
+    // The candidate IV may become poison on the last iteration.  If this
+    // value is not branched on, this is a well defined program.  We're
+    // about to add a new use to this IV, and we have to ensure we don't
+    // insert UB which didn't previously exist.
+    bool MustDropPoisonLocal = false;
+    Instruction *PostIncV =
+        cast<Instruction>(PN.getIncomingValueForBlock(LoopLatch));
+    if (!mustExecuteUBIfPoisonOnPathTo(PostIncV, LoopLatch->getTerminator(),
+                                       &DT)) {
+      LLVM_DEBUG(dbgs() << "Can not prove poison safety to insert use" << PN
+                        << "\n");
+
+      // If this is a complex recurrance with multiple instructions computing
+      // the backedge value, we might need to strip poison flags from all of
+      // them.
+      if (PostIncV->getOperand(0) != &PN)
+        continue;
+
+      // In order to perform the transform, we need to drop the poison
+      // generating flags on this instruction (if any).
+      MustDropPoisonLocal = PostIncV->hasPoisonGeneratingFlags();
+    }
+
+    // We pick the last legal alternate IV.  We could expore choosing an optimal
+    // alternate IV if we had a decent heuristic to do so.
+    ToHelpFold = &PN;
+    TermValueS = TermValueSLocal;
+    MustDropPoison = MustDropPoisonLocal;
+  }
+
+  LLVM_DEBUG(if (ToFold && !ToHelpFold) dbgs()
+                 << "Cannot find other AddRec IV to help folding\n";);
+
+  LLVM_DEBUG(if (ToFold && ToHelpFold) dbgs()
+             << "\nFound loop that can fold terminating condition\n"
+             << "  BECount (SCEV): " << *SE.getBackedgeTakenCount(L) << "\n"
+             << "  TermCond: " << *TermCond << "\n"
+             << "  BrandInst: " << *BI << "\n"
+             << "  ToFold: " << *ToFold << "\n"
+             << "  ToHelpFold: " << *ToHelpFold << "\n");
+
+  if (!ToFold || !ToHelpFold)
+    return std::nullopt;
+  return std::make_tuple(ToFold, ToHelpFold, TermValueS, MustDropPoison);
+}
+
+static bool RunTermFold(Loop *L, ScalarEvolution &SE, DominatorTree &DT,
+                        LoopInfo &LI, const TargetTransformInfo &TTI,
+                        TargetLibraryInfo &TLI, MemorySSA *MSSA) {
+  std::unique_ptr<MemorySSAUpdater> MSSAU;
+  if (MSSA)
+    MSSAU = std::make_unique<MemorySSAUpdater>(MSSA);
+
+  auto Opt = canFoldTermCondOfLoop(L, SE, DT, LI, TTI);
+  if (!Opt)
+    return false;
+
+  auto [ToFold, ToHelpFold, TermValueS, MustDrop] = *Opt;
+
+  NumTermFold++;
+
+  BasicBlock *LoopPreheader = L->getLoopPreheader();
+  BasicBlock *LoopLatch = L->getLoopLatch();
+
+  (void)ToFold;
+  LLVM_DEBUG(dbgs() << "To fold phi-node:\n"
+                    << *ToFold << "\n"
+                    << "New term-cond phi-node:\n"
+                    << *ToHelpFold << "\n");
+
+  Value *StartValue = ToHelpFold->getIncomingValueForBlock(LoopPreheader);
+  (void)StartValue;
+  Value *LoopValue = ToHelpFold->getIncomingValueForBlock(LoopLatch);
+
+  // See comment in canFoldTermCondOfLoop on why this is sufficient.
+  if (MustDrop)
+    cast<Instruction>(LoopValue)->dropPoisonGeneratingFlags();
+
+  // SCEVExpander for both use in preheader and latch
+  const DataLayout &DL = L->getHeader()->getDataLayout();
+  SCEVExpander Expander(SE, DL, "lsr_fold_term_cond");
+
+  assert(Expander.isSafeToExpand(TermValueS) &&
+         "Terminating value was checked safe in canFoldTerminatingCondition");
+
+  // Create new terminating value at loop preheader
+  Value *TermValue = Expander.expandCodeFor(TermValueS, ToHelpFold->getType(),
+                                            LoopPreheader->getTerminator());
+
+  LLVM_DEBUG(dbgs() << "Start value of new term-cond phi-node:\n"
+                    << *StartValue << "\n"
+                    << "Terminating value of new term-cond phi-node:\n"
+                    << *TermValue << "\n");
+
+  // Create new terminating condition at loop latch
+  BranchInst *BI = cast<BranchInst>(LoopLatch->getTerminator());
+  ICmpInst *OldTermCond = cast<ICmpInst>(BI->getCondition());
+  IRBuilder<> LatchBuilder(LoopLatch->getTerminator());
+  Value *NewTermCond =
+      LatchBuilder.CreateICmp(CmpInst::ICMP_EQ, LoopValue, TermValue,
+                              "lsr_fold_term_cond.replaced_term_cond");
+  // Swap successors to exit loop body if IV equals to new TermValue
+  if (BI->getSuccessor(0) == L->getHeader())
+    BI->swapSuccessors();
+
+  LLVM_DEBUG(dbgs() << "Old term-cond:\n"
+                    << *OldTermCond << "\n"
+                    << "New term-cond:\n"
+                    << *NewTermCond << "\n");
+
+  BI->setCondition(NewTermCond);
+
+  Expander.clear();
+  OldTermCond->eraseFromParent();
+  DeleteDeadPHIs(L->getHeader(), &TLI, MSSAU.get());
+  return true;
+}
+
+namespace {
+
+class LoopTermFold : public LoopPass {
+public:
+  static char ID; // Pass ID, replacement for typeid
+
+  LoopTermFold();
+
+private:
+  bool runOnLoop(Loop *L, LPPassManager &LPM) override;
+  void getAnalysisUsage(AnalysisUsage &AU) const override;
+};
+
+} // end anonymous namespace
+
+LoopTermFold::LoopTermFold() : LoopPass(ID) {
+  initializeLoopTermFoldPass(*PassRegistry::getPassRegistry());
+}
+
+void LoopTermFold::getAnalysisUsage(AnalysisUsage &AU) const {
+  AU.addRequired<LoopInfoWrapperPass>();
+  AU.addPreserved<LoopInfoWrapperPass>();
+  AU.addPreservedID(LoopSimplifyID);
+  AU.addRequiredID(LoopSimplifyID);
+  AU.addRequired<DominatorTreeWrapperPass>();
+  AU.addPreserved<DominatorTreeWrapperPass>();
+  AU.addRequired<ScalarEvolutionWrapperPass>();
+  AU.addPreserved<ScalarEvolutionWrapperPass>();
+  AU.addRequired<TargetLibraryInfoWrapperPass>();
+  AU.addRequired<TargetTransformInfoWrapperPass>();
+  AU.addPreserved<MemorySSAWrapperPass>();
+}
+
+bool LoopTermFold::runOnLoop(Loop *L, LPPassManager & /*LPM*/) {
+  if (skipLoop(L))
+    return false;
+
+  auto &SE = getAnalysis<ScalarEvolutionWrapperPass>().getSE();
+  auto &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
+  auto &LI = getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
+  const auto &TTI = getAnalysis<TargetTransformInfoWrapperPass>().getTTI(
+      *L->getHeader()->getParent());
+  auto &TLI = getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(
+      *L->getHeader()->getParent());
+  auto *MSSAAnalysis = getAnalysisIfAvailable<MemorySSAWrapperPass>();
+  MemorySSA *MSSA = nullptr;
+  if (MSSAAnalysis)
+    MSSA = &MSSAAnalysis->getMSSA();
+  return RunTermFold(L, SE, DT, LI, TTI, TLI, MSSA);
+}
+
+PreservedAnalyses LoopTermFoldPass::run(Loop &L, LoopAnalysisManager &AM,
+                                        LoopStandardAnalysisResults &AR,
+                                        LPMUpdater &) {
+  if (!RunTermFold(&L, AR.SE, AR.DT, AR.LI, AR.TTI, AR.TLI, AR.MSSA))
+    return PreservedAnalyses::all();
+
+  auto PA = getLoopPassPreservedAnalyses();
+  if (AR.MSSA)
+    PA.preserve<MemorySSAAnalysis>();
+  return PA;
+}
+
+char LoopTermFold::ID = 0;
+
+INITIALIZE_PASS_BEGIN(LoopTermFold, "loop-term-fold", "Loop Terminator Folding",
+                      false, false)
+INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(ScalarEvolutionWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(LoopSimplify)
+INITIALIZE_PASS_END(LoopTermFold, "loop-term-fold", "Loop Terminator Folding",
+                    false, false)
+
+Pass *llvm::createLoopTermFoldPass() { return new LoopTermFold(); }
diff --git a/llvm/lib/Transforms/Scalar/Scalar.cpp b/llvm/lib/Transforms/Scalar/Scalar.cpp
index 86669e8c5aa49b1..7aeee1d31f7e791 100644
--- a/llvm/lib/Transforms/Scalar/Scalar.cpp
+++ b/llvm/lib/Transforms/Scalar/Scalar.cpp
@@ -30,6 +30,7 @@ void llvm::initializeScalarOpts(PassRegistry &Registry) {
   initializeLegacyLICMPassPass(Registry);
   initializeLoopDataPrefetchLegacyPassPass(Registry);
   initializeLoopStrengthReducePass(Registry);
+  initializeLoopTermFoldPass(Registry);
   initializeLoopUnrollPass(Registry);
   initializeLowerAtomicLegacyPassPass(Registry);
   initializeMergeICmpsLegacyPassPass(Registry);
diff --git a/llvm/test/CodeGen/RISCV/O3-pipeline.ll b/llvm/test/CodeGen/RISCV/O3-pipeline.ll
index df9cb5de5d76825..44c270fdc3c2577 100644
--- a/llvm/test/CodeGen/RISCV/O3-pipeline.ll
+++ b/llvm/test/CodeGen/RISCV/O3-pipeline.ll
@@ -45,6 +45,7 @@
 ; CHECK-NEXT:         Canonicalize Freeze Instructions in Loops
 ; CHECK-NEXT:         Induction Variable Users
 ; CHECK-NEXT:         Loop Strength Reduction
+; CHECK-NEXT:         Loop Terminator Folding
 ; CHECK-NEXT:       Basic Alias Analysis (stateless AA impl)
 ; CHECK-NEXT:       Function Alias Analysis Results
 ; CHECK-NEXT:       Merge contiguous icmps into a memcmp
diff --git a/llvm/test/Transforms/LoopStrengthReduce/RISCV/lsr-cost-compare.ll b/llvm/test/Transforms/LoopStrengthReduce/RISCV/lsr-cost-compare.ll
index 9c11bd064ad47c3..cadee94ff409609 100644
--- a/llvm/test/Transforms/LoopStrengthReduce/RISCV/lsr-cost-compare.ll
+++ b/llvm/test/Transforms/LoopStrengthReduce/RISCV/lsr-cost-compare.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -loop-reduce -S | FileCheck %s
+; RUN: opt < %s -passes=loop-reduce,loop-term-fold -S | FileCheck %s
 
 target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n64-S128"
 target triple = "riscv64"
diff --git a/llvm/test/Transforms/LoopStrengthReduce/RISCV/term-fold-crash.ll b/llvm/test/Transforms/LoopStrengthReduce/RISCV/term-fold-crash.ll
index 8ca7f0010bbbe5f..9fb240684d232b4 100644
--- a/llvm/test/Transforms/LoopStrengthReduce/RISCV/term-fold-crash.ll
+++ b/llvm/test/Transforms/LoopStrengthReduce/RISCV/term-fold-crash.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
-; RUN: opt -S -passes=loop-reduce -mtriple=riscv64-unknown-linux-gnu < %s | FileCheck %s
+; RUN: opt -S -passes=loop-reduce,loop-term-fold -mtriple=riscv64-unknown-linux-gnu < %s | FileCheck %s
 
 define void @test(ptr %p, i8 %arg, i32 %start) {
 ; CHECK-LABEL: define void @test(
diff --git a/llvm/test/Transforms/LoopStrengthReduce/lsr-term-fold-negative-testcase.ll b/llvm/test/Transforms/LoopStrengthReduce/lsr-term-fold-negative-testcase.ll
index 2d3d3a4b72a1ac9..89ddba3343ffa24 100644
--- a/llvm/test/Transforms/LoopStrengthReduce/lsr-term-fold-negative-testcase.ll
+++ b/llvm/test/Transforms/LoopStrengthReduce/lsr-term-fold-negative-testcase.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
 ; REQUIRES: asserts
-; RUN: opt < %s -passes="loop-reduce" -S -debug -lsr-term-fold 2>&1 | FileCheck %s
+; RUN: opt < %s -passes=loop-reduce,loop-term-fold -S -debug 2>&1 | FileCheck %s
 
 target datalayout = "e-p:64:64:64-n64"
 
diff --git a/llvm/test/Transforms/LoopStrengthReduce/lsr-term-fold.ll b/llvm/test/Transforms/LoopStrengthReduce/lsr-term-fold.ll
index 7299a014b798351..6f34dc843ae1ee3 100644
--- a/llvm/test/Transforms/LoopStrengthReduce/lsr-term-fold.ll
+++ b/llvm/test/Transforms/LoopStrengthReduce/lsr-term-fold.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -passes="loop-reduce" -S -lsr-term-fold | FileCheck %s
+; RUN: opt < %s -passes="loop-reduce,loop-term-fold" -S | FileCheck %s
 
 target datalayout = "e-p:64:64:64-n64"
 
diff --git a/llvm/test/Transforms/LoopStrengthReduce/lsr-unreachable-bb-phi-node.ll b/llvm/test/Transforms/LoopStrengthReduce/lsr-unreachable-bb-phi-node.ll
index 1454535b52bccbb..67a71496e4cec8d 100644
--- a/llvm/test/Transforms/LoopStrengthReduce/lsr-unreachable-bb-phi-node.ll
+++ b/llvm/test/Transforms/LoopStrengthReduce/lsr-unreachable-bb-phi-node.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -loop-reduce -S -lsr-term-fold | FileCheck %s
+; RUN: opt < %s -passes=loop-reduce,loop-term-fold -S | FileCheck %s
 
 ; This test used to crash due to matchSimpleRecurrence matching the simple
 ; recurrence in pn-loop when evaluating unrelated-loop. Since unrelated-loop
@@ -13,9 +13,10 @@ define void @phi_node_different_bb() {
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp ugt i32 [[TMP2]], 1
 ; CHECK-NEXT:    br i1 [[TMP3]], label [[PN_LOOP]], label [[UNRELATED_LOOP_PREHEADER:%.*]]
 ; CHECK:       unrelated-loop.preheader:
+; CHECK-NEXT:    [[DOTLCSSA:%.*]] = phi i32 [ [[TMP2]], [[PN_LOOP]] ]
 ; CHECK-NEXT:    br label [[UNRELATED_LOOP:%.*]]
 ; CHECK:       unrelated-loop:
-; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[TMP2]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[DOTLCSSA]], 0
 ; CHECK-NEXT:    br i1 [[TMP4]], label [[END:%.*]], label [[UNRELATED_LOOP]]
 ; CHECK:       end:
 ; CHECK-NEXT:    ret void

From addeb22f109158921bae9d91fb76eda99081fa82 Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Sun, 18 Aug 2024 01:34:59 +0000
Subject: [PATCH 281/441] [gn build] Port 27a62ec72aed

---
 llvm/utils/gn/secondary/llvm/lib/Transforms/Scalar/BUILD.gn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/utils/gn/secondary/llvm/lib/Transforms/Scalar/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Transforms/Scalar/BUILD.gn
index f080c06f10dfe11..e297aa9da4e50d6 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Transforms/Scalar/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Transforms/Scalar/BUILD.gn
@@ -55,6 +55,7 @@ static_library("Scalar") {
     "LoopSimplifyCFG.cpp",
     "LoopSink.cpp",
     "LoopStrengthReduce.cpp",
+    "LoopTermFold.cpp",
     "LoopUnrollAndJamPass.cpp",
     "LoopUnrollPass.cpp",
     "LoopVersioningLICM.cpp",

From f499a3fe5fb6fd816d35b541e67a3761093cfded Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Sat, 17 Aug 2024 19:56:19 -0700
Subject: [PATCH 282/441] [VE] Use
 SelectionDAG::getSignedConstant/getAllOnesConstant.

---
 llvm/lib/Target/VE/VECustomDAG.cpp | 2 +-
 llvm/lib/Target/VE/VEInstrInfo.td  | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/llvm/lib/Target/VE/VECustomDAG.cpp b/llvm/lib/Target/VE/VECustomDAG.cpp
index 873ffea498de839..2855a65f654c96e 100644
--- a/llvm/lib/Target/VE/VECustomDAG.cpp
+++ b/llvm/lib/Target/VE/VECustomDAG.cpp
@@ -410,7 +410,7 @@ SDValue VECustomDAG::getConstantMask(Packing Packing, bool AllTrue) const {
   auto MaskVT = getLegalVectorType(Packing, MVT::i1);
 
   // VEISelDAGtoDAG will replace this pattern with the constant-true VM.
-  auto TrueVal = DAG.getConstant(-1, DL, MVT::i32);
+  auto TrueVal = DAG.getAllOnesConstant(DL, MVT::i32);
   auto AVL = getConstant(MaskVT.getVectorNumElements(), MVT::i32);
   auto Res = getNode(VEISD::VEC_BROADCAST, MaskVT, {TrueVal, AVL});
   if (AllTrue)
diff --git a/llvm/lib/Target/VE/VEInstrInfo.td b/llvm/lib/Target/VE/VEInstrInfo.td
index 75ef3b7336dbd5d..eb6a85298007935 100644
--- a/llvm/lib/Target/VE/VEInstrInfo.td
+++ b/llvm/lib/Target/VE/VEInstrInfo.td
@@ -44,8 +44,8 @@ def ULO7 : SDNodeXForm<imm, [{
                                    SDLoc(N), MVT::i32);
 }]>;
 def LO7 : SDNodeXForm<imm, [{
-  return CurDAG->getTargetConstant(SignExtend32(N->getSExtValue(), 7),
-                                   SDLoc(N), MVT::i32);
+  return CurDAG->getSignedConstant(SignExtend64(N->getSExtValue(), 7),
+                                   SDLoc(N), MVT::i32, /*isTarget=*/true);
 }]>;
 def MIMM : SDNodeXForm<imm, [{
   return CurDAG->getTargetConstant(val2MImm(getImmVal(N)),

From c4c9f39ba46b52ea88dffd54217f0205788751c6 Mon Sep 17 00:00:00 2001
From: Kareem Ergawy <kareem.ergawy@amd.com>
Date: Sun, 18 Aug 2024 06:33:09 +0200
Subject: [PATCH 283/441] [MLIR][OpenMP] Fix MLIR->LLVM value matching in
 privatization logic (#103718)

Fixes #102935

Updates matching logic for finding the LLVM value that corresponds to an
MLIR value. We need that matching to find the delayed privatizer for an
LLVM value being privatized.

The issue occures when there is an "indirect" correspondence between
MLIR and LLVM values: in some cases the values we are trying to match
stem from a pair of load/store ops that point to the same memref. This
PR adds such matching logic.
---
 .../OpenMP/OpenMPToLLVMIRTranslation.cpp      | 128 ++++++++++++++----
 .../Target/LLVMIR/openmp-firstprivate.mlir    |  42 ++++++
 2 files changed, 142 insertions(+), 28 deletions(-)

diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index 458d05d5059db70..1f3fb95c339c7c2 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -690,14 +690,14 @@ inlineOmpRegionCleanup(llvm::SmallVectorImpl<Region *> &cleanupRegions,
                                           : &builder.GetInsertBlock()->back();
     if (potentialTerminator && potentialTerminator->isTerminator())
       builder.SetInsertPoint(potentialTerminator);
-    llvm::Value *prviateVarValue =
+    llvm::Value *privateVarValue =
         shouldLoadCleanupRegionArg
             ? builder.CreateLoad(
                   moduleTranslation.convertType(entry.getArgument(0).getType()),
                   privateVariables[i])
             : privateVariables[i];
 
-    moduleTranslation.mapValue(entry.getArgument(0), prviateVarValue);
+    moduleTranslation.mapValue(entry.getArgument(0), privateVarValue);
 
     if (failed(inlineConvertOmpRegions(*cleanupRegion, regionName, builder,
                                        moduleTranslation)))
@@ -1424,35 +1424,106 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder,
     }
   };
 
-  SmallVector<omp::PrivateClauseOp> privatizerClones;
-  SmallVector<llvm::Value *> privateVariables;
+  SmallVector<omp::PrivateClauseOp> mlirPrivatizerClones;
+  SmallVector<llvm::Value *> llvmPrivateVars;
 
   // TODO: Perform appropriate actions according to the data-sharing
   // attribute (shared, private, firstprivate, ...) of variables.
   // Currently shared and private are supported.
   auto privCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP,
-                    llvm::Value &, llvm::Value &vPtr,
-                    llvm::Value *&replacementValue) -> InsertPointTy {
-    replacementValue = &vPtr;
+                    llvm::Value &, llvm::Value &llvmOmpRegionInput,
+                    llvm::Value *&llvmReplacementValue) -> InsertPointTy {
+    llvmReplacementValue = &llvmOmpRegionInput;
 
     // If this is a private value, this lambda will return the corresponding
     // mlir value and its `PrivateClauseOp`. Otherwise, empty values are
     // returned.
-    auto [privVar, privatizerClone] =
+    auto [mlirPrivVar, mlirPrivatizerClone] =
         [&]() -> std::pair<mlir::Value, omp::PrivateClauseOp> {
       if (!opInst.getPrivateVars().empty()) {
-        auto privateVars = opInst.getPrivateVars();
-        auto privateSyms = opInst.getPrivateSyms();
+        auto mlirPrivVars = opInst.getPrivateVars();
+        auto mlirPrivSyms = opInst.getPrivateSyms();
 
-        for (auto [privVar, privatizerAttr] :
-             llvm::zip_equal(privateVars, *privateSyms)) {
+        // Try to find a privatizer that corresponds to the LLVM value being
+        // privatized.
+        for (auto [mlirPrivVar, mlirPrivatizerAttr] :
+             llvm::zip_equal(mlirPrivVars, *mlirPrivSyms)) {
           // Find the MLIR private variable corresponding to the LLVM value
           // being privatized.
-          llvm::Value *llvmPrivVar = moduleTranslation.lookupValue(privVar);
-          if (llvmPrivVar != &vPtr)
+          llvm::Value *mlirToLLVMPrivVar =
+              moduleTranslation.lookupValue(mlirPrivVar);
+
+          // Check if the LLVM value being privatized matches the LLVM value
+          // mapped to privVar. In some cases, this is not trivial ...
+          auto isMatch = [&]() {
+            if (mlirToLLVMPrivVar == nullptr)
+              return false;
+
+            // If both values are trivially equal, we found a match.
+            if (mlirToLLVMPrivVar == &llvmOmpRegionInput)
+              return true;
+
+            // Otherwise, we check if both llvmOmpRegionInputPtr and
+            // mlirToLLVMPrivVar refer to the same memory (through a load/store
+            // pair). This happens if a struct (i.e. multi-field value) is being
+            // privatized.
+            //
+            // For example, if the privatized value is defined by:
+            // ```
+            //   %priv_val = alloca { ptr, i64 }, align 8
+            // ```
+            //
+            // The initialization of this value (outside the omp region) will be
+            // something like this:
+            //
+            // clang-format off
+            // ```
+            //   %partially_init_priv_val = insertvalue { ptr, i64 } undef,
+            //                              ptr %some_ptr, 0
+            //   %fully_init_priv_val = insertvalue { ptr, i64 } %partially_init_priv_val,
+            //                          i64 %some_i64, 1
+            //   ...
+            //   store { ptr, i64 } %fully_init_priv_val, ptr %priv_val, align 8
+            // ```
+            // clang-format on
+            //
+            // Now, we enter the OMP region, in order to access this privatized
+            // value, we need to load from the allocated memory:
+            // ```
+            // omp.par.entry:
+            //   %priv_val_load = load { ptr, i64 }, ptr %priv_val, align 8
+            // ```
+            //
+            // The 2 LLVM values tracked here map as follows:
+            // - `mlirToLLVMPrivVar`     -> `%fully_init_priv_val`
+            // - `llvmOmpRegionInputPtr` -> `%priv_val_load`
+            //
+            // Even though they eventually refer to the same memory reference
+            // (the memory being privatized), they are 2 distinct LLVM values.
+            // Therefore, we need to discover their correspondence by finding
+            // out if they store into and load from the same mem ref.
+            auto *llvmOmpRegionInputPtrLoad =
+                llvm::dyn_cast_if_present<llvm::LoadInst>(&llvmOmpRegionInput);
+
+            if (llvmOmpRegionInputPtrLoad == nullptr)
+              return false;
+
+            for (auto &use : mlirToLLVMPrivVar->uses()) {
+              auto *mlirToLLVMPrivVarStore =
+                  llvm::dyn_cast_if_present<llvm::StoreInst>(use.getUser());
+              if (mlirToLLVMPrivVarStore &&
+                  (llvmOmpRegionInputPtrLoad->getPointerOperand() ==
+                   mlirToLLVMPrivVarStore->getPointerOperand()))
+                return true;
+            }
+
+            return false;
+          };
+
+          if (!isMatch())
             continue;
 
-          SymbolRefAttr privSym = llvm::cast<SymbolRefAttr>(privatizerAttr);
+          SymbolRefAttr privSym = llvm::cast<SymbolRefAttr>(mlirPrivatizerAttr);
           omp::PrivateClauseOp privatizer =
               SymbolTable::lookupNearestSymbolFrom<omp::PrivateClauseOp>(
                   opInst, privSym);
@@ -1480,24 +1551,24 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder,
               counter);
 
           clone.setSymName(cloneName);
-          return {privVar, clone};
+          return {mlirPrivVar, clone};
         }
       }
 
       return {mlir::Value(), omp::PrivateClauseOp()};
     }();
 
-    if (privVar) {
-      Region &allocRegion = privatizerClone.getAllocRegion();
+    if (mlirPrivVar) {
+      Region &allocRegion = mlirPrivatizerClone.getAllocRegion();
 
       // If this is a `firstprivate` clause, prepare the `omp.private` op by:
-      if (privatizerClone.getDataSharingType() ==
+      if (mlirPrivatizerClone.getDataSharingType() ==
           omp::DataSharingClauseType::FirstPrivate) {
         auto oldAllocBackBlock = std::prev(allocRegion.end());
         omp::YieldOp oldAllocYieldOp =
             llvm::cast<omp::YieldOp>(oldAllocBackBlock->getTerminator());
 
-        Region &copyRegion = privatizerClone.getCopyRegion();
+        Region &copyRegion = mlirPrivatizerClone.getCopyRegion();
 
         mlir::IRRewriter copyCloneBuilder(&moduleTranslation.getContext());
         // 1. Cloning the `copy` region to the end of the `alloc` region.
@@ -1524,7 +1595,7 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder,
       // This way, the body of the privatizer will be changed from using the
       // region/block argument to the value being privatized.
       auto allocRegionArg = allocRegion.getArgument(0);
-      replaceAllUsesInRegionWith(allocRegionArg, privVar, allocRegion);
+      replaceAllUsesInRegionWith(allocRegionArg, mlirPrivVar, allocRegion);
 
       auto oldIP = builder.saveIP();
       builder.restoreIP(allocaIP);
@@ -1535,15 +1606,15 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder,
         opInst.emitError("failed to inline `alloc` region of an `omp.private` "
                          "op in the parallel region");
         bodyGenStatus = failure();
-        privatizerClone.erase();
+        mlirPrivatizerClone.erase();
       } else {
         assert(yieldedValues.size() == 1);
-        replacementValue = yieldedValues.front();
+        llvmReplacementValue = yieldedValues.front();
 
         // Keep the LLVM replacement value and the op clone in case we need to
         // emit cleanup (i.e. deallocation) logic.
-        privateVariables.push_back(replacementValue);
-        privatizerClones.push_back(privatizerClone);
+        llvmPrivateVars.push_back(llvmReplacementValue);
+        mlirPrivatizerClones.push_back(mlirPrivatizerClone);
       }
 
       builder.restoreIP(oldIP);
@@ -1571,13 +1642,14 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder,
       bodyGenStatus = failure();
 
     SmallVector<Region *> privateCleanupRegions;
-    llvm::transform(privatizerClones, std::back_inserter(privateCleanupRegions),
+    llvm::transform(mlirPrivatizerClones,
+                    std::back_inserter(privateCleanupRegions),
                     [](omp::PrivateClauseOp privatizer) {
                       return &privatizer.getDeallocRegion();
                     });
 
     if (failed(inlineOmpRegionCleanup(
-            privateCleanupRegions, privateVariables, moduleTranslation, builder,
+            privateCleanupRegions, llvmPrivateVars, moduleTranslation, builder,
             "omp.private.dealloc", /*shouldLoadCleanupRegionArg=*/false)))
       bodyGenStatus = failure();
 
@@ -1604,7 +1676,7 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder,
       ompBuilder->createParallel(ompLoc, allocaIP, bodyGenCB, privCB, finiCB,
                                  ifCond, numThreads, pbKind, isCancellable));
 
-  for (mlir::omp::PrivateClauseOp privatizerClone : privatizerClones)
+  for (mlir::omp::PrivateClauseOp privatizerClone : mlirPrivatizerClones)
     privatizerClone.erase();
 
   return bodyGenStatus;
diff --git a/mlir/test/Target/LLVMIR/openmp-firstprivate.mlir b/mlir/test/Target/LLVMIR/openmp-firstprivate.mlir
index 65ae98b2a74c6ec..b06ad96f4592c5f 100644
--- a/mlir/test/Target/LLVMIR/openmp-firstprivate.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-firstprivate.mlir
@@ -114,3 +114,45 @@ omp.private {type = firstprivate} @multi_block.privatizer : !llvm.ptr alloc {
   llvm.store %arg2, %arg3 : f32, !llvm.ptr
   omp.yield(%arg3 : !llvm.ptr)
 }
+
+// -----
+
+// Verifies fix for https://github.com/llvm/llvm-project/issues/102935.
+//
+// The issue happens since we previously failed to match MLIR values to their
+// corresponding LLVM values in some cases (e.g. char strings with non-const
+// length).
+llvm.func @non_const_len_char_test(%n: !llvm.ptr {fir.bindc_name = "n"}) {
+  %n_val = llvm.load %n : !llvm.ptr -> i64
+  %orig_alloc = llvm.alloca %n_val x i8 {bindc_name = "str"} : (i64) -> !llvm.ptr
+  %orig_val = llvm.mlir.undef : !llvm.struct<(ptr, i64)>
+  %orig_val_init = llvm.insertvalue %orig_alloc, %orig_val[0] : !llvm.struct<(ptr, i64)>
+  omp.parallel private(@non_const_len_char %orig_val_init -> %priv_arg : !llvm.struct<(ptr, i64)>) {
+    %dummy = llvm.extractvalue %priv_arg[0] : !llvm.struct<(ptr, i64)>
+    omp.terminator
+  }
+  llvm.return
+}
+
+omp.private {type = firstprivate} @non_const_len_char : !llvm.struct<(ptr, i64)> alloc {
+^bb0(%orig_val: !llvm.struct<(ptr, i64)>):
+  %str_len = llvm.extractvalue %orig_val[1] : !llvm.struct<(ptr, i64)>
+  %priv_alloc = llvm.alloca %str_len x i8 {bindc_name = "str", pinned} : (i64) -> !llvm.ptr
+  %priv_val = llvm.mlir.undef : !llvm.struct<(ptr, i64)>
+  %priv_val_init = llvm.insertvalue %priv_alloc, %priv_val[0] : !llvm.struct<(ptr, i64)>
+  omp.yield(%priv_val_init : !llvm.struct<(ptr, i64)>)
+} copy {
+^bb0(%orig_val: !llvm.struct<(ptr, i64)>, %priv_val: !llvm.struct<(ptr, i64)>):
+  llvm.call @foo() : () -> ()
+  omp.yield(%priv_val : !llvm.struct<(ptr, i64)>)
+}
+
+llvm.func @foo()
+
+// CHECK-LABEL: @non_const_len_char_test..omp_par({{.*}})
+// CHECK-NEXT:    omp.par.entry:
+// Verify that we found the privatizer by checking that we properly inlined the
+// bodies of the alloc and copy regions.
+// CHECK:         %[[STR_LEN:.*]] = extractvalue { ptr, i64 } %{{.*}}, 1
+// CHECK:         %{{.*}} = alloca i8, i64 %[[STR_LEN]], align 1
+// CHECK:         call void @foo()

From 93ab50ab3530d82579062fc471f5e91c42eacdde Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Sat, 17 Aug 2024 22:18:35 -0700
Subject: [PATCH 284/441] [APInt] Replace enum with static constexpr member
 variables. NFC

With C++17 we no longer need the enum to prevent ODR use.
---
 llvm/include/llvm/ADT/APInt.h | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/llvm/include/llvm/ADT/APInt.h b/llvm/include/llvm/ADT/APInt.h
index 108df7e0eaeaa33..65ba3f15305c78f 100644
--- a/llvm/include/llvm/ADT/APInt.h
+++ b/llvm/include/llvm/ADT/APInt.h
@@ -79,13 +79,11 @@ class [[nodiscard]] APInt {
 public:
   typedef uint64_t WordType;
 
-  /// This enum is used to hold the constants we needed for APInt.
-  enum : unsigned {
-    /// Byte size of a word.
-    APINT_WORD_SIZE = sizeof(WordType),
-    /// Bits in a word.
-    APINT_BITS_PER_WORD = APINT_WORD_SIZE * CHAR_BIT
-  };
+  /// Byte size of a word.
+  static constexpr unsigned APINT_WORD_SIZE = sizeof(WordType);
+
+  /// Bits in a word.
+  static constexpr unsigned APINT_BITS_PER_WORD = APINT_WORD_SIZE * CHAR_BIT;
 
   enum class Rounding {
     DOWN,

From 88f801037338a2c273a8f6dea8ebcb0c48a2cc06 Mon Sep 17 00:00:00 2001
From: Mike Crowe <mac@mcrowe.com>
Date: Sun, 18 Aug 2024 06:58:25 +0100
Subject: [PATCH 285/441] [clang-tidy] Correct typo in ReleaseNotes.rst
 (#104674)

---
 clang-tools-extra/docs/ReleaseNotes.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst
index b72d109b3d3938b..0f6b34cf4005215 100644
--- a/clang-tools-extra/docs/ReleaseNotes.rst
+++ b/clang-tools-extra/docs/ReleaseNotes.rst
@@ -106,7 +106,7 @@ Changes in existing checks
 
 - Improved :doc:`readability-redundant-smartptr-get
   <clang-tidy/checks/readability/redundant-smartptr-get>` check to
-  remove `->`, when reduntant `get()` is removed.
+  remove `->`, when redundant `get()` is removed.
 
 Removed checks
 ^^^^^^^^^^^^^^

From e994494a590ea14d4bc9bf14b6cd7b8bc7ce06b6 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Sat, 17 Aug 2024 23:12:52 -0700
Subject: [PATCH 286/441] [PowerPC] Use MathExtras helpers to simplify code.
 NFC (#104691)

---
 llvm/lib/Target/PowerPC/PPCISelLowering.cpp | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index fff39a37265f20f..459a96eca1ff202 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -9513,10 +9513,8 @@ SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op,
     } else {
       // We may lose precision, so we have to use XXSPLTI32DX.
 
-      uint32_t Hi =
-          (uint32_t)((APSplatBits.getZExtValue() & 0xFFFFFFFF00000000LL) >> 32);
-      uint32_t Lo =
-          (uint32_t)(APSplatBits.getZExtValue() & 0xFFFFFFFF);
+      uint32_t Hi = Hi_32(APSplatBits.getZExtValue());
+      uint32_t Lo = Lo_32(APSplatBits.getZExtValue());
       SDValue SplatNode = DAG.getUNDEF(MVT::v2i64);
 
       if (!Hi || !Lo)
@@ -9660,8 +9658,7 @@ SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op,
                                   dl);
 
   // If the sign extended value is in the range [-16,15], use VSPLTI[bhw].
-  int32_t SextVal= (int32_t(SplatBits << (32-SplatBitSize)) >>
-                    (32-SplatBitSize));
+  int32_t SextVal = SignExtend32(SplatBits, SplatBitSize);
   if (SextVal >= -16 && SextVal <= 15)
     return getCanonicalConstSplat(SextVal, SplatSize, Op.getValueType(), DAG,
                                   dl);

From e59c8241fa6a5a9c8f9175b0ed7b0dfdb5c945cf Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Sat, 17 Aug 2024 23:41:09 -0700
Subject: [PATCH 287/441] [RISCV] Remove unused tablegen classes from
 unratified Zbp instructions. NFC

These weren't removed when we removed Zbp.
---
 .../Target/RISCV/AsmParser/RISCVAsmParser.cpp |  4 ----
 llvm/lib/Target/RISCV/RISCVInstrInfoZb.td     | 24 -------------------
 2 files changed, 28 deletions(-)

diff --git a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
index 5ade54dc3fccd49..f299e823aa3f500 100644
--- a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
+++ b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
@@ -1483,10 +1483,6 @@ bool RISCVAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
     if (isRV64())
       return generateImmOutOfRangeError(Operands, ErrorInfo, 1, (1 << 6) - 1);
     return generateImmOutOfRangeError(Operands, ErrorInfo, 1, (1 << 5) - 1);
-  case Match_InvalidUImmLog2XLenHalf:
-    if (isRV64())
-      return generateImmOutOfRangeError(Operands, ErrorInfo, 0, (1 << 5) - 1);
-    return generateImmOutOfRangeError(Operands, ErrorInfo, 0, (1 << 4) - 1);
   case Match_InvalidUImm1:
     return generateImmOutOfRangeError(Operands, ErrorInfo, 0, (1 << 1) - 1);
   case Match_InvalidUImm2:
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td
index 0cc0605ea870148..f27039b1c658d20 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td
@@ -45,30 +45,6 @@ def riscv_clmul   : SDNode<"RISCVISD::CLMUL",   SDTIntBinOp>;
 def riscv_clmulh  : SDNode<"RISCVISD::CLMULH",  SDTIntBinOp>;
 def riscv_clmulr  : SDNode<"RISCVISD::CLMULR",  SDTIntBinOp>;
 
-def UImmLog2XLenHalfAsmOperand : AsmOperandClass {
-  let Name = "UImmLog2XLenHalf";
-  let RenderMethod = "addImmOperands";
-  let DiagnosticType = "InvalidUImmLog2XLenHalf";
-}
-
-def shfl_uimm : RISCVOp, ImmLeaf<XLenVT, [{
-  if (Subtarget->is64Bit())
-    return isUInt<5>(Imm);
-  return isUInt<4>(Imm);
-}]> {
-  let ParserMatchClass = UImmLog2XLenHalfAsmOperand;
-  let DecoderMethod = "decodeUImmOperand<5>";
-  let OperandType = "OPERAND_UIMM_SHFL";
-  let MCOperandPredicate = [{
-    int64_t Imm;
-    if (!MCOp.evaluateAsConstantImm(Imm))
-      return false;
-    if (STI.getTargetTriple().isArch64Bit())
-      return  isUInt<5>(Imm);
-    return isUInt<4>(Imm);
-  }];
-}
-
 def BCLRXForm : SDNodeXForm<imm, [{
   // Find the lowest 0.
   return CurDAG->getTargetConstant(llvm::countr_one(N->getZExtValue()),

From 07bd3bb9b7eb34426a81de2b988f53f08611ab35 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timm=20B=C3=A4der?= <tbaeder@redhat.com>
Date: Sun, 18 Aug 2024 00:29:48 +0200
Subject: [PATCH 288/441] [clang][bytecode][NFC] Improve Pointer::print()

Do not access PointeeStorage.BS.Pointee if we have a non-block pointer
and extend printing to handle function pointers as well.
---
 clang/lib/AST/ByteCode/Pointer.cpp | 21 ++++++++++++++-------
 1 file changed, 14 insertions(+), 7 deletions(-)

diff --git a/clang/lib/AST/ByteCode/Pointer.cpp b/clang/lib/AST/ByteCode/Pointer.cpp
index 466e61666c76e92..e39459578a5f526 100644
--- a/clang/lib/AST/ByteCode/Pointer.cpp
+++ b/clang/lib/AST/ByteCode/Pointer.cpp
@@ -265,10 +265,10 @@ APValue Pointer::toAPValue(const ASTContext &ASTCtx) const {
 }
 
 void Pointer::print(llvm::raw_ostream &OS) const {
-  OS << PointeeStorage.BS.Pointee << " (";
-  if (isBlockPointer()) {
+  switch (StorageKind) {
+  case Storage::Block: {
     const Block *B = PointeeStorage.BS.Pointee;
-    OS << "Block) {";
+    OS << "(Block) " << B << " {";
 
     if (isRoot())
       OS << "rootptr(" << PointeeStorage.BS.Base << "), ";
@@ -284,11 +284,18 @@ void Pointer::print(llvm::raw_ostream &OS) const {
       OS << B->getSize();
     else
       OS << "nullptr";
-  } else {
-    OS << "Int) {";
-    OS << PointeeStorage.Int.Value << ", " << PointeeStorage.Int.Desc;
+    OS << "}";
+  } break;
+  case Storage::Int:
+    OS << "(Int) {";
+    OS << PointeeStorage.Int.Value << " + " << Offset << ", "
+       << PointeeStorage.Int.Desc;
+    OS << "}";
+    break;
+  case Storage::Fn:
+    OS << "(Fn) { " << asFunctionPointer().getFunction() << " + " << Offset
+       << " }";
   }
-  OS << "}";
 }
 
 std::string Pointer::toDiagnosticString(const ASTContext &Ctx) const {

From dac182990dabe8d15cfb8079aba68df2ded015aa Mon Sep 17 00:00:00 2001
From: Timm Baeder <tbaeder@redhat.com>
Date: Sun, 18 Aug 2024 08:59:34 +0200
Subject: [PATCH 289/441] [clang][bytecode] IntPointer::atOffset() should
 append (#104686)

... to current offset. This breaks other tests which this commit also
fixes. Namely, getIndex() should return the integer representation for
non-block pointers.
---
 clang/lib/AST/ByteCode/Interp.h    | 11 +++++++++++
 clang/lib/AST/ByteCode/Pointer.cpp |  2 +-
 clang/lib/AST/ByteCode/Pointer.h   |  2 +-
 clang/test/AST/ByteCode/codegen.c  | 19 +++++++++++++++++++
 4 files changed, 32 insertions(+), 2 deletions(-)
 create mode 100644 clang/test/AST/ByteCode/codegen.c

diff --git a/clang/lib/AST/ByteCode/Interp.h b/clang/lib/AST/ByteCode/Interp.h
index 9891e3dba0d309e..6cb7a42482ab25b 100644
--- a/clang/lib/AST/ByteCode/Interp.h
+++ b/clang/lib/AST/ByteCode/Interp.h
@@ -1853,6 +1853,17 @@ bool OffsetHelper(InterpState &S, CodePtr OpPC, const T &Offset,
   if (!CheckArray(S, OpPC, Ptr))
     return false;
 
+  // This is much simpler for integral pointers, so handle them first.
+  if (Ptr.isIntegralPointer()) {
+    uint64_t V = Ptr.getIntegerRepresentation();
+    uint64_t O = static_cast<uint64_t>(Offset) * Ptr.elemSize();
+    if constexpr (Op == ArithOp::Add)
+      S.Stk.push<Pointer>(V + O, Ptr.asIntPointer().Desc);
+    else
+      S.Stk.push<Pointer>(V - O, Ptr.asIntPointer().Desc);
+    return true;
+  }
+
   uint64_t MaxIndex = static_cast<uint64_t>(Ptr.getNumElems());
   uint64_t Index;
   if (Ptr.isOnePastEnd())
diff --git a/clang/lib/AST/ByteCode/Pointer.cpp b/clang/lib/AST/ByteCode/Pointer.cpp
index e39459578a5f526..5b9e83764cfa503 100644
--- a/clang/lib/AST/ByteCode/Pointer.cpp
+++ b/clang/lib/AST/ByteCode/Pointer.cpp
@@ -654,5 +654,5 @@ IntPointer IntPointer::atOffset(const ASTContext &ASTCtx,
   uint64_t FieldOffset =
       ASTCtx.toCharUnitsFromBits(Layout.getFieldOffset(FieldIndex))
           .getQuantity();
-  return IntPointer{this->Desc, FieldOffset};
+  return IntPointer{this->Desc, this->Value + FieldOffset};
 }
diff --git a/clang/lib/AST/ByteCode/Pointer.h b/clang/lib/AST/ByteCode/Pointer.h
index 8db081c0ec82b33..ba30449977376bb 100644
--- a/clang/lib/AST/ByteCode/Pointer.h
+++ b/clang/lib/AST/ByteCode/Pointer.h
@@ -571,7 +571,7 @@ class Pointer {
   /// Returns the index into an array.
   int64_t getIndex() const {
     if (!isBlockPointer())
-      return 0;
+      return getIntegerRepresentation();
 
     if (isZero())
       return 0;
diff --git a/clang/test/AST/ByteCode/codegen.c b/clang/test/AST/ByteCode/codegen.c
new file mode 100644
index 000000000000000..8434992823010e7
--- /dev/null
+++ b/clang/test/AST/ByteCode/codegen.c
@@ -0,0 +1,19 @@
+// RUN: %clang_cc1 -triple x86_64-linux -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple x86_64-linux -emit-llvm -o - %s -fexperimental-new-constant-interpreter | FileCheck %s
+
+typedef __INTPTR_TYPE__ intptr_t;
+
+const intptr_t Z1 = (intptr_t)(((char*)-1LL) + 1);
+// CHECK: @Z1 = constant i64 0
+
+const intptr_t Z2 = (intptr_t)(((char*)1LL) - 1);
+// CHECK: @Z2 = constant i64 0
+
+struct A {
+  char num_fields;
+};
+struct B {
+  char a, b[1];
+};
+const int A = (char *)(&( (struct B *)(16) )->b[0]) - (char *)(16);
+// CHECK: @A = constant i32 1

From 11259343593043c77678b59d420159fcd147a858 Mon Sep 17 00:00:00 2001
From: Mital Ashok <mital@mitalashok.co.uk>
Date: Sun, 18 Aug 2024 10:50:42 +0100
Subject: [PATCH 290/441] [Clang] `constexpr` builtin floating point
 classification / comparison functions (#94118)

As per [P0533R9](https://wg21.link/P0533R9), the corresponding C++
`[c.math.fpclass]` standard library functions for the C macros are now
`constexpr`.

The only classification function that wasn't already `constexpr` was
`__builtin_signbit`.
The floating point comparison functions `__builtin_isgreater`,
`__builtin_isgreaterequal`, `__builtin_isless`, `__builtin_islessequal`,
`__builtin_islessgreater` and `__builtin_isunordered` are now
`constexpr`.
The C23 macro `iseqsig` is not currently supported because
`__bulitin_iseqsig` doesn't exist yet (and C++26 is still currently
based on C18).

This also allows them to be constant folded in C, matching the behaviour
of GCC.
---
 clang/docs/ReleaseNotes.rst              |   4 +
 clang/include/clang/Basic/Builtins.td    |  20 ++--
 clang/lib/AST/ByteCode/InterpBuiltin.cpp |  60 +++++++++++
 clang/lib/AST/ExprConstant.cpp           |  48 +++++++++
 clang/test/Analysis/builtin_signbit.cpp  | 126 ++++++++++-------------
 clang/test/Sema/constant-builtins-2.c    |  48 +++++++++
 6 files changed, 224 insertions(+), 82 deletions(-)

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index ffdd063ec99037a..12a3707db8a39f7 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -161,6 +161,10 @@ C23 Feature Support
 Non-comprehensive list of changes in this release
 -------------------------------------------------
 
+- The floating point comparison builtins (``__builtin_isgreater``,
+  ``__builtin_isgreaterequal``, ``__builtin_isless``, etc.) and
+  ``__builtin_signbit`` can now be used in constant expressions.
+
 New Compiler Flags
 ------------------
 
diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td
index 0a874d8638df435..036366cdadf4aa2 100644
--- a/clang/include/clang/Basic/Builtins.td
+++ b/clang/include/clang/Basic/Builtins.td
@@ -533,42 +533,42 @@ def BuiltinComplex : Builtin {
 def IsGreater : Builtin {
   let Spellings = ["__builtin_isgreater"];
   let Attributes = [FunctionWithBuiltinPrefix, NoThrow, Const,
-                    CustomTypeChecking];
+                    CustomTypeChecking, Constexpr];
   let Prototype = "int(...)";
 }
 
 def IsGreaterEqual : Builtin {
   let Spellings = ["__builtin_isgreaterequal"];
   let Attributes = [FunctionWithBuiltinPrefix, NoThrow, Const,
-                    CustomTypeChecking];
+                    CustomTypeChecking, Constexpr];
   let Prototype = "int(...)";
 }
 
 def IsLess : Builtin {
   let Spellings = ["__builtin_isless"];
   let Attributes = [FunctionWithBuiltinPrefix, NoThrow, Const,
-                    CustomTypeChecking];
+                    CustomTypeChecking, Constexpr];
   let Prototype = "int(...)";
 }
 
 def IsLessEqual : Builtin {
   let Spellings = ["__builtin_islessequal"];
   let Attributes = [FunctionWithBuiltinPrefix, NoThrow, Const,
-                    CustomTypeChecking];
+                    CustomTypeChecking, Constexpr];
   let Prototype = "int(...)";
 }
 
 def IsLessGreater : Builtin {
   let Spellings = ["__builtin_islessgreater"];
   let Attributes = [FunctionWithBuiltinPrefix, NoThrow, Const,
-                    CustomTypeChecking];
+                    CustomTypeChecking, Constexpr];
   let Prototype = "int(...)";
 }
 
 def IsUnordered : Builtin {
   let Spellings = ["__builtin_isunordered"];
   let Attributes = [FunctionWithBuiltinPrefix, NoThrow, Const,
-                    CustomTypeChecking];
+                    CustomTypeChecking, Constexpr];
   let Prototype = "int(...)";
 }
 
@@ -646,19 +646,21 @@ def IsFPClass : Builtin {
 def Signbit : Builtin {
   let Spellings = ["__builtin_signbit"];
   let Attributes = [FunctionWithBuiltinPrefix, NoThrow, Const,
-                    CustomTypeChecking];
+                    CustomTypeChecking, Constexpr];
   let Prototype = "int(...)";
 }
 
 def SignbitF : Builtin {
   let Spellings = ["__builtin_signbitf"];
-  let Attributes = [FunctionWithBuiltinPrefix, NoThrow, Const];
+  let Attributes = [FunctionWithBuiltinPrefix, NoThrow, Const,
+                    Constexpr];
   let Prototype = "int(float)";
 }
 
 def SignbitL : Builtin {
   let Spellings = ["__builtin_signbitl"];
-  let Attributes = [FunctionWithBuiltinPrefix, NoThrow, Const];
+  let Attributes = [FunctionWithBuiltinPrefix, NoThrow, Const,
+                    Constexpr];
   let Prototype = "int(long double)";
 }
 
diff --git a/clang/lib/AST/ByteCode/InterpBuiltin.cpp b/clang/lib/AST/ByteCode/InterpBuiltin.cpp
index c3370e2e5286e07..26abf5820510673 100644
--- a/clang/lib/AST/ByteCode/InterpBuiltin.cpp
+++ b/clang/lib/AST/ByteCode/InterpBuiltin.cpp
@@ -433,6 +433,51 @@ static bool interp__builtin_iszero(InterpState &S, CodePtr OpPC,
   return true;
 }
 
+static bool interp__builtin_signbit(InterpState &S, CodePtr OpPC,
+                                    const InterpFrame *Frame, const Function *F,
+                                    const CallExpr *Call) {
+  const Floating &Arg = S.Stk.peek<Floating>();
+
+  pushInteger(S, Arg.isNegative(), Call->getType());
+  return true;
+}
+
+static bool interp_floating_comparison(InterpState &S, CodePtr OpPC,
+                                       const InterpFrame *Frame,
+                                       const Function *F,
+                                       const CallExpr *Call) {
+  const Floating &RHS = S.Stk.peek<Floating>();
+  const Floating &LHS = S.Stk.peek<Floating>(align(2u * primSize(PT_Float)));
+  unsigned ID = F->getBuiltinID();
+
+  pushInteger(
+      S,
+      [&] {
+        switch (ID) {
+        case Builtin::BI__builtin_isgreater:
+          return LHS > RHS;
+        case Builtin::BI__builtin_isgreaterequal:
+          return LHS >= RHS;
+        case Builtin::BI__builtin_isless:
+          return LHS < RHS;
+        case Builtin::BI__builtin_islessequal:
+          return LHS <= RHS;
+        case Builtin::BI__builtin_islessgreater: {
+          ComparisonCategoryResult cmp = LHS.compare(RHS);
+          return cmp == ComparisonCategoryResult::Less ||
+                 cmp == ComparisonCategoryResult::Greater;
+        }
+        case Builtin::BI__builtin_isunordered:
+          return LHS.compare(RHS) == ComparisonCategoryResult::Unordered;
+        default:
+          llvm_unreachable("Unexpected builtin ID: Should be a floating point "
+                           "comparison function");
+        }
+      }(),
+      Call->getType());
+  return true;
+}
+
 /// First parameter to __builtin_isfpclass is the floating value, the
 /// second one is an integral value.
 static bool interp__builtin_isfpclass(InterpState &S, CodePtr OpPC,
@@ -1313,6 +1358,21 @@ bool InterpretBuiltin(InterpState &S, CodePtr OpPC, const Function *F,
     if (!interp__builtin_iszero(S, OpPC, Frame, F, Call))
       return false;
     break;
+  case Builtin::BI__builtin_signbit:
+  case Builtin::BI__builtin_signbitf:
+  case Builtin::BI__builtin_signbitl:
+    if (!interp__builtin_signbit(S, OpPC, Frame, F, Call))
+      return false;
+    break;
+  case Builtin::BI__builtin_isgreater:
+  case Builtin::BI__builtin_isgreaterequal:
+  case Builtin::BI__builtin_isless:
+  case Builtin::BI__builtin_islessequal:
+  case Builtin::BI__builtin_islessgreater:
+  case Builtin::BI__builtin_isunordered:
+    if (!interp_floating_comparison(S, OpPC, Frame, F, Call))
+      return false;
+    break;
   case Builtin::BI__builtin_isfpclass:
     if (!interp__builtin_isfpclass(S, OpPC, Frame, F, Call))
       return false;
diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp
index 7bfc63ffd81e288..31b65ff4bcc135c 100644
--- a/clang/lib/AST/ExprConstant.cpp
+++ b/clang/lib/AST/ExprConstant.cpp
@@ -12833,6 +12833,54 @@ bool IntExprEvaluator::VisitBuiltinCallExpr(const CallExpr *E,
            Success(Val.isZero() ? 1 : 0, E);
   }
 
+  case Builtin::BI__builtin_signbit:
+  case Builtin::BI__builtin_signbitf:
+  case Builtin::BI__builtin_signbitl: {
+    APFloat Val(0.0);
+    return EvaluateFloat(E->getArg(0), Val, Info) &&
+           Success(Val.isNegative() ? 1 : 0, E);
+  }
+
+  case Builtin::BI__builtin_isgreater:
+  case Builtin::BI__builtin_isgreaterequal:
+  case Builtin::BI__builtin_isless:
+  case Builtin::BI__builtin_islessequal:
+  case Builtin::BI__builtin_islessgreater:
+  case Builtin::BI__builtin_isunordered: {
+    APFloat LHS(0.0);
+    APFloat RHS(0.0);
+    if (!EvaluateFloat(E->getArg(0), LHS, Info) ||
+        !EvaluateFloat(E->getArg(1), RHS, Info))
+      return false;
+
+    return Success(
+        [&] {
+          switch (BuiltinOp) {
+          case Builtin::BI__builtin_isgreater:
+            return LHS > RHS;
+          case Builtin::BI__builtin_isgreaterequal:
+            return LHS >= RHS;
+          case Builtin::BI__builtin_isless:
+            return LHS < RHS;
+          case Builtin::BI__builtin_islessequal:
+            return LHS <= RHS;
+          case Builtin::BI__builtin_islessgreater: {
+            APFloat::cmpResult cmp = LHS.compare(RHS);
+            return cmp == APFloat::cmpResult::cmpLessThan ||
+                   cmp == APFloat::cmpResult::cmpGreaterThan;
+          }
+          case Builtin::BI__builtin_isunordered:
+            return LHS.compare(RHS) == APFloat::cmpResult::cmpUnordered;
+          default:
+            llvm_unreachable("Unexpected builtin ID: Should be a floating "
+                             "point comparison function");
+          }
+        }()
+            ? 1
+            : 0,
+        E);
+  }
+
   case Builtin::BI__builtin_issignaling: {
     APFloat Val(0.0);
     return EvaluateFloat(E->getArg(0), Val, Info) &&
diff --git a/clang/test/Analysis/builtin_signbit.cpp b/clang/test/Analysis/builtin_signbit.cpp
index be10f0950f69b7c..ad185d855cfff30 100644
--- a/clang/test/Analysis/builtin_signbit.cpp
+++ b/clang/test/Analysis/builtin_signbit.cpp
@@ -12,103 +12,83 @@ long double ld = -1.0L;
 // CHECK-BE32-LABEL: define dso_local void @_Z12test_signbitv(
 // CHECK-BE32-SAME: ) #[[ATTR0:[0-9]+]] {
 // CHECK-BE32-NEXT:  entry:
-// CHECK-BE32-NEXT:    [[TMP0:%.*]] = lshr i128 bitcast (ppc_fp128 0xM3FF00000000000000000000000000000 to i128), 64
-// CHECK-BE32-NEXT:    [[TMP1:%.*]] = trunc i128 [[TMP0]] to i64
-// CHECK-BE32-NEXT:    [[TMP2:%.*]] = icmp slt i64 [[TMP1]], 0
-// CHECK-BE32-NEXT:    [[FROMBOOL:%.*]] = zext i1 [[TMP2]] to i8
+// CHECK-BE32-NEXT:    store i8 0, ptr @b, align 1
+// CHECK-BE32-NEXT:    [[TMP0:%.*]] = load ppc_fp128, ptr @ld, align 16
+// CHECK-BE32-NEXT:    [[TMP1:%.*]] = bitcast ppc_fp128 [[TMP0]] to i128
+// CHECK-BE32-NEXT:    [[TMP2:%.*]] = lshr i128 [[TMP1]], 64
+// CHECK-BE32-NEXT:    [[TMP3:%.*]] = trunc i128 [[TMP2]] to i64
+// CHECK-BE32-NEXT:    [[TMP4:%.*]] = icmp slt i64 [[TMP3]], 0
+// CHECK-BE32-NEXT:    [[FROMBOOL:%.*]] = zext i1 [[TMP4]] to i8
 // CHECK-BE32-NEXT:    store i8 [[FROMBOOL]], ptr @b, align 1
-// CHECK-BE32-NEXT:    [[TMP3:%.*]] = load ppc_fp128, ptr @ld, align 16
-// CHECK-BE32-NEXT:    [[TMP4:%.*]] = bitcast ppc_fp128 [[TMP3]] to i128
-// CHECK-BE32-NEXT:    [[TMP5:%.*]] = lshr i128 [[TMP4]], 64
-// CHECK-BE32-NEXT:    [[TMP6:%.*]] = trunc i128 [[TMP5]] to i64
-// CHECK-BE32-NEXT:    [[TMP7:%.*]] = icmp slt i64 [[TMP6]], 0
+// CHECK-BE32-NEXT:    store i8 0, ptr @b, align 1
+// CHECK-BE32-NEXT:    [[TMP5:%.*]] = load double, ptr @d, align 8
+// CHECK-BE32-NEXT:    [[CONV:%.*]] = fptrunc double [[TMP5]] to float
+// CHECK-BE32-NEXT:    [[TMP6:%.*]] = bitcast float [[CONV]] to i32
+// CHECK-BE32-NEXT:    [[TMP7:%.*]] = icmp slt i32 [[TMP6]], 0
 // CHECK-BE32-NEXT:    [[FROMBOOL1:%.*]] = zext i1 [[TMP7]] to i8
 // CHECK-BE32-NEXT:    store i8 [[FROMBOOL1]], ptr @b, align 1
 // CHECK-BE32-NEXT:    store i8 0, ptr @b, align 1
-// CHECK-BE32-NEXT:    [[TMP8:%.*]] = load double, ptr @d, align 8
-// CHECK-BE32-NEXT:    [[CONV:%.*]] = fptrunc double [[TMP8]] to float
-// CHECK-BE32-NEXT:    [[TMP9:%.*]] = bitcast float [[CONV]] to i32
-// CHECK-BE32-NEXT:    [[TMP10:%.*]] = icmp slt i32 [[TMP9]], 0
-// CHECK-BE32-NEXT:    [[FROMBOOL2:%.*]] = zext i1 [[TMP10]] to i8
+// CHECK-BE32-NEXT:    [[TMP8:%.*]] = load ppc_fp128, ptr @ld, align 16
+// CHECK-BE32-NEXT:    [[TMP9:%.*]] = bitcast ppc_fp128 [[TMP8]] to i128
+// CHECK-BE32-NEXT:    [[TMP10:%.*]] = lshr i128 [[TMP9]], 64
+// CHECK-BE32-NEXT:    [[TMP11:%.*]] = trunc i128 [[TMP10]] to i64
+// CHECK-BE32-NEXT:    [[TMP12:%.*]] = icmp slt i64 [[TMP11]], 0
+// CHECK-BE32-NEXT:    [[FROMBOOL2:%.*]] = zext i1 [[TMP12]] to i8
 // CHECK-BE32-NEXT:    store i8 [[FROMBOOL2]], ptr @b, align 1
-// CHECK-BE32-NEXT:    [[TMP11:%.*]] = lshr i128 bitcast (ppc_fp128 0xM3FF00000000000000000000000000000 to i128), 64
-// CHECK-BE32-NEXT:    [[TMP12:%.*]] = trunc i128 [[TMP11]] to i64
-// CHECK-BE32-NEXT:    [[TMP13:%.*]] = icmp slt i64 [[TMP12]], 0
-// CHECK-BE32-NEXT:    [[FROMBOOL3:%.*]] = zext i1 [[TMP13]] to i8
-// CHECK-BE32-NEXT:    store i8 [[FROMBOOL3]], ptr @b, align 1
-// CHECK-BE32-NEXT:    [[TMP14:%.*]] = load ppc_fp128, ptr @ld, align 16
-// CHECK-BE32-NEXT:    [[TMP15:%.*]] = bitcast ppc_fp128 [[TMP14]] to i128
-// CHECK-BE32-NEXT:    [[TMP16:%.*]] = lshr i128 [[TMP15]], 64
-// CHECK-BE32-NEXT:    [[TMP17:%.*]] = trunc i128 [[TMP16]] to i64
-// CHECK-BE32-NEXT:    [[TMP18:%.*]] = icmp slt i64 [[TMP17]], 0
-// CHECK-BE32-NEXT:    [[FROMBOOL4:%.*]] = zext i1 [[TMP18]] to i8
-// CHECK-BE32-NEXT:    store i8 [[FROMBOOL4]], ptr @b, align 1
 // CHECK-BE32-NEXT:    ret void
 //
 // CHECK-BE64-LABEL: define dso_local void @_Z12test_signbitv(
 // CHECK-BE64-SAME: ) #[[ATTR0:[0-9]+]] {
 // CHECK-BE64-NEXT:  entry:
-// CHECK-BE64-NEXT:    [[TMP0:%.*]] = lshr i128 bitcast (ppc_fp128 0xM3FF00000000000000000000000000000 to i128), 64
-// CHECK-BE64-NEXT:    [[TMP1:%.*]] = trunc i128 [[TMP0]] to i64
-// CHECK-BE64-NEXT:    [[TMP2:%.*]] = icmp slt i64 [[TMP1]], 0
-// CHECK-BE64-NEXT:    [[FROMBOOL:%.*]] = zext i1 [[TMP2]] to i8
+// CHECK-BE64-NEXT:    store i8 0, ptr @b, align 1
+// CHECK-BE64-NEXT:    [[TMP0:%.*]] = load ppc_fp128, ptr @ld, align 16
+// CHECK-BE64-NEXT:    [[TMP1:%.*]] = bitcast ppc_fp128 [[TMP0]] to i128
+// CHECK-BE64-NEXT:    [[TMP2:%.*]] = lshr i128 [[TMP1]], 64
+// CHECK-BE64-NEXT:    [[TMP3:%.*]] = trunc i128 [[TMP2]] to i64
+// CHECK-BE64-NEXT:    [[TMP4:%.*]] = icmp slt i64 [[TMP3]], 0
+// CHECK-BE64-NEXT:    [[FROMBOOL:%.*]] = zext i1 [[TMP4]] to i8
 // CHECK-BE64-NEXT:    store i8 [[FROMBOOL]], ptr @b, align 1
-// CHECK-BE64-NEXT:    [[TMP3:%.*]] = load ppc_fp128, ptr @ld, align 16
-// CHECK-BE64-NEXT:    [[TMP4:%.*]] = bitcast ppc_fp128 [[TMP3]] to i128
-// CHECK-BE64-NEXT:    [[TMP5:%.*]] = lshr i128 [[TMP4]], 64
-// CHECK-BE64-NEXT:    [[TMP6:%.*]] = trunc i128 [[TMP5]] to i64
-// CHECK-BE64-NEXT:    [[TMP7:%.*]] = icmp slt i64 [[TMP6]], 0
+// CHECK-BE64-NEXT:    store i8 0, ptr @b, align 1
+// CHECK-BE64-NEXT:    [[TMP5:%.*]] = load double, ptr @d, align 8
+// CHECK-BE64-NEXT:    [[CONV:%.*]] = fptrunc double [[TMP5]] to float
+// CHECK-BE64-NEXT:    [[TMP6:%.*]] = bitcast float [[CONV]] to i32
+// CHECK-BE64-NEXT:    [[TMP7:%.*]] = icmp slt i32 [[TMP6]], 0
 // CHECK-BE64-NEXT:    [[FROMBOOL1:%.*]] = zext i1 [[TMP7]] to i8
 // CHECK-BE64-NEXT:    store i8 [[FROMBOOL1]], ptr @b, align 1
 // CHECK-BE64-NEXT:    store i8 0, ptr @b, align 1
-// CHECK-BE64-NEXT:    [[TMP8:%.*]] = load double, ptr @d, align 8
-// CHECK-BE64-NEXT:    [[CONV:%.*]] = fptrunc double [[TMP8]] to float
-// CHECK-BE64-NEXT:    [[TMP9:%.*]] = bitcast float [[CONV]] to i32
-// CHECK-BE64-NEXT:    [[TMP10:%.*]] = icmp slt i32 [[TMP9]], 0
-// CHECK-BE64-NEXT:    [[FROMBOOL2:%.*]] = zext i1 [[TMP10]] to i8
+// CHECK-BE64-NEXT:    [[TMP8:%.*]] = load ppc_fp128, ptr @ld, align 16
+// CHECK-BE64-NEXT:    [[TMP9:%.*]] = bitcast ppc_fp128 [[TMP8]] to i128
+// CHECK-BE64-NEXT:    [[TMP10:%.*]] = lshr i128 [[TMP9]], 64
+// CHECK-BE64-NEXT:    [[TMP11:%.*]] = trunc i128 [[TMP10]] to i64
+// CHECK-BE64-NEXT:    [[TMP12:%.*]] = icmp slt i64 [[TMP11]], 0
+// CHECK-BE64-NEXT:    [[FROMBOOL2:%.*]] = zext i1 [[TMP12]] to i8
 // CHECK-BE64-NEXT:    store i8 [[FROMBOOL2]], ptr @b, align 1
-// CHECK-BE64-NEXT:    [[TMP11:%.*]] = lshr i128 bitcast (ppc_fp128 0xM3FF00000000000000000000000000000 to i128), 64
-// CHECK-BE64-NEXT:    [[TMP12:%.*]] = trunc i128 [[TMP11]] to i64
-// CHECK-BE64-NEXT:    [[TMP13:%.*]] = icmp slt i64 [[TMP12]], 0
-// CHECK-BE64-NEXT:    [[FROMBOOL3:%.*]] = zext i1 [[TMP13]] to i8
-// CHECK-BE64-NEXT:    store i8 [[FROMBOOL3]], ptr @b, align 1
-// CHECK-BE64-NEXT:    [[TMP14:%.*]] = load ppc_fp128, ptr @ld, align 16
-// CHECK-BE64-NEXT:    [[TMP15:%.*]] = bitcast ppc_fp128 [[TMP14]] to i128
-// CHECK-BE64-NEXT:    [[TMP16:%.*]] = lshr i128 [[TMP15]], 64
-// CHECK-BE64-NEXT:    [[TMP17:%.*]] = trunc i128 [[TMP16]] to i64
-// CHECK-BE64-NEXT:    [[TMP18:%.*]] = icmp slt i64 [[TMP17]], 0
-// CHECK-BE64-NEXT:    [[FROMBOOL4:%.*]] = zext i1 [[TMP18]] to i8
-// CHECK-BE64-NEXT:    store i8 [[FROMBOOL4]], ptr @b, align 1
 // CHECK-BE64-NEXT:    ret void
 //
 // CHECK-LE-LABEL: define dso_local void @_Z12test_signbitv(
 // CHECK-LE-SAME: ) #[[ATTR0:[0-9]+]] {
 // CHECK-LE-NEXT:  entry:
-// CHECK-LE-NEXT:    [[TMP0:%.*]] = icmp slt i64 trunc (i128 bitcast (ppc_fp128 0xM3FF00000000000000000000000000000 to i128) to i64), 0
-// CHECK-LE-NEXT:    [[FROMBOOL:%.*]] = zext i1 [[TMP0]] to i8
+// CHECK-LE-NEXT:    store i8 0, ptr @b, align 1
+// CHECK-LE-NEXT:    [[TMP0:%.*]] = load ppc_fp128, ptr @ld, align 16
+// CHECK-LE-NEXT:    [[TMP1:%.*]] = bitcast ppc_fp128 [[TMP0]] to i128
+// CHECK-LE-NEXT:    [[TMP2:%.*]] = trunc i128 [[TMP1]] to i64
+// CHECK-LE-NEXT:    [[TMP3:%.*]] = icmp slt i64 [[TMP2]], 0
+// CHECK-LE-NEXT:    [[FROMBOOL:%.*]] = zext i1 [[TMP3]] to i8
 // CHECK-LE-NEXT:    store i8 [[FROMBOOL]], ptr @b, align 1
-// CHECK-LE-NEXT:    [[TMP1:%.*]] = load ppc_fp128, ptr @ld, align 16
-// CHECK-LE-NEXT:    [[TMP2:%.*]] = bitcast ppc_fp128 [[TMP1]] to i128
-// CHECK-LE-NEXT:    [[TMP3:%.*]] = trunc i128 [[TMP2]] to i64
-// CHECK-LE-NEXT:    [[TMP4:%.*]] = icmp slt i64 [[TMP3]], 0
-// CHECK-LE-NEXT:    [[FROMBOOL1:%.*]] = zext i1 [[TMP4]] to i8
+// CHECK-LE-NEXT:    store i8 0, ptr @b, align 1
+// CHECK-LE-NEXT:    [[TMP4:%.*]] = load double, ptr @d, align 8
+// CHECK-LE-NEXT:    [[CONV:%.*]] = fptrunc double [[TMP4]] to float
+// CHECK-LE-NEXT:    [[TMP5:%.*]] = bitcast float [[CONV]] to i32
+// CHECK-LE-NEXT:    [[TMP6:%.*]] = icmp slt i32 [[TMP5]], 0
+// CHECK-LE-NEXT:    [[FROMBOOL1:%.*]] = zext i1 [[TMP6]] to i8
 // CHECK-LE-NEXT:    store i8 [[FROMBOOL1]], ptr @b, align 1
 // CHECK-LE-NEXT:    store i8 0, ptr @b, align 1
-// CHECK-LE-NEXT:    [[TMP5:%.*]] = load double, ptr @d, align 8
-// CHECK-LE-NEXT:    [[CONV:%.*]] = fptrunc double [[TMP5]] to float
-// CHECK-LE-NEXT:    [[TMP6:%.*]] = bitcast float [[CONV]] to i32
-// CHECK-LE-NEXT:    [[TMP7:%.*]] = icmp slt i32 [[TMP6]], 0
-// CHECK-LE-NEXT:    [[FROMBOOL2:%.*]] = zext i1 [[TMP7]] to i8
+// CHECK-LE-NEXT:    [[TMP7:%.*]] = load ppc_fp128, ptr @ld, align 16
+// CHECK-LE-NEXT:    [[TMP8:%.*]] = bitcast ppc_fp128 [[TMP7]] to i128
+// CHECK-LE-NEXT:    [[TMP9:%.*]] = trunc i128 [[TMP8]] to i64
+// CHECK-LE-NEXT:    [[TMP10:%.*]] = icmp slt i64 [[TMP9]], 0
+// CHECK-LE-NEXT:    [[FROMBOOL2:%.*]] = zext i1 [[TMP10]] to i8
 // CHECK-LE-NEXT:    store i8 [[FROMBOOL2]], ptr @b, align 1
-// CHECK-LE-NEXT:    [[TMP8:%.*]] = icmp slt i64 trunc (i128 bitcast (ppc_fp128 0xM3FF00000000000000000000000000000 to i128) to i64), 0
-// CHECK-LE-NEXT:    [[FROMBOOL3:%.*]] = zext i1 [[TMP8]] to i8
-// CHECK-LE-NEXT:    store i8 [[FROMBOOL3]], ptr @b, align 1
-// CHECK-LE-NEXT:    [[TMP9:%.*]] = load ppc_fp128, ptr @ld, align 16
-// CHECK-LE-NEXT:    [[TMP10:%.*]] = bitcast ppc_fp128 [[TMP9]] to i128
-// CHECK-LE-NEXT:    [[TMP11:%.*]] = trunc i128 [[TMP10]] to i64
-// CHECK-LE-NEXT:    [[TMP12:%.*]] = icmp slt i64 [[TMP11]], 0
-// CHECK-LE-NEXT:    [[FROMBOOL4:%.*]] = zext i1 [[TMP12]] to i8
-// CHECK-LE-NEXT:    store i8 [[FROMBOOL4]], ptr @b, align 1
 // CHECK-LE-NEXT:    ret void
 //
 void test_signbit()
diff --git a/clang/test/Sema/constant-builtins-2.c b/clang/test/Sema/constant-builtins-2.c
index 37b63cf4f6b3285..c359eba4e3d16ca 100644
--- a/clang/test/Sema/constant-builtins-2.c
+++ b/clang/test/Sema/constant-builtins-2.c
@@ -1,4 +1,5 @@
 // RUN: %clang_cc1 -fsyntax-only -verify %s
+// RUN: %clang_cc1 -fsyntax-only -fexperimental-new-constant-interpreter -verify %s
 
 // Math stuff
 
@@ -204,6 +205,53 @@ char isfpclass_snan_1   [!__builtin_isfpclass(__builtin_nans(""), 0x0002) ? 1 :
 char isfpclass_snan_2   [__builtin_isfpclass(__builtin_nansl(""), 0x0207) ? 1 : -1]; // ~fcFinite
 char isfpclass_snan_3   [!__builtin_isfpclass(__builtin_nans(""), 0x01F8) ? 1 : -1]; // fcFinite
 
+__extension__ _Static_assert(
+  !__builtin_signbit(1.0) && __builtin_signbit(-1.0) && !__builtin_signbit(0.0) && __builtin_signbit(-0.0) &&
+  !__builtin_signbitf(1.0f) && __builtin_signbitf(-1.0f) && !__builtin_signbitf(0.0f) && __builtin_signbitf(-0.0f) &&
+  !__builtin_signbitl(1.0L) && __builtin_signbitf(-1.0L) && !__builtin_signbitf(0.0L) && __builtin_signbitf(-0.0L) &&
+  !__builtin_signbit(1.0f) && __builtin_signbit(-1.0f) && !__builtin_signbit(0.0f) && __builtin_signbit(-0.0f) &&
+  !__builtin_signbit(1.0L) && __builtin_signbit(-1.0L) && !__builtin_signbit(0.0L) && __builtin_signbit(-0.0L) &&
+#if defined(__FLOAT128__) || defined(__SIZEOF_FLOAT128__)
+  !__builtin_signbit(1.0q) && __builtin_signbit(-1.0q) && !__builtin_signbit(0.0q) && __builtin_signbit(-0.0q) &&
+#endif
+  1, ""
+);
+
+#define LESS(X, Y) \
+  !__builtin_isgreater(X, Y) && __builtin_isgreater(Y, X) &&             \
+  !__builtin_isgreaterequal(X, Y) && __builtin_isgreaterequal(Y, X) &&   \
+  __builtin_isless(X, Y) && !__builtin_isless(Y, X) &&                   \
+  __builtin_islessequal(X, Y) && !__builtin_islessequal(Y, X) &&         \
+  __builtin_islessgreater(X, Y) && __builtin_islessgreater(Y, X) &&      \
+  !__builtin_isunordered(X, Y) && !__builtin_isunordered(Y, X)
+#define EQUAL(X, Y) \
+  !__builtin_isgreater(X, Y) && !__builtin_isgreater(Y, X) &&            \
+  __builtin_isgreaterequal(X, Y) && __builtin_isgreaterequal(Y, X) &&    \
+  !__builtin_isless(X, Y) && !__builtin_isless(Y, X) &&                  \
+  __builtin_islessequal(X, Y) && __builtin_islessequal(Y, X) &&          \
+  !__builtin_islessgreater(X, Y) && !__builtin_islessgreater(Y, X) &&    \
+  !__builtin_isunordered(X, Y) && !__builtin_isunordered(Y, X)
+#define UNORDERED(X, Y) \
+  !__builtin_isgreater(X, Y) && !__builtin_isgreater(Y, X) &&            \
+  !__builtin_isgreaterequal(X, Y) && !__builtin_isgreaterequal(Y, X) &&  \
+  !__builtin_isless(X, Y) && !__builtin_isless(Y, X) &&                  \
+  !__builtin_islessequal(X, Y) && !__builtin_islessequal(Y, X) &&        \
+  !__builtin_islessgreater(X, Y) && !__builtin_islessgreater(Y, X) &&    \
+  __builtin_isunordered(X, Y) && __builtin_isunordered(Y, X)
+
+__extension__ _Static_assert(
+  LESS(0.0, 1.0) && EQUAL(1.0, 1.0) && EQUAL(0.0, -0.0) &&
+  UNORDERED(__builtin_nan(""), 1.0) && UNORDERED(__builtin_nan(""), __builtin_inf()) && LESS(0.0, __builtin_inf()) &&
+  LESS(0.0f, 1.0f) && EQUAL(1.0f, 1.0f) && EQUAL(0.0f, -0.0f) &&
+  UNORDERED(__builtin_nanf(""), 1.0f) && UNORDERED(__builtin_nanf(""), __builtin_inff()) && LESS(0.0f, __builtin_inff()) &&
+  LESS(0.0L, 1.0L) && EQUAL(1.0L, 1.0L) && EQUAL(0.0L, -0.0L) &&
+  UNORDERED(__builtin_nanl(""), 1.0L) && UNORDERED(__builtin_nanl(""), __builtin_infl()) && LESS(0.0L, __builtin_infl()) &&
+#if defined(__FLOAT128__) || defined(__SIZEOF_FLOAT128__)
+  LESS(0.0q, 1.0q) && EQUAL(1.0q, 1.0q) && EQUAL(0.0q, -0.0q) &&
+#endif
+  1, ""
+);
+
 //double       g19 = __builtin_powi(2.0, 4);
 //float        g20 = __builtin_powif(2.0f, 4);
 //long double  g21 = __builtin_powil(2.0L, 4);

From 10fe531d6c9d46fb9584d33782f26d27887c72fd Mon Sep 17 00:00:00 2001
From: David Green <david.green@arm.com>
Date: Sun, 18 Aug 2024 11:08:49 +0100
Subject: [PATCH 291/441] [GlobalISel] Add and use an Opcode variable and
 update match-table-cxx.td checks. NFC

---
 .../CodeGen/GlobalISel/LegalizerHelper.cpp    | 19 ++++++------
 .../match-table-cxx.td                        | 30 +++++++++----------
 2 files changed, 25 insertions(+), 24 deletions(-)

diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index 225ec19246231da..bdbef20e20960d3 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -2523,7 +2523,8 @@ LegalizerHelper::widenScalarMulo(MachineInstr &MI, unsigned TypeIdx,
 
 LegalizerHelper::LegalizeResult
 LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
-  switch (MI.getOpcode()) {
+  unsigned Opcode = MI.getOpcode();
+  switch (Opcode) {
   default:
     return UnableToLegalize;
   case TargetOpcode::G_ATOMICRMW_XCHG:
@@ -2607,13 +2608,13 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
     Register SrcReg = MI.getOperand(1).getReg();
 
     // First extend the input.
-    unsigned ExtOpc = MI.getOpcode() == TargetOpcode::G_CTTZ ||
-                              MI.getOpcode() == TargetOpcode::G_CTTZ_ZERO_UNDEF
+    unsigned ExtOpc = Opcode == TargetOpcode::G_CTTZ ||
+                              Opcode == TargetOpcode::G_CTTZ_ZERO_UNDEF
                           ? TargetOpcode::G_ANYEXT
                           : TargetOpcode::G_ZEXT;
     auto MIBSrc = MIRBuilder.buildInstr(ExtOpc, {WideTy}, {SrcReg});
     LLT CurTy = MRI.getType(SrcReg);
-    unsigned NewOpc = MI.getOpcode();
+    unsigned NewOpc = Opcode;
     if (NewOpc == TargetOpcode::G_CTTZ) {
       // The count is the same in the larger type except if the original
       // value was zero.  This can be handled by setting the bit just off
@@ -2628,7 +2629,7 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
 
     unsigned SizeDiff = WideTy.getSizeInBits() - CurTy.getSizeInBits();
 
-    if (MI.getOpcode() == TargetOpcode::G_CTLZ_ZERO_UNDEF) {
+    if (Opcode == TargetOpcode::G_CTLZ_ZERO_UNDEF) {
       // An optimization where the result is the CTLZ after the left shift by
       // (Difference in widety and current ty), that is,
       // MIBSrc = MIBSrc << (sizeinbits(WideTy) - sizeinbits(CurTy))
@@ -2640,7 +2641,7 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
     // Perform the operation at the larger size.
     auto MIBNewOp = MIRBuilder.buildInstr(NewOpc, {WideTy}, {MIBSrc});
     // This is already the correct result for CTPOP and CTTZs
-    if (MI.getOpcode() == TargetOpcode::G_CTLZ) {
+    if (Opcode == TargetOpcode::G_CTLZ) {
       // The correct result is NewOp - (Difference in widety and current ty).
       MIBNewOp = MIRBuilder.buildSub(
           WideTy, MIBNewOp, MIRBuilder.buildConstant(WideTy, SizeDiff));
@@ -2788,8 +2789,8 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
     Observer.changingInstr(MI);
 
     if (TypeIdx == 0) {
-      unsigned CvtOp = MI.getOpcode() == TargetOpcode::G_ASHR ?
-        TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT;
+      unsigned CvtOp = Opcode == TargetOpcode::G_ASHR ? TargetOpcode::G_SEXT
+                                                      : TargetOpcode::G_ZEXT;
 
       widenScalarSrc(MI, WideTy, 1, CvtOp);
       widenScalarDst(MI, WideTy);
@@ -3101,7 +3102,7 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
   case TargetOpcode::G_FLDEXP:
   case TargetOpcode::G_STRICT_FLDEXP: {
     if (TypeIdx == 0) {
-      if (MI.getOpcode() == TargetOpcode::G_STRICT_FLDEXP)
+      if (Opcode == TargetOpcode::G_STRICT_FLDEXP)
         return UnableToLegalize;
 
       Observer.changingInstr(MI);
diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-cxx.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-cxx.td
index 0dcd0c2dd50a8e5..9a1a15c8d30df3e 100644
--- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-cxx.td
+++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-cxx.td
@@ -86,12 +86,12 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK:      const uint8_t *GenMyCombiner::getMatchTable() const {
 // CHECK-NEXT:   constexpr static uint8_t MatchTable0[] = {
 // CHECK-NEXT:     GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2([[#LOWER:]]), GIMT_Encode2([[#UPPER:]]), /*)*//*default:*//*Label 4*/ GIMT_Encode4([[#DEFAULT:]]),
-// CHECK-NEXT:     /*TargetOpcode::G_STORE*//*Label 0*/ GIMT_Encode4(418), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
-// CHECK-NEXT:     /*TargetOpcode::G_SEXT*//*Label 1*/ GIMT_Encode4(436), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
-// CHECK-NEXT:     /*TargetOpcode::G_FNEG*//*Label 2*/ GIMT_Encode4(448), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
-// CHECK-NEXT:     /*TargetOpcode::G_FABS*//*Label 3*/ GIMT_Encode4(460),
+// CHECK-NEXT:     /*TargetOpcode::G_STORE*//*Label 0*/ GIMT_Encode4([[L418:[0-9]+]]), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
+// CHECK-NEXT:     /*TargetOpcode::G_SEXT*//*Label 1*/ GIMT_Encode4([[L436:[0-9]+]]), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
+// CHECK-NEXT:     /*TargetOpcode::G_FNEG*//*Label 2*/ GIMT_Encode4([[L448:[0-9]+]]), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
+// CHECK-NEXT:     /*TargetOpcode::G_FABS*//*Label 3*/ GIMT_Encode4([[L460:[0-9]+]]),
 // CHECK-NEXT:     // Label 0: @[[#%u, mul(UPPER-LOWER, 4) + 10]]
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 5*/ GIMT_Encode4(435), // Rule ID 2 //
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 5*/ GIMT_Encode4([[L435:[0-9]+]]), // Rule ID 2 //
 // CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule2Enabled),
 // CHECK-NEXT:       // MIs[0] x
 // CHECK-NEXT:       // No operand predicates
@@ -101,10 +101,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:       GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_GICombiner1),
 // CHECK-NEXT:       // Combiner Rule #2: TwoMatchNoApply
 // CHECK-NEXT:       GIR_EraseRootFromParent_Done,
-// CHECK-NEXT:     // Label 5: @435
+// CHECK-NEXT:     // Label 5: @[[L435]]
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     // Label 1: @436
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 6*/ GIMT_Encode4(447), // Rule ID 3 //
+// CHECK-NEXT:     // Label 1: @[[L436]]
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 6*/ GIMT_Encode4([[L447:[0-9]+]]), // Rule ID 3 //
 // CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule3Enabled),
 // CHECK-NEXT:       // MIs[0] a
 // CHECK-NEXT:       // No operand predicates
@@ -112,10 +112,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:       // No operand predicates
 // CHECK-NEXT:       // Combiner Rule #3: NoMatchTwoApply
 // CHECK-NEXT:       GIR_DoneWithCustomAction, /*Fn*/GIMT_Encode2(GICXXCustomAction_GICombiner2),
-// CHECK-NEXT:     // Label 6: @447
+// CHECK-NEXT:     // Label 6: @[[L447]]
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     // Label 2: @448
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 7*/ GIMT_Encode4(459), // Rule ID 1 //
+// CHECK-NEXT:     // Label 2: @[[L448]]
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 7*/ GIMT_Encode4([[L459:[0-9]+]]), // Rule ID 1 //
 // CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule1Enabled),
 // CHECK-NEXT:       // MIs[0] a
 // CHECK-NEXT:       // No operand predicates
@@ -123,10 +123,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:       // No operand predicates
 // CHECK-NEXT:       // Combiner Rule #1: TwoMatchTwoApply
 // CHECK-NEXT:       GIR_DoneWithCustomAction, /*Fn*/GIMT_Encode2(GICXXCustomAction_GICombiner1),
-// CHECK-NEXT:     // Label 7: @459
+// CHECK-NEXT:     // Label 7: @[[L459]]
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     // Label 3: @460
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 8*/ GIMT_Encode4(471), // Rule ID 0 //
+// CHECK-NEXT:     // Label 3: @[[L460]]
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 8*/ GIMT_Encode4([[L471:[0-9]+]]), // Rule ID 0 //
 // CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled),
 // CHECK-NEXT:       // MIs[0] a
 // CHECK-NEXT:       // No operand predicates
@@ -134,7 +134,7 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:       // No operand predicates
 // CHECK-NEXT:       // Combiner Rule #0: OneMatchOneApply
 // CHECK-NEXT:       GIR_DoneWithCustomAction, /*Fn*/GIMT_Encode2(GICXXCustomAction_GICombiner0),
-// CHECK-NEXT:     // Label 8: @471
+// CHECK-NEXT:     // Label 8: @[[L471]]
 // CHECK-NEXT:     GIM_Reject,
 // CHECK-NEXT:     // Label 4: @[[#%u, DEFAULT]]
 // CHECK-NEXT:     GIM_Reject,

From c4092d326ae4989f54c5f01d3a077841fd76bc2f Mon Sep 17 00:00:00 2001
From: "Oleksandr T." <oleksandr.tarasiuk@outlook.com>
Date: Sun, 18 Aug 2024 13:45:20 +0300
Subject: [PATCH 292/441] [Clang] warn on discarded [[nodiscard]] function
 results after casting in C (#104677)

Fixes #104391
---
 clang/docs/ReleaseNotes.rst     | 2 ++
 clang/lib/Sema/SemaStmt.cpp     | 3 ++-
 clang/test/Sema/c2x-nodiscard.c | 6 ++++++
 3 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 12a3707db8a39f7..11301e4c7dc4a22 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -218,6 +218,8 @@ Improvements to Clang's diagnostics
 
 - Clang now diagnoses the use of ``main`` in an ``extern`` context as invalid according to [basic.start.main] p3. Fixes #GH101512.
 
+- Clang now diagnoses when the result of a [[nodiscard]] function is discarded after being cast in C. Fixes #GH104391.
+
 Improvements to Clang's time-trace
 ----------------------------------
 
diff --git a/clang/lib/Sema/SemaStmt.cpp b/clang/lib/Sema/SemaStmt.cpp
index d283eaa511011b3..ba681671eb3290f 100644
--- a/clang/lib/Sema/SemaStmt.cpp
+++ b/clang/lib/Sema/SemaStmt.cpp
@@ -281,7 +281,8 @@ void Sema::DiagnoseUnusedExprResult(const Stmt *S, unsigned DiagID) {
   E = WarnExpr;
   if (const auto *Cast = dyn_cast<CastExpr>(E))
     if (Cast->getCastKind() == CK_NoOp ||
-        Cast->getCastKind() == CK_ConstructorConversion)
+        Cast->getCastKind() == CK_ConstructorConversion ||
+        Cast->getCastKind() == CK_IntegralCast)
       E = Cast->getSubExpr()->IgnoreImpCasts();
 
   if (const CallExpr *CE = dyn_cast<CallExpr>(E)) {
diff --git a/clang/test/Sema/c2x-nodiscard.c b/clang/test/Sema/c2x-nodiscard.c
index cb33c0c242e7db9..f8b0567366465df 100644
--- a/clang/test/Sema/c2x-nodiscard.c
+++ b/clang/test/Sema/c2x-nodiscard.c
@@ -54,3 +54,9 @@ void test_missiles(void) {
   launch_missiles();
 }
 
+[[nodiscard]] int f3();
+
+void GH104391() {
+#define M (unsigned int) f3()
+  M; // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
+}

From 71801707e33c235656b172fa7dfb8662473a95c2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timm=20B=C3=A4der?= <tbaeder@redhat.com>
Date: Sun, 18 Aug 2024 13:02:37 +0200
Subject: [PATCH 293/441] [clang][test] Remove bytecode interpreter RUN line
 from test

Specifically, from constant-builtins2. This breaks runners using
sanitizers.

See https://github.com/llvm/llvm-project/pull/94118
---
 clang/test/Sema/constant-builtins-2.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/clang/test/Sema/constant-builtins-2.c b/clang/test/Sema/constant-builtins-2.c
index c359eba4e3d16ca..da2264500d76802 100644
--- a/clang/test/Sema/constant-builtins-2.c
+++ b/clang/test/Sema/constant-builtins-2.c
@@ -1,5 +1,4 @@
 // RUN: %clang_cc1 -fsyntax-only -verify %s
-// RUN: %clang_cc1 -fsyntax-only -fexperimental-new-constant-interpreter -verify %s
 
 // Math stuff
 

From e9e3a183d6d8d8729223b7131aa57fd9f02f0440 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Sun, 18 Aug 2024 12:51:17 +0100
Subject: [PATCH 294/441] [LV] Don't cost branches and conditions to empty
 blocks.

Update the legacy cost model skip branches with successors blocks
that are empty or only contain dead instructions, together with their
conditions. Such branches and conditions won't result in any
generated code and will be cleaned up by VPlan transforms.

This fixes a difference between the legacy and VPlan-based cost model.

When running LV in its usual pipeline position, such dead blocks should
already have been cleaned up, but they might be generated manually or by
fuzzers.

Fixes https://github.com/llvm/llvm-project/issues/100591.
---
 .../Transforms/Vectorize/LoopVectorize.cpp    |  42 +-
 .../RISCV/blocks-with-dead-instructions.ll    | 850 ++++++++++++++++++
 2 files changed, 890 insertions(+), 2 deletions(-)
 create mode 100644 llvm/test/Transforms/LoopVectorize/RISCV/blocks-with-dead-instructions.ll

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 27e99694aafd305..55c0ba3dd8f9bc1 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6730,9 +6730,12 @@ void LoopVectorizationCostModel::collectValuesToIgnore() {
     return RequiresScalarEpilogue &&
            !TheLoop->contains(cast<Instruction>(U)->getParent());
   };
+
+  LoopBlocksDFS DFS(TheLoop);
+  DFS.perform(LI);
   MapVector<Value *, SmallVector<Value *>> DeadInvariantStoreOps;
-  for (BasicBlock *BB : TheLoop->blocks())
-    for (Instruction &I : *BB) {
+  for (BasicBlock *BB : reverse(make_range(DFS.beginRPO(), DFS.endRPO())))
+    for (Instruction &I : reverse(*BB)) {
       // Find all stores to invariant variables. Since they are going to sink
       // outside the loop we do not need calculate cost for them.
       StoreInst *SI;
@@ -6765,6 +6768,13 @@ void LoopVectorizationCostModel::collectValuesToIgnore() {
         Value *PointerOp = getLoadStorePointerOperand(&I);
         DeadInterleavePointerOps.push_back(PointerOp);
       }
+
+      // Queue branches for analysis. They are dead, if their successors only
+      // contain dead instructions.
+      if (auto *Br = dyn_cast<BranchInst>(&I)) {
+        if (Br->isConditional())
+          DeadOps.push_back(&I);
+      }
     }
 
   // Mark ops feeding interleave group members as free, if they are only used
@@ -6789,8 +6799,36 @@ void LoopVectorizationCostModel::collectValuesToIgnore() {
   // Mark ops that would be trivially dead and are only used by ignored
   // instructions as free.
   BasicBlock *Header = TheLoop->getHeader();
+
+  // Returns true if the block contains only dead instructions. Such blocks will
+  // be removed by VPlan-to-VPlan transforms and won't be considered by the
+  // VPlan-based cost model, so skip them in the legacy cost-model as well.
+  auto IsEmptyBlock = [this](BasicBlock *BB) {
+    return all_of(*BB, [this](Instruction &I) {
+      return ValuesToIgnore.contains(&I) || VecValuesToIgnore.contains(&I) ||
+             (isa<BranchInst>(&I) && !cast<BranchInst>(&I)->isConditional());
+    });
+  };
   for (unsigned I = 0; I != DeadOps.size(); ++I) {
     auto *Op = dyn_cast<Instruction>(DeadOps[I]);
+
+    // Check if the branch should be considered dead.
+    if (auto *Br = dyn_cast_or_null<BranchInst>(Op)) {
+      BasicBlock *ThenBB = Br->getSuccessor(0);
+      BasicBlock *ElseBB = Br->getSuccessor(1);
+      bool ThenEmpty = IsEmptyBlock(ThenBB);
+      bool ElseEmpty = IsEmptyBlock(ElseBB);
+      if ((ThenEmpty && ElseEmpty) ||
+          (ThenEmpty && ThenBB->getSingleSuccessor() == ElseBB &&
+           ElseBB->phis().empty()) ||
+          (ElseEmpty && ElseBB->getSingleSuccessor() == ThenBB &&
+           ThenBB->phis().empty())) {
+        VecValuesToIgnore.insert(Br);
+        DeadOps.push_back(Br->getCondition());
+      }
+      continue;
+    }
+
     // Skip any op that shouldn't be considered dead.
     if (!Op || !TheLoop->contains(Op) ||
         (isa<PHINode>(Op) && Op->getParent() == Header) ||
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/blocks-with-dead-instructions.ll b/llvm/test/Transforms/LoopVectorize/RISCV/blocks-with-dead-instructions.ll
new file mode 100644
index 000000000000000..d970b427d035da7
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/blocks-with-dead-instructions.ll
@@ -0,0 +1,850 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -p loop-vectorize -S %s | FileCheck %s
+
+target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128"
+target triple = "riscv64-unknown-linux-gnu"
+
+define void @block_with_dead_inst_1(ptr %src, i64 %N) #0 {
+; CHECK-LABEL: define void @block_with_dead_inst_1(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[N]], -3
+; CHECK-NEXT:    [[TMP1:%.*]] = udiv i64 [[TMP0]], 3
+; CHECK-NEXT:    [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 8
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP2]], [[TMP4]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 8
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], [[TMP6]]
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], i64 [[TMP6]], i64 [[N_MOD_VF]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP2]], [[TMP8]]
+; CHECK-NEXT:    [[IND_END:%.*]] = mul i64 [[N_VEC]], 3
+; CHECK-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 8
+; CHECK-NEXT:    [[TMP11:%.*]] = call <vscale x 8 x i64> @llvm.experimental.stepvector.nxv8i64()
+; CHECK-NEXT:    [[TMP12:%.*]] = add <vscale x 8 x i64> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = mul <vscale x 8 x i64> [[TMP12]], shufflevector (<vscale x 8 x i64> insertelement (<vscale x 8 x i64> poison, i64 3, i64 0), <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer)
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 8 x i64> zeroinitializer, [[TMP13]]
+; CHECK-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP15:%.*]] = mul i64 [[TMP14]], 8
+; CHECK-NEXT:    [[TMP16:%.*]] = mul i64 3, [[TMP15]]
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[TMP16]], i64 0
+; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[DOTSPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP18:%.*]] = mul i32 [[TMP17]], 8
+; CHECK-NEXT:    [[TMP19:%.*]] = sub i32 [[TMP18]], 1
+; CHECK-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 8 x i16> poison, i16 1, i32 [[TMP19]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 8 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <vscale x 8 x i16> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ zeroinitializer, %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr i16, ptr [[SRC]], <vscale x 8 x i64> [[VEC_IND]]
+; CHECK-NEXT:    call void @llvm.masked.scatter.nxv8i16.nxv8p0(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x ptr> [[TMP20]], i32 2, <vscale x 8 x i1> shufflevector (<vscale x 8 x i1> insertelement (<vscale x 8 x i1> poison, i1 true, i64 0), <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer))
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]]
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 8 x i64> [[VEC_IND]], [[DOTSPLAT]]
+; CHECK-NEXT:    [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[TMP22:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP23:%.*]] = mul i32 [[TMP22]], 8
+; CHECK-NEXT:    [[TMP24:%.*]] = sub i32 [[TMP23]], 1
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 8 x i16> zeroinitializer, i32 [[TMP24]]
+; CHECK-NEXT:    br label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 1, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
+; CHECK:       [[LOOP_HEADER]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[XOR1315:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[XOR:%.*]], %[[LOOP_LATCH]] ]
+; CHECK-NEXT:    [[XOR]] = xor i16 0, 0
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[GEP]], align 2
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i16 [[L]], 0
+; CHECK-NEXT:    br i1 [[C]], label %[[THEN:.*]], label %[[LOOP_LATCH]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[DEAD_GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    br label %[[LOOP_LATCH]]
+; CHECK:       [[LOOP_LATCH]]:
+; CHECK-NEXT:    store i16 [[XOR]], ptr [[GEP]], align 2
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
+; CHECK-NEXT:    [[TMP25:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[TMP25]], label %[[EXIT:.*]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop.header
+
+loop.header:                                         ; preds = %cond.end7, %entry
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  %xor1315 = phi i16 [ 1, %entry ], [ %xor, %loop.latch ]
+  %xor = xor i16 0, 0
+  %gep = getelementptr i16, ptr %src, i64 %iv
+  %l = load i16, ptr %gep, align 2
+  %c = icmp eq i16 %l, 0
+  br i1 %c, label %then, label %loop.latch
+
+then:
+  %dead.gep = getelementptr i64, ptr %src, i64 %iv
+  br label %loop.latch
+
+loop.latch:
+  store i16 %xor, ptr %gep
+  %iv.next = add nsw i64 %iv, 3
+  %1 = icmp eq i64 %iv.next, %N
+  br i1 %1, label %exit, label %loop.header
+
+exit:
+  ret void
+}
+
+define void @block_with_dead_inst_2(ptr %src) #0 {
+; CHECK-LABEL: define void @block_with_dead_inst_2(
+; CHECK-SAME: ptr [[SRC:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 333, [[TMP1]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 333, [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = select i1 [[TMP4]], i64 [[TMP3]], i64 [[N_MOD_VF]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 333, [[TMP5]]
+; CHECK-NEXT:    [[IND_END:%.*]] = mul i64 [[N_VEC]], 3
+; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 4
+; CHECK-NEXT:    [[TMP8:%.*]] = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64()
+; CHECK-NEXT:    [[TMP9:%.*]] = add <vscale x 4 x i64> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = mul <vscale x 4 x i64> [[TMP9]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 3, i64 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i64> zeroinitializer, [[TMP10]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP12:%.*]] = mul i64 [[TMP11]], 4
+; CHECK-NEXT:    [[TMP13:%.*]] = mul i64 3, [[TMP12]]
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP13]], i64 0
+; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP15:%.*]] = mul i32 [[TMP14]], 4
+; CHECK-NEXT:    [[TMP16:%.*]] = sub i32 [[TMP15]], 1
+; CHECK-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i16> poison, i16 0, i32 [[TMP16]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <vscale x 4 x i16> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ zeroinitializer, %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr i16, ptr [[SRC]], <vscale x 4 x i64> [[VEC_IND]]
+; CHECK-NEXT:    call void @llvm.masked.scatter.nxv4i16.nxv4p0(<vscale x 4 x i16> zeroinitializer, <vscale x 4 x ptr> [[TMP17]], i32 2, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP7]]
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]
+; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[TMP19:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP20:%.*]] = mul i32 [[TMP19]], 4
+; CHECK-NEXT:    [[TMP21:%.*]] = sub i32 [[TMP20]], 1
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 4 x i16> zeroinitializer, i32 [[TMP21]]
+; CHECK-NEXT:    br label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
+; CHECK:       [[LOOP_HEADER]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[XOR1315:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[XOR:%.*]], %[[LOOP_LATCH]] ]
+; CHECK-NEXT:    [[XOR]] = xor i16 0, 0
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[GEP]], align 2
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i16 [[L]], 0
+; CHECK-NEXT:    br i1 [[C]], label %[[LOOP_LATCH]], label %[[ELSE:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[DEAD_GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    br label %[[LOOP_LATCH]]
+; CHECK:       [[LOOP_LATCH]]:
+; CHECK-NEXT:    store i16 [[XOR]], ptr [[GEP]], align 2
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop.header
+
+loop.header:                                         ; preds = %cond.end7, %entry
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  %xor1315 = phi i16 [ 0, %entry ], [ %xor, %loop.latch ]
+  %xor = xor i16 0, 0
+  %gep = getelementptr i16, ptr %src, i64 %iv
+  %l = load i16, ptr %gep, align 2
+  %c = icmp eq i16 %l, 0
+  br i1 %c, label %loop.latch, label %else
+
+else:
+  %dead.gep = getelementptr i64, ptr %src, i64 %iv
+  br label %loop.latch
+
+loop.latch:
+  store i16 %xor, ptr %gep
+  %iv.next = add nsw i64 %iv, 3
+  %ec = icmp eq i64 %iv.next, 1000
+  br i1 %ec, label %exit, label %loop.header
+
+exit:
+  ret void
+}
+
+define void @multiple_blocks_with_dead_insts_3(ptr %src) #0 {
+; CHECK-LABEL: define void @multiple_blocks_with_dead_insts_3(
+; CHECK-SAME: ptr [[SRC:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 333, [[TMP1]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 333, [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = select i1 [[TMP4]], i64 [[TMP3]], i64 [[N_MOD_VF]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 333, [[TMP5]]
+; CHECK-NEXT:    [[IND_END:%.*]] = mul i64 [[N_VEC]], 3
+; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 4
+; CHECK-NEXT:    [[TMP8:%.*]] = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64()
+; CHECK-NEXT:    [[TMP9:%.*]] = add <vscale x 4 x i64> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = mul <vscale x 4 x i64> [[TMP9]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 3, i64 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i64> zeroinitializer, [[TMP10]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP12:%.*]] = mul i64 [[TMP11]], 4
+; CHECK-NEXT:    [[TMP13:%.*]] = mul i64 3, [[TMP12]]
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP13]], i64 0
+; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP15:%.*]] = mul i32 [[TMP14]], 4
+; CHECK-NEXT:    [[TMP16:%.*]] = sub i32 [[TMP15]], 1
+; CHECK-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i16> poison, i16 0, i32 [[TMP16]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <vscale x 4 x i16> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ zeroinitializer, %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr i16, ptr [[SRC]], <vscale x 4 x i64> [[VEC_IND]]
+; CHECK-NEXT:    call void @llvm.masked.scatter.nxv4i16.nxv4p0(<vscale x 4 x i16> zeroinitializer, <vscale x 4 x ptr> [[TMP17]], i32 2, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP7]]
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]
+; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[TMP19:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP20:%.*]] = mul i32 [[TMP19]], 4
+; CHECK-NEXT:    [[TMP21:%.*]] = sub i32 [[TMP20]], 1
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 4 x i16> zeroinitializer, i32 [[TMP21]]
+; CHECK-NEXT:    br label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
+; CHECK:       [[LOOP_HEADER]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[XOR1315:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[XOR:%.*]], %[[LOOP_LATCH]] ]
+; CHECK-NEXT:    [[XOR]] = xor i16 0, 0
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[GEP]], align 2
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i16 [[L]], 0
+; CHECK-NEXT:    br i1 [[C]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[DEAD_GEP_1:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    br label %[[LOOP_LATCH]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[DEAD_GEP_2:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    br label %[[LOOP_LATCH]]
+; CHECK:       [[LOOP_LATCH]]:
+; CHECK-NEXT:    store i16 [[XOR]], ptr [[GEP]], align 2
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop.header
+
+loop.header:                                         ; preds = %cond.end7, %entry
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  %xor1315 = phi i16 [ 0, %entry ], [ %xor, %loop.latch ]
+  %xor = xor i16 0, 0
+  %gep = getelementptr i16, ptr %src, i64 %iv
+  %l = load i16, ptr %gep, align 2
+  %c = icmp eq i16 %l, 0
+  br i1 %c, label %then, label %else
+
+then:
+  %dead.gep.1 = getelementptr i64, ptr %src, i64 %iv
+  br label %loop.latch
+
+else:
+  %dead.gep.2 = getelementptr i64, ptr %src, i64 %iv
+  br label %loop.latch
+
+loop.latch:
+  store i16 %xor, ptr %gep
+  %iv.next = add nsw i64 %iv, 3
+  %ec = icmp eq i64 %iv.next, 1000
+  br i1 %ec, label %exit, label %loop.header
+
+exit:
+  ret void
+}
+
+define void @multiple_blocks_with_dead_insts_4(ptr %src, i64 %N) #0 {
+; CHECK-LABEL: define void @multiple_blocks_with_dead_insts_4(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[N]], -3
+; CHECK-NEXT:    [[TMP1:%.*]] = udiv i64 [[TMP0]], 3
+; CHECK-NEXT:    [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 8
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP2]], [[TMP4]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 8
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], [[TMP6]]
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], i64 [[TMP6]], i64 [[N_MOD_VF]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP2]], [[TMP8]]
+; CHECK-NEXT:    [[IND_END:%.*]] = mul i64 [[N_VEC]], 3
+; CHECK-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 8
+; CHECK-NEXT:    [[TMP11:%.*]] = call <vscale x 8 x i64> @llvm.experimental.stepvector.nxv8i64()
+; CHECK-NEXT:    [[TMP12:%.*]] = add <vscale x 8 x i64> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = mul <vscale x 8 x i64> [[TMP12]], shufflevector (<vscale x 8 x i64> insertelement (<vscale x 8 x i64> poison, i64 3, i64 0), <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer)
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 8 x i64> zeroinitializer, [[TMP13]]
+; CHECK-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP15:%.*]] = mul i64 [[TMP14]], 8
+; CHECK-NEXT:    [[TMP16:%.*]] = mul i64 3, [[TMP15]]
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[TMP16]], i64 0
+; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[DOTSPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP18:%.*]] = mul i32 [[TMP17]], 8
+; CHECK-NEXT:    [[TMP19:%.*]] = sub i32 [[TMP18]], 1
+; CHECK-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 8 x i16> poison, i16 1, i32 [[TMP19]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 8 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <vscale x 8 x i16> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ zeroinitializer, %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr i16, ptr [[SRC]], <vscale x 8 x i64> [[VEC_IND]]
+; CHECK-NEXT:    call void @llvm.masked.scatter.nxv8i16.nxv8p0(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x ptr> [[TMP20]], i32 2, <vscale x 8 x i1> shufflevector (<vscale x 8 x i1> insertelement (<vscale x 8 x i1> poison, i1 true, i64 0), <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer))
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]]
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 8 x i64> [[VEC_IND]], [[DOTSPLAT]]
+; CHECK-NEXT:    [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[TMP22:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP23:%.*]] = mul i32 [[TMP22]], 8
+; CHECK-NEXT:    [[TMP24:%.*]] = sub i32 [[TMP23]], 1
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 8 x i16> zeroinitializer, i32 [[TMP24]]
+; CHECK-NEXT:    br label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 1, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
+; CHECK:       [[LOOP_HEADER]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[XOR1315:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[XOR:%.*]], %[[LOOP_LATCH]] ]
+; CHECK-NEXT:    [[XOR]] = xor i16 0, 0
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[GEP]], align 2
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i16 [[L]], 0
+; CHECK-NEXT:    br i1 [[C]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    br label %[[THEN_1:.*]]
+; CHECK:       [[THEN_1]]:
+; CHECK-NEXT:    [[DEAD_GEP_1:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    br label %[[LOOP_LATCH]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[DEAD_GEP_2:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    br label %[[LOOP_LATCH]]
+; CHECK:       [[LOOP_LATCH]]:
+; CHECK-NEXT:    store i16 [[XOR]], ptr [[GEP]], align 2
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop.header
+
+loop.header:                                         ; preds = %cond.end7, %entry
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  %xor1315 = phi i16 [ 1, %entry ], [ %xor, %loop.latch ]
+  %xor = xor i16 0, 0
+  %gep = getelementptr i16, ptr %src, i64 %iv
+  %l = load i16, ptr %gep, align 2
+  %c = icmp eq i16 %l, 0
+  br i1 %c, label %then, label %else
+
+then:
+  br label %then.1
+
+then.1:
+  %dead.gep.1 = getelementptr i64, ptr %src, i64 %iv
+  br label %loop.latch
+
+else:
+  %dead.gep.2 = getelementptr i64, ptr %src, i64 %iv
+  br label %loop.latch
+
+loop.latch:
+  store i16 %xor, ptr %gep
+  %iv.next = add nsw i64 %iv, 3
+  %ec = icmp eq i64 %iv.next, %N
+  br i1 %ec, label %exit, label %loop.header
+
+exit:
+  ret void
+}
+
+define void @multiple_blocks_with_dead_inst_multiple_successors_5(ptr %src) #0 {
+; CHECK-LABEL: define void @multiple_blocks_with_dead_inst_multiple_successors_5(
+; CHECK-SAME: ptr [[SRC:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 333, [[TMP1]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 333, [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = select i1 [[TMP4]], i64 [[TMP3]], i64 [[N_MOD_VF]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 333, [[TMP5]]
+; CHECK-NEXT:    [[IND_END:%.*]] = mul i64 [[N_VEC]], 3
+; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 4
+; CHECK-NEXT:    [[TMP8:%.*]] = call <vscale x 4 x i64> @llvm.experimental.stepvector.nxv4i64()
+; CHECK-NEXT:    [[TMP9:%.*]] = add <vscale x 4 x i64> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = mul <vscale x 4 x i64> [[TMP9]], shufflevector (<vscale x 4 x i64> insertelement (<vscale x 4 x i64> poison, i64 3, i64 0), <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer)
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i64> zeroinitializer, [[TMP10]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP12:%.*]] = mul i64 [[TMP11]], 4
+; CHECK-NEXT:    [[TMP13:%.*]] = mul i64 3, [[TMP12]]
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP13]], i64 0
+; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP15:%.*]] = mul i32 [[TMP14]], 4
+; CHECK-NEXT:    [[TMP16:%.*]] = sub i32 [[TMP15]], 1
+; CHECK-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i16> poison, i16 1, i32 [[TMP16]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <vscale x 4 x i16> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ zeroinitializer, %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr i16, ptr [[SRC]], <vscale x 4 x i64> [[VEC_IND]]
+; CHECK-NEXT:    call void @llvm.masked.scatter.nxv4i16.nxv4p0(<vscale x 4 x i16> zeroinitializer, <vscale x 4 x ptr> [[TMP17]], i32 2, <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1 true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP7]]
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]
+; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[TMP19:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP20:%.*]] = mul i32 [[TMP19]], 4
+; CHECK-NEXT:    [[TMP21:%.*]] = sub i32 [[TMP20]], 1
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 4 x i16> zeroinitializer, i32 [[TMP21]]
+; CHECK-NEXT:    br label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 1, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
+; CHECK:       [[LOOP_HEADER]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[XOR1315:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[XOR:%.*]], %[[LOOP_LATCH]] ]
+; CHECK-NEXT:    [[XOR]] = xor i16 0, 0
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[GEP]], align 2
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i16 [[L]], 0
+; CHECK-NEXT:    br i1 [[C]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    br label %[[THEN_1:.*]]
+; CHECK:       [[THEN_1]]:
+; CHECK-NEXT:    [[DEAD_GEP_1:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    br label %[[LOOP_LATCH]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    br label %[[ELSE_2:.*]]
+; CHECK:       [[ELSE_2]]:
+; CHECK-NEXT:    [[DEAD_GEP_2:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    br label %[[LOOP_LATCH]]
+; CHECK:       [[LOOP_LATCH]]:
+; CHECK-NEXT:    store i16 [[XOR]], ptr [[GEP]], align 2
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop.header
+
+loop.header:                                         ; preds = %cond.end7, %entry
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  %xor1315 = phi i16 [ 1, %entry ], [ %xor, %loop.latch ]
+  %xor = xor i16 0, 0
+  %gep = getelementptr i16, ptr %src, i64 %iv
+  %l = load i16, ptr %gep, align 2
+  %c = icmp eq i16 %l, 0
+  br i1 %c, label %then, label %else
+
+then:
+  br label %then.1
+
+then.1:
+  %dead.gep.1 = getelementptr i64, ptr %src, i64 %iv
+  br label %loop.latch
+
+else:
+  br label %else.2
+
+else.2:
+  %dead.gep.2 = getelementptr i64, ptr %src, i64 %iv
+  br label %loop.latch
+
+loop.latch:
+  store i16 %xor, ptr %gep
+  %iv.next = add nsw i64 %iv, 3
+  %ec = icmp eq i64 %iv.next, 1000
+  br i1 %ec, label %exit, label %loop.header
+
+exit:
+  ret void
+}
+
+define void @multiple_blocks_with_dead_inst_multiple_successors_6(ptr %src, i1 %ic, i64 %N) #0 {
+; CHECK-LABEL: define void @multiple_blocks_with_dead_inst_multiple_successors_6(
+; CHECK-SAME: ptr [[SRC:%.*]], i1 [[IC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[N]], -3
+; CHECK-NEXT:    [[TMP1:%.*]] = udiv i64 [[TMP0]], 3
+; CHECK-NEXT:    [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 8
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP2]], [[TMP4]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 8
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], [[TMP6]]
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], i64 [[TMP6]], i64 [[N_MOD_VF]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP2]], [[TMP8]]
+; CHECK-NEXT:    [[IND_END:%.*]] = mul i64 [[N_VEC]], 3
+; CHECK-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 8
+; CHECK-NEXT:    [[TMP11:%.*]] = call <vscale x 8 x i64> @llvm.experimental.stepvector.nxv8i64()
+; CHECK-NEXT:    [[TMP12:%.*]] = add <vscale x 8 x i64> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = mul <vscale x 8 x i64> [[TMP12]], shufflevector (<vscale x 8 x i64> insertelement (<vscale x 8 x i64> poison, i64 3, i64 0), <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer)
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 8 x i64> zeroinitializer, [[TMP13]]
+; CHECK-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP15:%.*]] = mul i64 [[TMP14]], 8
+; CHECK-NEXT:    [[TMP16:%.*]] = mul i64 3, [[TMP15]]
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[TMP16]], i64 0
+; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[DOTSPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP18:%.*]] = mul i32 [[TMP17]], 8
+; CHECK-NEXT:    [[TMP19:%.*]] = sub i32 [[TMP18]], 1
+; CHECK-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 8 x i16> poison, i16 1, i32 [[TMP19]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 8 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <vscale x 8 x i16> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ zeroinitializer, %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr i16, ptr [[SRC]], <vscale x 8 x i64> [[VEC_IND]]
+; CHECK-NEXT:    call void @llvm.masked.scatter.nxv8i16.nxv8p0(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x ptr> [[TMP20]], i32 2, <vscale x 8 x i1> shufflevector (<vscale x 8 x i1> insertelement (<vscale x 8 x i1> poison, i1 true, i64 0), <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer))
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]]
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 8 x i64> [[VEC_IND]], [[DOTSPLAT]]
+; CHECK-NEXT:    [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[TMP22:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP23:%.*]] = mul i32 [[TMP22]], 8
+; CHECK-NEXT:    [[TMP24:%.*]] = sub i32 [[TMP23]], 1
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 8 x i16> zeroinitializer, i32 [[TMP24]]
+; CHECK-NEXT:    br label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 1, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
+; CHECK:       [[LOOP_HEADER]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[XOR1315:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[XOR:%.*]], %[[LOOP_LATCH]] ]
+; CHECK-NEXT:    [[XOR]] = xor i16 0, 0
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[GEP]], align 2
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i16 [[L]], 0
+; CHECK-NEXT:    br i1 [[C]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    br i1 [[IC]], label %[[THEN_1:.*]], label %[[ELSE]]
+; CHECK:       [[THEN_1]]:
+; CHECK-NEXT:    [[DEAD_GEP_1:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    br label %[[LOOP_LATCH]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    br label %[[ELSE_2:.*]]
+; CHECK:       [[ELSE_2]]:
+; CHECK-NEXT:    [[DEAD_GEP_2:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    br label %[[LOOP_LATCH]]
+; CHECK:       [[LOOP_LATCH]]:
+; CHECK-NEXT:    store i16 [[XOR]], ptr [[GEP]], align 2
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop.header
+
+loop.header:                                         ; preds = %cond.end7, %entry
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  %xor1315 = phi i16 [ 1, %entry ], [ %xor, %loop.latch ]
+  %xor = xor i16 0, 0
+  %gep = getelementptr i16, ptr %src, i64 %iv
+  %l = load i16, ptr %gep, align 2
+  %c = icmp eq i16 %l, 0
+  br i1 %c, label %then, label %else
+
+then:
+  br i1 %ic, label %then.1, label %else
+
+then.1:
+  %dead.gep.1 = getelementptr i64, ptr %src, i64 %iv
+  br label %loop.latch
+
+else:
+  br label %else.2
+
+else.2:
+  %dead.gep.2 = getelementptr i64, ptr %src, i64 %iv
+  br label %loop.latch
+
+loop.latch:
+  store i16 %xor, ptr %gep
+  %iv.next = add nsw i64 %iv, 3
+  %ec = icmp eq i64 %iv.next, %N
+  br i1 %ec, label %exit, label %loop.header
+
+exit:
+  ret void
+}
+
+define void @empty_block_with_phi_1(ptr %src, i64 %N) #0 {
+; CHECK-LABEL: define void @empty_block_with_phi_1(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 8
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
+; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP7:%.*]] = mul i32 [[TMP6]], 8
+; CHECK-NEXT:    [[TMP8:%.*]] = sub i32 [[TMP7]], 1
+; CHECK-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 8 x i32> poison, i32 1, i32 [[TMP8]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <vscale x 8 x i32> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ zeroinitializer, %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr i16, ptr [[TMP10]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP11]], align 2
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq <vscale x 8 x i16> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <vscale x 8 x i1> [[TMP12]], <vscale x 8 x i16> shufflevector (<vscale x 8 x i16> insertelement (<vscale x 8 x i16> poison, i16 99, i64 0), <vscale x 8 x i16> poison, <vscale x 8 x i32> zeroinitializer), <vscale x 8 x i16> [[WIDE_LOAD]]
+; CHECK-NEXT:    store <vscale x 8 x i16> [[PREDPHI]], ptr [[TMP11]], align 2
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[TMP14:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP15:%.*]] = mul i32 [[TMP14]], 8
+; CHECK-NEXT:    [[TMP16:%.*]] = sub i32 [[TMP15]], 1
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 8 x i32> zeroinitializer, i32 [[TMP16]]
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 1, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
+; CHECK:       [[LOOP_HEADER]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[XOR1315:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[XOR:%.*]], %[[LOOP_LATCH]] ]
+; CHECK-NEXT:    [[XOR]] = xor i32 0, 0
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[GEP]], align 2
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i16 [[L]], 0
+; CHECK-NEXT:    br i1 [[C]], label %[[THEN:.*]], label %[[LOOP_LATCH]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    br label %[[LOOP_LATCH]]
+; CHECK:       [[LOOP_LATCH]]:
+; CHECK-NEXT:    [[P:%.*]] = phi i16 [ [[L]], %[[LOOP_HEADER]] ], [ 99, %[[THEN]] ]
+; CHECK-NEXT:    store i16 [[P]], ptr [[GEP]], align 2
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[TMP17]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop.header
+
+loop.header:                                         ; preds = %cond.end7, %entry
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  %xor1315 = phi i32 [ 1, %entry ], [ %xor, %loop.latch ]
+  %xor = xor i32 0, 0
+  %gep = getelementptr i16, ptr %src, i64 %iv
+  %l = load i16, ptr %gep, align 2
+  %c = icmp eq i16 %l, 0
+  br i1 %c, label %then, label %loop.latch
+
+then:
+  br label %loop.latch
+
+loop.latch:
+  %p = phi i16 [ %l, %loop.header ], [ 99, %then ]
+  store i16 %p, ptr %gep
+  %iv.next = add nsw i64 %iv, 1
+  %1 = icmp eq i64 %iv.next, %N
+  br i1 %1, label %exit, label %loop.header
+
+exit:
+  ret void
+}
+
+define void @empty_block_with_phi_2(ptr %src, i64 %N) #0 {
+; CHECK-LABEL: define void @empty_block_with_phi_2(
+; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 8
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
+; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP7:%.*]] = mul i32 [[TMP6]], 8
+; CHECK-NEXT:    [[TMP8:%.*]] = sub i32 [[TMP7]], 1
+; CHECK-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 8 x i32> poison, i32 1, i32 [[TMP8]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <vscale x 8 x i32> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ zeroinitializer, %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr i16, ptr [[TMP10]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP11]], align 2
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq <vscale x 8 x i16> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <vscale x 8 x i1> [[TMP12]], shufflevector (<vscale x 8 x i1> insertelement (<vscale x 8 x i1> poison, i1 true, i64 0), <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer)
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <vscale x 8 x i1> [[TMP13]], <vscale x 8 x i16> shufflevector (<vscale x 8 x i16> insertelement (<vscale x 8 x i16> poison, i16 99, i64 0), <vscale x 8 x i16> poison, <vscale x 8 x i32> zeroinitializer), <vscale x 8 x i16> [[WIDE_LOAD]]
+; CHECK-NEXT:    store <vscale x 8 x i16> [[PREDPHI]], ptr [[TMP11]], align 2
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[TMP15:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP16:%.*]] = mul i32 [[TMP15]], 8
+; CHECK-NEXT:    [[TMP17:%.*]] = sub i32 [[TMP16]], 1
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 8 x i32> zeroinitializer, i32 [[TMP17]]
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 1, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
+; CHECK:       [[LOOP_HEADER]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[XOR1315:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[XOR:%.*]], %[[LOOP_LATCH]] ]
+; CHECK-NEXT:    [[XOR]] = xor i32 0, 0
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[GEP]], align 2
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i16 [[L]], 0
+; CHECK-NEXT:    br i1 [[C]], label %[[LOOP_LATCH]], label %[[ELSE:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    br label %[[LOOP_LATCH]]
+; CHECK:       [[LOOP_LATCH]]:
+; CHECK-NEXT:    [[P:%.*]] = phi i16 [ [[L]], %[[LOOP_HEADER]] ], [ 99, %[[ELSE]] ]
+; CHECK-NEXT:    store i16 [[P]], ptr [[GEP]], align 2
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[TMP18]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop.header
+
+loop.header:                                         ; preds = %cond.end7, %entry
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  %xor1315 = phi i32 [ 1, %entry ], [ %xor, %loop.latch ]
+  %xor = xor i32 0, 0
+  %gep = getelementptr i16, ptr %src, i64 %iv
+  %l = load i16, ptr %gep, align 2
+  %c = icmp eq i16 %l, 0
+  br i1 %c, label %loop.latch, label %else
+
+else:
+  br label %loop.latch
+
+loop.latch:
+  %p = phi i16 [ %l, %loop.header ], [ 99, %else ]
+  store i16 %p, ptr %gep
+  %iv.next = add nsw i64 %iv, 1
+  %1 = icmp eq i64 %iv.next, %N
+  br i1 %1, label %exit, label %loop.header
+
+exit:
+  ret void
+}
+
+attributes #0 = { "target-features"="+64bit,+v" }
+;.
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
+; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]}
+; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]}
+; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META2]], [[META1]]}
+; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]}
+; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META2]], [[META1]]}
+; CHECK: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]], [[META2]]}
+; CHECK: [[LOOP11]] = distinct !{[[LOOP11]], [[META2]], [[META1]]}
+; CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META1]], [[META2]]}
+; CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META2]], [[META1]]}
+; CHECK: [[LOOP14]] = distinct !{[[LOOP14]], [[META1]], [[META2]]}
+; CHECK: [[LOOP15]] = distinct !{[[LOOP15]], [[META2]], [[META1]]}
+; CHECK: [[LOOP16]] = distinct !{[[LOOP16]], [[META1]], [[META2]]}
+; CHECK: [[LOOP17]] = distinct !{[[LOOP17]], [[META2]], [[META1]]}
+;.

From ff80e1ffe79d893be990cda981526a321697eb9a Mon Sep 17 00:00:00 2001
From: Yingwei Zheng <dtcxzyw2333@gmail.com>
Date: Sun, 18 Aug 2024 20:55:05 +0800
Subject: [PATCH 295/441] [InstSimplify] Simplify `uadd.sat(X, Y) u>= X + Y`
 and `usub.sat(X, Y) u<= X, Y` (#104698)

These patterns are found in harfbuzz/typst.

Alive2: https://alive2.llvm.org/ce/z/cxyjYV
---
 llvm/lib/Analysis/InstructionSimplify.cpp     |  16 +++
 .../InstSimplify/saturating-add-sub.ll        | 116 ++++++++++++++++++
 2 files changed, 132 insertions(+)

diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp
index eff8a7cfc8ce6c9..32a9f1ab34fb3fc 100644
--- a/llvm/lib/Analysis/InstructionSimplify.cpp
+++ b/llvm/lib/Analysis/InstructionSimplify.cpp
@@ -3718,6 +3718,14 @@ static Value *simplifyICmpWithIntrinsicOnLHS(CmpInst::Predicate Pred,
       if (Pred == ICmpInst::ICMP_ULT)
         return ConstantInt::getFalse(getCompareTy(II));
     }
+    // uadd.sat(X, Y) uge X + Y
+    if (match(RHS, m_c_Add(m_Specific(II->getArgOperand(0)),
+                           m_Specific(II->getArgOperand(1))))) {
+      if (Pred == ICmpInst::ICMP_UGE)
+        return ConstantInt::getTrue(getCompareTy(II));
+      if (Pred == ICmpInst::ICMP_ULT)
+        return ConstantInt::getFalse(getCompareTy(II));
+    }
     return nullptr;
   case Intrinsic::usub_sat:
     // usub.sat(X, Y) ule X
@@ -3727,6 +3735,14 @@ static Value *simplifyICmpWithIntrinsicOnLHS(CmpInst::Predicate Pred,
       if (Pred == ICmpInst::ICMP_UGT)
         return ConstantInt::getFalse(getCompareTy(II));
     }
+    // usub.sat(X, Y) ule X - Y
+    if (match(RHS, m_Sub(m_Specific(II->getArgOperand(0)),
+                         m_Specific(II->getArgOperand(1))))) {
+      if (Pred == ICmpInst::ICMP_ULE)
+        return ConstantInt::getTrue(getCompareTy(II));
+      if (Pred == ICmpInst::ICMP_UGT)
+        return ConstantInt::getFalse(getCompareTy(II));
+    }
     return nullptr;
   default:
     return nullptr;
diff --git a/llvm/test/Transforms/InstSimplify/saturating-add-sub.ll b/llvm/test/Transforms/InstSimplify/saturating-add-sub.ll
index 40b22c619f76867..ea139f2411cc3df 100644
--- a/llvm/test/Transforms/InstSimplify/saturating-add-sub.ll
+++ b/llvm/test/Transforms/InstSimplify/saturating-add-sub.ll
@@ -932,3 +932,119 @@ define i1 @usub_uge_fail(i8 %x, i8 %y) {
   %cmp = icmp uge i8 %sat, %x
   ret i1 %cmp
 }
+
+define i1 @icmp_ult_uaddsat_add(i32 %x, i32 %y) {
+; CHECK-LABEL: @icmp_ult_uaddsat_add(
+; CHECK-NEXT:    ret i1 false
+;
+  %uaddsat = call i32 @llvm.uadd.sat.i32(i32 %x, i32 %y)
+  %add = add i32 %x, %y
+  %cmp = icmp ult i32 %uaddsat, %add
+  ret i1 %cmp
+}
+
+define i1 @icmp_uge_uaddsat_add(i32 %x, i32 %y) {
+; CHECK-LABEL: @icmp_uge_uaddsat_add(
+; CHECK-NEXT:    ret i1 true
+;
+  %uaddsat = call i32 @llvm.uadd.sat.i32(i32 %x, i32 %y)
+  %add = add i32 %x, %y
+  %cmp = icmp uge i32 %uaddsat, %add
+  ret i1 %cmp
+}
+
+define i1 @icmp_ugt_uaddsat_add_commuted1(i32 %x, i32 %y) {
+; CHECK-LABEL: @icmp_ugt_uaddsat_add_commuted1(
+; CHECK-NEXT:    ret i1 false
+;
+  %uaddsat = call i32 @llvm.uadd.sat.i32(i32 %x, i32 %y)
+  %add = add i32 %x, %y
+  %cmp = icmp ugt i32 %add, %uaddsat
+  ret i1 %cmp
+}
+
+define i1 @icmp_ult_uaddsat_add_commuted2(i32 %x, i32 %y) {
+; CHECK-LABEL: @icmp_ult_uaddsat_add_commuted2(
+; CHECK-NEXT:    ret i1 false
+;
+  %xx = mul i32 %x, 998244353
+  %yy = mul i32 %y, 998244353
+  %uaddsat = call i32 @llvm.uadd.sat.i32(i32 %xx, i32 %yy)
+  %add = add i32 %yy, %xx ; thwart complexity-based canonicalization
+  %cmp = icmp ult i32 %uaddsat, %add
+  ret i1 %cmp
+}
+
+define i1 @icmp_ule_usubsat_sub(i32 %x, i32 %y) {
+; CHECK-LABEL: @icmp_ule_usubsat_sub(
+; CHECK-NEXT:    ret i1 true
+;
+  %usubsat = call i32 @llvm.usub.sat.i32(i32 %x, i32 %y)
+  %add = sub i32 %x, %y
+  %cmp = icmp ule i32 %usubsat, %add
+  ret i1 %cmp
+}
+
+define i1 @icmp_ugt_usubsat_sub(i32 %x, i32 %y) {
+; CHECK-LABEL: @icmp_ugt_usubsat_sub(
+; CHECK-NEXT:    ret i1 false
+;
+  %usubsat = call i32 @llvm.usub.sat.i32(i32 %x, i32 %y)
+  %add = sub i32 %x, %y
+  %cmp = icmp ugt i32 %usubsat, %add
+  ret i1 %cmp
+}
+
+; Negative tests
+
+define i1 @icmp_ult_uaddsat_add_mismatch(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @icmp_ult_uaddsat_add_mismatch(
+; CHECK-NEXT:    [[UADDSAT:%.*]] = call i32 @llvm.uadd.sat.i32(i32 [[X:%.*]], i32 [[Z:%.*]])
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[X]], [[Y:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[UADDSAT]], [[ADD]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %uaddsat = call i32 @llvm.uadd.sat.i32(i32 %x, i32 %z)
+  %add = add i32 %x, %y
+  %cmp = icmp ult i32 %uaddsat, %add
+  ret i1 %cmp
+}
+
+define i1 @icmp_ult_uaddsat_add_wrong_pred(i32 %x, i32 %y) {
+; CHECK-LABEL: @icmp_ult_uaddsat_add_wrong_pred(
+; CHECK-NEXT:    [[UADDSAT:%.*]] = call i32 @llvm.uadd.sat.i32(i32 [[X:%.*]], i32 [[Y:%.*]])
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ule i32 [[UADDSAT]], [[ADD]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %uaddsat = call i32 @llvm.uadd.sat.i32(i32 %x, i32 %y)
+  %add = add i32 %x, %y
+  %cmp = icmp ule i32 %uaddsat, %add
+  ret i1 %cmp
+}
+
+define i1 @icmp_ult_uaddsat_add_wrong_op(i32 %x, i32 %y) {
+; CHECK-LABEL: @icmp_ult_uaddsat_add_wrong_op(
+; CHECK-NEXT:    [[UADDSAT:%.*]] = call i32 @llvm.uadd.sat.i32(i32 [[X:%.*]], i32 [[Y:%.*]])
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[UADDSAT]], [[SUB]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %uaddsat = call i32 @llvm.uadd.sat.i32(i32 %x, i32 %y)
+  %sub = sub i32 %x, %y
+  %cmp = icmp ult i32 %uaddsat, %sub
+  ret i1 %cmp
+}
+
+define i1 @icmp_ule_usubsat_sub_commuted(i32 %x, i32 %y) {
+; CHECK-LABEL: @icmp_ule_usubsat_sub_commuted(
+; CHECK-NEXT:    [[USUBSAT:%.*]] = call i32 @llvm.usub.sat.i32(i32 [[X:%.*]], i32 [[Y:%.*]])
+; CHECK-NEXT:    [[ADD:%.*]] = sub i32 [[Y]], [[X]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ule i32 [[USUBSAT]], [[ADD]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %usubsat = call i32 @llvm.usub.sat.i32(i32 %x, i32 %y)
+  %add = sub i32 %y, %x
+  %cmp = icmp ule i32 %usubsat, %add
+  ret i1 %cmp
+}

From 29811a81f77c22523dbfffbd2d39ac731baea37b Mon Sep 17 00:00:00 2001
From: PaulXiCao <paulxicao7@gmail.com>
Date: Sun, 18 Aug 2024 15:00:11 +0200
Subject: [PATCH 296/441] [libc++][docs] Fixing hyperlink for mathematical
 special function documentation (#104444)

Inter-documentation link for progress tracking of mathematical special
function is fixed.
---
 libcxx/docs/Status/Cxx17.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcxx/docs/Status/Cxx17.rst b/libcxx/docs/Status/Cxx17.rst
index 3f1f2071300c803..9263bd7c4af0c8e 100644
--- a/libcxx/docs/Status/Cxx17.rst
+++ b/libcxx/docs/Status/Cxx17.rst
@@ -41,7 +41,7 @@ Paper Status
 .. note::
 
    .. [#note-P0067] P0067: ``std::(to|from)_chars`` for integrals has been available since version 7.0. ``std::to_chars`` for ``float`` and ``double`` since version 14.0 ``std::to_chars`` for ``long double`` uses the implementation for ``double``.
-   .. [#note-P0226] P0226: Progress is tracked `here <https://https://libcxx.llvm.org/Status/SpecialMath.html>`_.
+   .. [#note-P0226] P0226: Progress is tracked `here <https://libcxx.llvm.org/Status/SpecialMath.html>`_.
    .. [#note-P0607] P0607: The parts of P0607 that are not done are the ``<regex>`` bits.
    .. [#note-P0154] P0154: The required macros are only implemented as of clang 19.
    .. [#note-P0452] P0452: The changes to ``std::transform_inclusive_scan`` and ``std::transform_exclusive_scan`` have not yet been implemented.

From 6ed2a6bf0d26122210c8732d80431529784c4000 Mon Sep 17 00:00:00 2001
From: Daniel Chen <cdchen@ca.ibm.com>
Date: Sun, 18 Aug 2024 09:28:17 -0400
Subject: [PATCH 297/441] [Flang]: Use actual endianness for Integer<80>
 (#103928)

This PR is to use the actual endianness of the platform when instantiate
Integer<80> so that AIX (big-endian) will not get confused.
---
 flang/include/flang/Evaluate/integer.h | 5 +++--
 flang/lib/Evaluate/integer.cpp         | 3 ++-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/flang/include/flang/Evaluate/integer.h b/flang/include/flang/Evaluate/integer.h
index 4c7d88328a71e7e..e69876f07b63cbc 100644
--- a/flang/include/flang/Evaluate/integer.h
+++ b/flang/include/flang/Evaluate/integer.h
@@ -1056,8 +1056,9 @@ extern template class Integer<16>;
 extern template class Integer<32>;
 extern template class Integer<64>;
 using X87IntegerContainer =
-    Integer<80, true, 16, std::uint16_t, std::uint32_t, 128>;
-extern template class Integer<80, true, 16, std::uint16_t, std::uint32_t, 128>;
+    Integer<80, isHostLittleEndian, 16, std::uint16_t, std::uint32_t, 128>;
+extern template class Integer<80, isHostLittleEndian, 16, std::uint16_t,
+    std::uint32_t, 128>;
 extern template class Integer<128>;
 } // namespace Fortran::evaluate::value
 #endif // FORTRAN_EVALUATE_INTEGER_H_
diff --git a/flang/lib/Evaluate/integer.cpp b/flang/lib/Evaluate/integer.cpp
index b982a3a0796cc5c..17a38fa17a67ccb 100644
--- a/flang/lib/Evaluate/integer.cpp
+++ b/flang/lib/Evaluate/integer.cpp
@@ -14,7 +14,8 @@ template class Integer<8>;
 template class Integer<16>;
 template class Integer<32>;
 template class Integer<64>;
-template class Integer<80, true, 16, std::uint16_t, std::uint32_t, 128>;
+template class Integer<80, isHostLittleEndian, 16, std::uint16_t, std::uint32_t,
+    128>;
 template class Integer<128>;
 
 // Sanity checks against misconfiguration bugs

From d082f1f37d8cb7a0c6875537ba873a631b154d53 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timm=20B=C3=A4der?= <tbaeder@redhat.com>
Date: Sun, 18 Aug 2024 15:59:59 +0200
Subject: [PATCH 298/441] [clang][bytecode] Only booleans can be inverted

No need to have the Inv() function be templated.
---
 clang/lib/AST/ByteCode/Boolean.h    |  1 +
 clang/lib/AST/ByteCode/Compiler.cpp |  4 ++--
 clang/lib/AST/ByteCode/Interp.h     | 12 +++---------
 clang/lib/AST/ByteCode/Opcodes.td   | 11 ++---------
 4 files changed, 8 insertions(+), 20 deletions(-)

diff --git a/clang/lib/AST/ByteCode/Boolean.h b/clang/lib/AST/ByteCode/Boolean.h
index 0a2e0fe332110e9..f1914ddb9970dc5 100644
--- a/clang/lib/AST/ByteCode/Boolean.h
+++ b/clang/lib/AST/ByteCode/Boolean.h
@@ -44,6 +44,7 @@ class Boolean final {
   Boolean operator-() const { return Boolean(V); }
   Boolean operator-(const Boolean &Other) const { return Boolean(V - Other.V); }
   Boolean operator~() const { return Boolean(true); }
+  Boolean operator!() const { return Boolean(!V); }
 
   template <typename Ty, typename = std::enable_if_t<std::is_integral_v<Ty>>>
   explicit operator Ty() const {
diff --git a/clang/lib/AST/ByteCode/Compiler.cpp b/clang/lib/AST/ByteCode/Compiler.cpp
index c3ecaa40b86f2a1..53144a50ccf4a92 100644
--- a/clang/lib/AST/ByteCode/Compiler.cpp
+++ b/clang/lib/AST/ByteCode/Compiler.cpp
@@ -5112,7 +5112,7 @@ bool Compiler<Emitter>::VisitUnaryOperator(const UnaryOperator *E) {
     if (!this->visitBool(SubExpr))
       return false;
 
-    if (!this->emitInvBool(E))
+    if (!this->emitInv(E))
       return false;
 
     if (PrimType ET = classifyPrim(E->getType()); ET != PT_Bool)
@@ -5231,7 +5231,7 @@ bool Compiler<Emitter>::VisitComplexUnaryOperator(const UnaryOperator *E) {
       return false;
     if (!this->emitComplexBoolCast(SubExpr))
       return false;
-    if (!this->emitInvBool(E))
+    if (!this->emitInv(E))
       return false;
     if (PrimType ET = classifyPrim(E->getType()); ET != PT_Bool)
       return this->emitCast(PT_Bool, ET, E);
diff --git a/clang/lib/AST/ByteCode/Interp.h b/clang/lib/AST/ByteCode/Interp.h
index 6cb7a42482ab25b..5aa3d24ebc75827 100644
--- a/clang/lib/AST/ByteCode/Interp.h
+++ b/clang/lib/AST/ByteCode/Interp.h
@@ -656,15 +656,9 @@ inline bool Divf(InterpState &S, CodePtr OpPC, llvm::RoundingMode RM) {
 // Inv
 //===----------------------------------------------------------------------===//
 
-template <PrimType Name, class T = typename PrimConv<Name>::T>
-bool Inv(InterpState &S, CodePtr OpPC) {
-  using BoolT = PrimConv<PT_Bool>::T;
-  const T &Val = S.Stk.pop<T>();
-  const unsigned Bits = Val.bitWidth();
-  Boolean R;
-  Boolean::inv(BoolT::from(Val, Bits), &R);
-
-  S.Stk.push<BoolT>(R);
+inline bool Inv(InterpState &S, CodePtr OpPC) {
+  const auto &Val = S.Stk.pop<Boolean>();
+  S.Stk.push<Boolean>(!Val);
   return true;
 }
 
diff --git a/clang/lib/AST/ByteCode/Opcodes.td b/clang/lib/AST/ByteCode/Opcodes.td
index 3478eb174e518ed..61319e1633d9ad4 100644
--- a/clang/lib/AST/ByteCode/Opcodes.td
+++ b/clang/lib/AST/ByteCode/Opcodes.td
@@ -103,10 +103,6 @@ def PtrTypeClass : TypeClass {
   let Types = [Ptr, FnPtr, MemberPtr];
 }
 
-def BoolTypeClass : TypeClass {
-  let Types = [Bool];
-}
-
 def NonPtrTypeClass : TypeClass {
   let Types = !listconcat(IntegerTypeClass.Types, [Bool], [Float]);
 }
@@ -574,11 +570,8 @@ def Shr : Opcode {
 // Unary operators.
 //===----------------------------------------------------------------------===//
 
-// [Real] -> [Real]
-def Inv: Opcode {
-  let Types = [BoolTypeClass];
-  let HasGroup = 1;
-}
+// [Bool] -> [Bool]
+def Inv: Opcode;
 
 // Increment and decrement.
 def Inc: AluOpcode;

From e05307f6633ca405834a4fd24d858ffb676c9170 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek@amd.com>
Date: Sun, 18 Aug 2024 10:25:00 -0500
Subject: [PATCH 299/441] [clang][OpenMP] Avoid multiple calls to
 getCurrentDirective in DSAChecker, NFC

---
 clang/lib/Sema/SemaOpenMP.cpp | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index 87d81dfaad601b4..ac51bee29bc5b01 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -3717,6 +3717,7 @@ namespace {
 class DSAAttrChecker final : public StmtVisitor<DSAAttrChecker, void> {
   DSAStackTy *Stack;
   Sema &SemaRef;
+  OpenMPDirectiveKind DKind = OMPD_unknown;
   bool ErrorFound = false;
   bool TryCaptureCXXThisMembers = false;
   CapturedStmt *CS = nullptr;
@@ -3748,7 +3749,7 @@ class DSAAttrChecker final : public StmtVisitor<DSAAttrChecker, void> {
     // Try to capture inner this->member references to generate correct mappings
     // and diagnostics.
     if (TryCaptureCXXThisMembers ||
-        (isOpenMPTargetExecutionDirective(Stack->getCurrentDirective()) &&
+        (isOpenMPTargetExecutionDirective(DKind) &&
          llvm::any_of(S->getInnermostCapturedStmt()->captures(),
                       [](const CapturedStmt::Capture &C) {
                         return C.capturesThis();
@@ -3818,7 +3819,6 @@ class DSAAttrChecker final : public StmtVisitor<DSAAttrChecker, void> {
         return;
 
       SourceLocation ELoc = E->getExprLoc();
-      OpenMPDirectiveKind DKind = Stack->getCurrentDirective();
       // The default(none) clause requires that each variable that is referenced
       // in the construct, and does not have a predetermined data-sharing
       // attribute, must have its data-sharing attribute explicitly determined
@@ -3983,7 +3983,6 @@ class DSAAttrChecker final : public StmtVisitor<DSAAttrChecker, void> {
         E->containsUnexpandedParameterPack() || E->isInstantiationDependent())
       return;
     auto *FD = dyn_cast<FieldDecl>(E->getMemberDecl());
-    OpenMPDirectiveKind DKind = Stack->getCurrentDirective();
     if (auto *TE = dyn_cast<CXXThisExpr>(E->getBase()->IgnoreParenCasts())) {
       if (!FD)
         return;
@@ -4065,8 +4064,7 @@ class DSAAttrChecker final : public StmtVisitor<DSAAttrChecker, void> {
     if (isOpenMPTargetExecutionDirective(DKind)) {
       OMPClauseMappableExprCommon::MappableExprComponentList CurComponents;
       if (!checkMapClauseExpressionBase(SemaRef, E, CurComponents, OMPC_map,
-                                        Stack->getCurrentDirective(),
-                                        /*NoDiagnose=*/true))
+                                        DKind, /*NoDiagnose=*/true))
         return;
       const auto *VD = cast<ValueDecl>(
           CurComponents.back().getAssociatedDeclaration()->getCanonicalDecl());
@@ -4119,8 +4117,7 @@ class DSAAttrChecker final : public StmtVisitor<DSAAttrChecker, void> {
       // Skip analysis of arguments of implicitly defined map clause for target
       // directives.
       if (C && !((isa<OMPFirstprivateClause>(C) || isa<OMPMapClause>(C)) &&
-                 C->isImplicit() &&
-                 !isOpenMPTaskingDirective(Stack->getCurrentDirective()))) {
+                 C->isImplicit() && !isOpenMPTaskingDirective(DKind))) {
         for (Stmt *CC : C->children()) {
           if (CC)
             Visit(CC);
@@ -4169,7 +4166,7 @@ class DSAAttrChecker final : public StmtVisitor<DSAAttrChecker, void> {
       VarDecl *VD = Cap.getCapturedVar();
       // Do not try to map the variable if it or its sub-component was mapped
       // already.
-      if (isOpenMPTargetExecutionDirective(Stack->getCurrentDirective()) &&
+      if (isOpenMPTargetExecutionDirective(DKind) &&
           Stack->checkMappableExprComponentListsForDecl(
               VD, /*CurrentRegionOnly=*/true,
               [](OMPClauseMappableExprCommon::MappableExprComponentListRef,
@@ -4200,8 +4197,9 @@ class DSAAttrChecker final : public StmtVisitor<DSAAttrChecker, void> {
 
   DSAAttrChecker(DSAStackTy *S, Sema &SemaRef, CapturedStmt *CS)
       : Stack(S), SemaRef(SemaRef), ErrorFound(false), CS(CS) {
+    DKind = S->getCurrentDirective();
     // Process declare target link variables for the target directives.
-    if (isOpenMPTargetExecutionDirective(S->getCurrentDirective())) {
+    if (isOpenMPTargetExecutionDirective(DKind)) {
       for (DeclRefExpr *E : Stack->getLinkGlobals())
         Visit(E);
     }

From dd40632b52d8da2146a12254afc900315ac3c2e4 Mon Sep 17 00:00:00 2001
From: c8ef <c8ef@outlook.com>
Date: Sun, 18 Aug 2024 23:32:44 +0800
Subject: [PATCH 300/441] [clang] fix divide by zero in ComplexExprEvaluator
 (#104666)

fix: #55390.

---------

Co-authored-by: Sergei Barannikov <barannikov88@gmail.com>
---
 clang/docs/ReleaseNotes.rst                      | 2 ++
 clang/lib/AST/ExprConstant.cpp                   | 6 +++---
 clang/test/SemaCXX/constant-expression-cxx11.cpp | 3 +++
 3 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 11301e4c7dc4a22..a4257ea1f48c11b 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -274,6 +274,8 @@ Bug Fixes to C++ Support
 Bug Fixes to AST Handling
 ^^^^^^^^^^^^^^^^^^^^^^^^^
 
+- Fixed a crash that occurred when dividing by zero in complex integer division. (#GH55390).
+
 Miscellaneous Bug Fixes
 ^^^^^^^^^^^^^^^^^^^^^^^
 
diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp
index 31b65ff4bcc135c..5540f58b526705a 100644
--- a/clang/lib/AST/ExprConstant.cpp
+++ b/clang/lib/AST/ExprConstant.cpp
@@ -15615,12 +15615,12 @@ bool ComplexExprEvaluator::VisitBinaryOperator(const BinaryOperator *E) {
         HandleComplexComplexDiv(A, B, C, D, ResR, ResI);
       }
     } else {
-      if (RHS.getComplexIntReal() == 0 && RHS.getComplexIntImag() == 0)
-        return Error(E, diag::note_expr_divide_by_zero);
-
       ComplexValue LHS = Result;
       APSInt Den = RHS.getComplexIntReal() * RHS.getComplexIntReal() +
         RHS.getComplexIntImag() * RHS.getComplexIntImag();
+      if (Den.isZero())
+        return Error(E, diag::note_expr_divide_by_zero);
+
       Result.getComplexIntReal() =
         (LHS.getComplexIntReal() * RHS.getComplexIntReal() +
          LHS.getComplexIntImag() * RHS.getComplexIntImag()) / Den;
diff --git a/clang/test/SemaCXX/constant-expression-cxx11.cpp b/clang/test/SemaCXX/constant-expression-cxx11.cpp
index d888887bd8c6f39..44ef540f41fa8c7 100644
--- a/clang/test/SemaCXX/constant-expression-cxx11.cpp
+++ b/clang/test/SemaCXX/constant-expression-cxx11.cpp
@@ -1268,6 +1268,9 @@ constexpr complex_wrap makeComplexWrap(int re, int im) {
 static_assert(makeComplexWrap(1,0) == complex(1), "");
 static_assert(makeComplexWrap(1,0) != complex(0, 1), "");
 
+constexpr auto GH55390 = 1 / 65536j; // expected-note {{division by zero}} \
+                                     // expected-error {{constexpr variable 'GH55390' must be initialized by a constant expression}} \
+                                     // expected-warning {{imaginary constants are a GNU extension}}
 }
 
 namespace PR11595 {

From bbcb625798514f1cd6ef04818381d38ea26b23e5 Mon Sep 17 00:00:00 2001
From: Mike Crowe <mac@mcrowe.com>
Date: Sun, 18 Aug 2024 16:46:19 +0100
Subject: [PATCH 301/441] [clang-tidy] Support member functions with
 modernize-use-std-print/format (#104675)

These checks can be made to work on member functions quite easily and
it's not unknown to have at least printf-style functions as members.
Let's remove the restriction.
---
 .../modernize/UseStdFormatCheck.cpp           |  6 +-
 .../clang-tidy/modernize/UseStdPrintCheck.cpp |  8 +--
 clang-tools-extra/docs/ReleaseNotes.rst       |  8 +++
 .../checks/modernize/use-std-format.rst       |  5 +-
 .../checks/modernize/use-std-print.rst        | 15 ++--
 .../modernize/use-std-format-member.cpp       | 44 ++++++++++++
 .../modernize/use-std-print-member.cpp        | 70 +++++++++++++++++++
 7 files changed, 140 insertions(+), 16 deletions(-)
 create mode 100644 clang-tools-extra/test/clang-tidy/checkers/modernize/use-std-format-member.cpp
 create mode 100644 clang-tools-extra/test/clang-tidy/checkers/modernize/use-std-print-member.cpp

diff --git a/clang-tools-extra/clang-tidy/modernize/UseStdFormatCheck.cpp b/clang-tools-extra/clang-tidy/modernize/UseStdFormatCheck.cpp
index 6cef21f1318a2a9..cdb34aef1b0e61e 100644
--- a/clang-tools-extra/clang-tidy/modernize/UseStdFormatCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/UseStdFormatCheck.cpp
@@ -50,8 +50,7 @@ void UseStdFormatCheck::registerMatchers(MatchFinder *Finder) {
   Finder->addMatcher(
       callExpr(argumentCountAtLeast(1),
                hasArgument(0, stringLiteral(isOrdinary())),
-               callee(functionDecl(unless(cxxMethodDecl()),
-                                   matchers::matchesAnyListedName(
+               callee(functionDecl(matchers::matchesAnyListedName(
                                        StrFormatLikeFunctions))
                           .bind("func_decl")))
           .bind("strformat"),
@@ -93,7 +92,8 @@ void UseStdFormatCheck::check(const MatchFinder::MatchResult &Result) {
       diag(StrFormatCall->getBeginLoc(), "use '%0' instead of %1")
       << ReplacementFormatFunction << OldFunction->getIdentifier();
   Diag << FixItHint::CreateReplacement(
-      CharSourceRange::getTokenRange(StrFormatCall->getSourceRange()),
+      CharSourceRange::getTokenRange(StrFormatCall->getExprLoc(),
+                                     StrFormatCall->getEndLoc()),
       ReplacementFormatFunction);
   Converter.applyFixes(Diag, *Result.SourceManager);
 
diff --git a/clang-tools-extra/clang-tidy/modernize/UseStdPrintCheck.cpp b/clang-tools-extra/clang-tidy/modernize/UseStdPrintCheck.cpp
index ff990feadc0c1d9..16f2f4b3e7d1af6 100644
--- a/clang-tools-extra/clang-tidy/modernize/UseStdPrintCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/UseStdPrintCheck.cpp
@@ -100,8 +100,7 @@ void UseStdPrintCheck::registerMatchers(MatchFinder *Finder) {
         unusedReturnValue(
             callExpr(argumentCountAtLeast(1),
                      hasArgument(0, stringLiteral(isOrdinary())),
-                     callee(functionDecl(unless(cxxMethodDecl()),
-                                         matchers::matchesAnyListedName(
+                     callee(functionDecl(matchers::matchesAnyListedName(
                                              PrintfLikeFunctions))
                                 .bind("func_decl")))
                 .bind("printf")),
@@ -112,8 +111,7 @@ void UseStdPrintCheck::registerMatchers(MatchFinder *Finder) {
         unusedReturnValue(
             callExpr(argumentCountAtLeast(2),
                      hasArgument(1, stringLiteral(isOrdinary())),
-                     callee(functionDecl(unless(cxxMethodDecl()),
-                                         matchers::matchesAnyListedName(
+                     callee(functionDecl(matchers::matchesAnyListedName(
                                              FprintfLikeFunctions))
                                 .bind("func_decl")))
                 .bind("fprintf")),
@@ -152,7 +150,7 @@ void UseStdPrintCheck::check(const MatchFinder::MatchResult &Result) {
       << ReplacementFunction << OldFunction->getIdentifier();
 
   Diag << FixItHint::CreateReplacement(
-      CharSourceRange::getTokenRange(PrintfCall->getBeginLoc(),
+      CharSourceRange::getTokenRange(PrintfCall->getExprLoc(),
                                      PrintfCall->getEndLoc()),
       ReplacementFunction);
   Converter.applyFixes(Diag, *Result.SourceManager);
diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst
index 0f6b34cf4005215..1b025e8f90f7ba8 100644
--- a/clang-tools-extra/docs/ReleaseNotes.rst
+++ b/clang-tools-extra/docs/ReleaseNotes.rst
@@ -104,6 +104,14 @@ New check aliases
 Changes in existing checks
 ^^^^^^^^^^^^^^^^^^^^^^^^^^
 
+- Improved :doc:`modernize-use-std-format
+  <clang-tidy/checks/modernize/use-std-format>` check to support replacing
+  member function calls too.
+
+- Improved :doc:`modernize-use-std-print
+  <clang-tidy/checks/modernize/use-std-print>` check to support replacing
+  member function calls too.
+
 - Improved :doc:`readability-redundant-smartptr-get
   <clang-tidy/checks/readability/redundant-smartptr-get>` check to
   remove `->`, when redundant `get()` is removed.
diff --git a/clang-tools-extra/docs/clang-tidy/checks/modernize/use-std-format.rst b/clang-tools-extra/docs/clang-tidy/checks/modernize/use-std-format.rst
index a1599f0fc58fe68..1ec753ef090de19 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/modernize/use-std-format.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/modernize/use-std-format.rst
@@ -64,8 +64,9 @@ Options
    A semicolon-separated list of (fully qualified) function names to
    replace, with the requirement that the first parameter contains the
    printf-style format string and the arguments to be formatted follow
-   immediately afterwards. The default value for this option is
-   `absl::StrFormat`.
+   immediately afterwards. Qualified member function names are supported,
+   but the replacement function name must be unqualified. The default value
+   for this option is `absl::StrFormat`.
 
 .. option:: ReplacementFormatFunction
 
diff --git a/clang-tools-extra/docs/clang-tidy/checks/modernize/use-std-print.rst b/clang-tools-extra/docs/clang-tidy/checks/modernize/use-std-print.rst
index 79648a1104bca23..59bb722e2c24fc1 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/modernize/use-std-print.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/modernize/use-std-print.rst
@@ -121,9 +121,10 @@ Options
    A semicolon-separated list of (fully qualified) function names to
    replace, with the requirement that the first parameter contains the
    printf-style format string and the arguments to be formatted follow
-   immediately afterwards. If neither this option nor
-   `FprintfLikeFunctions` are set then the default value for this option
-   is `printf; absl::PrintF`, otherwise it is empty.
+   immediately afterwards. Qualified member function names are supported,
+   but the replacement function name must be unqualified. If neither this
+   option nor `FprintfLikeFunctions` are set then the default value for
+   this option is `printf; absl::PrintF`, otherwise it is empty.
 
 
 .. option:: FprintfLikeFunctions
@@ -131,9 +132,11 @@ Options
    A semicolon-separated list of (fully qualified) function names to
    replace, with the requirement that the first parameter is retained, the
    second parameter contains the printf-style format string and the
-   arguments to be formatted follow immediately afterwards. If neither this
-   option nor `PrintfLikeFunctions` are set then the default value for
-   this option is `fprintf; absl::FPrintF`, otherwise it is empty.
+   arguments to be formatted follow immediately afterwards. Qualified
+   member function names are supported, but the replacement function name
+   must be unqualified. If neither this option nor `PrintfLikeFunctions`
+   are set then the default value for this option is `fprintf;
+   absl::FPrintF`, otherwise it is empty.
 
 .. option:: ReplacementPrintFunction
 
diff --git a/clang-tools-extra/test/clang-tidy/checkers/modernize/use-std-format-member.cpp b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-std-format-member.cpp
new file mode 100644
index 000000000000000..b4e1720e0a106d0
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-std-format-member.cpp
@@ -0,0 +1,44 @@
+// RUN: %check_clang_tidy %s modernize-use-std-format %t -- \
+// RUN:   -config="{CheckOptions: \
+// RUN:             { \
+// RUN:               modernize-use-std-format.StrFormatLikeFunctions: 'MyClass::StrFormat', \
+// RUN:               modernize-use-std-format.ReplacementFormatFunction: 'format', \
+// RUN:             } \
+// RUN:            }" \
+// RUN:   -- -isystem %clang_tidy_headers
+
+#include <cstdio>
+#include <string.h>
+#include <string>
+
+struct MyClass
+{
+  template <typename S, typename... Args>
+  std::string StrFormat(const S &format, const Args&... args);
+};
+
+std::string StrFormat_simple(MyClass &myclass, MyClass *pmyclass) {
+  std::string s;
+
+  s += myclass.StrFormat("MyClass::StrFormat dot %d", 42);
+  // CHECK-MESSAGES: [[@LINE-1]]:8: warning: use 'format' instead of 'StrFormat' [modernize-use-std-format]
+  // CHECK-FIXES: s += myclass.format("MyClass::StrFormat dot {}", 42);
+
+  s += pmyclass->StrFormat("MyClass::StrFormat pointer %d", 43);
+  // CHECK-MESSAGES: [[@LINE-1]]:8: warning: use 'format' instead of 'StrFormat' [modernize-use-std-format]
+  // CHECK-FIXES: s += pmyclass->format("MyClass::StrFormat pointer {}", 43);
+
+  s += (*pmyclass).StrFormat("MyClass::StrFormat deref pointer %d", 44);
+  // CHECK-MESSAGES: [[@LINE-1]]:8: warning: use 'format' instead of 'StrFormat' [modernize-use-std-format]
+  // CHECK-FIXES: s += (*pmyclass).format("MyClass::StrFormat deref pointer {}", 44);
+
+  return s;
+}
+
+struct MyDerivedClass : public MyClass {};
+
+std::string StrFormat_derived(MyDerivedClass &derived) {
+  return derived.StrFormat("MyDerivedClass::StrFormat dot %d", 42);
+  // CHECK-MESSAGES: [[@LINE-1]]:10: warning: use 'format' instead of 'StrFormat' [modernize-use-std-format]
+  // CHECK-FIXES: return derived.format("MyDerivedClass::StrFormat dot {}", 42);
+}
diff --git a/clang-tools-extra/test/clang-tidy/checkers/modernize/use-std-print-member.cpp b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-std-print-member.cpp
new file mode 100644
index 000000000000000..1d1ac4b369bbaa7
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-std-print-member.cpp
@@ -0,0 +1,70 @@
+// RUN: %check_clang_tidy %s modernize-use-std-print %t -- \
+// RUN:   -config="{CheckOptions: \
+// RUN:             { \
+// RUN:               modernize-use-std-print.PrintfLikeFunctions: 'MyClass::printf', \
+// RUN:               modernize-use-std-print.FprintfLikeFunctions: 'MyClass::fprintf', \
+// RUN:               modernize-use-std-print.ReplacementPrintFunction: 'print', \
+// RUN:               modernize-use-std-print.ReplacementPrintlnFunction: 'println', \
+// RUN:             } \
+// RUN:            }" \
+// RUN:   -- -isystem %clang_tidy_headers
+
+#include <cstdio>
+#include <string.h>
+
+struct MyStruct {};
+
+struct MyClass
+{
+  template <typename... Args>
+  void printf(const char *fmt, Args &&...);
+  template <typename... Args>
+  int fprintf(MyStruct *param1, const char *fmt, Args &&...);
+};
+
+void printf_simple(MyClass &myclass, MyClass *pmyclass) {
+  myclass.printf("printf dot %d", 42);
+  // CHECK-MESSAGES: [[@LINE-1]]:3: warning: use 'print' instead of 'printf' [modernize-use-std-print]
+  // CHECK-FIXES: myclass.print("printf dot {}", 42);
+
+  pmyclass->printf("printf pointer %d", 43);
+  // CHECK-MESSAGES: [[@LINE-1]]:3: warning: use 'print' instead of 'printf' [modernize-use-std-print]
+  // CHECK-FIXES: pmyclass->print("printf pointer {}", 43);
+
+  (*pmyclass).printf("printf deref pointer %d", 44);
+  // CHECK-MESSAGES: [[@LINE-1]]:3: warning: use 'print' instead of 'printf' [modernize-use-std-print]
+  // CHECK-FIXES: (*pmyclass).print("printf deref pointer {}", 44);
+}
+
+void printf_newline(MyClass &myclass, MyClass *pmyclass) {
+  myclass.printf("printf dot newline %c\n", 'A');
+  // CHECK-MESSAGES: [[@LINE-1]]:3: warning: use 'println' instead of 'printf' [modernize-use-std-print]
+  // CHECK-FIXES: myclass.println("printf dot newline {}", 'A');
+
+  pmyclass->printf("printf pointer newline %c\n", 'B');
+  // CHECK-MESSAGES: [[@LINE-1]]:3: warning: use 'println' instead of 'printf' [modernize-use-std-print]
+  // CHECK-FIXES: pmyclass->println("printf pointer newline {}", 'B');
+}
+
+void fprintf_simple(MyStruct *mystruct, MyClass &myclass, MyClass *pmyclass) {
+  myclass.fprintf(mystruct, "fprintf dot %d", 142);
+  // CHECK-MESSAGES: [[@LINE-1]]:3: warning: use 'print' instead of 'fprintf' [modernize-use-std-print]
+  // CHECK-FIXES: myclass.print(mystruct, "fprintf dot {}", 142);
+
+  pmyclass->fprintf(mystruct, "fprintf pointer %d", 143);
+  // CHECK-MESSAGES: [[@LINE-1]]:3: warning: use 'print' instead of 'fprintf' [modernize-use-std-print]
+  // CHECK-FIXES: pmyclass->print(mystruct, "fprintf pointer {}", 143);
+
+  (*pmyclass).fprintf(mystruct, "fprintf deref pointer %d", 144);
+  // CHECK-MESSAGES: [[@LINE-1]]:3: warning: use 'print' instead of 'fprintf' [modernize-use-std-print]
+  // CHECK-FIXES: (*pmyclass).print(mystruct, "fprintf deref pointer {}", 144);
+}
+
+struct MyDerivedClass : public MyClass {};
+
+void printf_derived(MyDerivedClass &derived)
+{
+  derived.printf("printf on derived class %d", 42);
+  // CHECK-MESSAGES: [[@LINE-1]]:3: warning: use 'print' instead of 'printf' [modernize-use-std-print]
+  // CHECK-FIXES: derived.print("printf on derived class {}", 42);
+}

From de5ea2d122c31e1551654ff506c33df299f351b8 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek@amd.com>
Date: Sun, 18 Aug 2024 10:42:28 -0500
Subject: [PATCH 302/441] [clang][OpenMP] Change /* ParamName */ to
 /*ParamName=*/, NFC

Change
  foo(/* Index */ 0);
to
  foo(/*Index=*/0);

There was a mix of these two formats in the source. Clang-format treats
the latter one a bit better, so use that one consistently.
---
 clang/lib/Sema/SemaOpenMP.cpp | 87 +++++++++++++++++------------------
 1 file changed, 43 insertions(+), 44 deletions(-)

diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index ac51bee29bc5b01..5950bc272fae3c8 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -4021,7 +4021,7 @@ class DSAAttrChecker final : public StmtVisitor<DSAAttrChecker, void> {
         OpenMPDefaultmapClauseKind ClauseKind =
             getVariableCategoryFromDecl(SemaRef.getLangOpts(), FD);
         OpenMPMapClauseKind Kind = getMapClauseKindFromModifier(
-            Modifier, /*IsAggregateOrDeclareTarget*/ true);
+            Modifier, /*IsAggregateOrDeclareTarget=*/true);
         ImplicitMap[ClauseKind][Kind].emplace_back(E);
         return;
       }
@@ -4586,7 +4586,7 @@ static bool checkOrderedOrderSpecified(Sema &S,
 StmtResult SemaOpenMP::ActOnOpenMPRegionEnd(StmtResult S,
                                             ArrayRef<OMPClause *> Clauses) {
   handleDeclareVariantConstructTrait(DSAStack, DSAStack->getCurrentDirective(),
-                                     /* ScopeEntry */ false);
+                                     /*ScopeEntry=*/false);
   if (!isOpenMPCapturingDirective(DSAStack->getCurrentDirective()))
     return S;
 
@@ -7057,8 +7057,8 @@ void SemaOpenMP::ActOnStartOfFunctionDefinitionInOpenMPDeclareVariantScope(
     QualType UDeclTy = UDecl->getType();
     if (!UDeclTy->isDependentType()) {
       QualType NewType = getASTContext().mergeFunctionTypes(
-          FType, UDeclTy, /* OfBlockPointer */ false,
-          /* Unqualified */ false, /* AllowCXX */ true);
+          FType, UDeclTy, /*OfBlockPointer=*/false,
+          /*Unqualified=*/false, /*AllowCXX=*/true);
       if (NewType.isNull())
         continue;
     }
@@ -7104,9 +7104,8 @@ void SemaOpenMP::ActOnFinishedFunctionDefinitionInOpenMPDeclareVariantScope(
     FD = cast<FunctionDecl>(D);
   auto *VariantFuncRef = DeclRefExpr::Create(
       getASTContext(), NestedNameSpecifierLoc(), SourceLocation(), FD,
-      /* RefersToEnclosingVariableOrCapture */ false,
-      /* NameLoc */ FD->getLocation(), FD->getType(),
-      ExprValueKind::VK_PRValue);
+      /*RefersToEnclosingVariableOrCapture=*/false,
+      /*NameLoc=*/FD->getLocation(), FD->getType(), ExprValueKind::VK_PRValue);
 
   OMPDeclareVariantScope &DVScope = OMPDeclareVariantScopes.back();
   auto *OMPDeclareVariantA = OMPDeclareVariantAttr::CreateImplicit(
@@ -7168,7 +7167,7 @@ ExprResult SemaOpenMP::ActOnOpenMPCall(ExprResult Call, Scope *Scope,
       OMPTraitInfo &TI = A->getTraitInfo();
       TI.getAsVariantMatchInfo(Context, VMI);
       if (!isVariantApplicableInContext(VMI, OMPCtx,
-                                        /* DeviceSetOnly */ false))
+                                        /*DeviceSetOnly=*/false))
         continue;
 
       VMIs.push_back(VMI);
@@ -7204,7 +7203,7 @@ ExprResult SemaOpenMP::ActOnOpenMPCall(ExprResult Call, Scope *Scope,
         auto *MemberCall = dyn_cast<CXXMemberCallExpr>(CE);
         BestExpr = MemberExpr::CreateImplicit(
             Context, MemberCall->getImplicitObjectArgument(),
-            /* IsArrow */ false, SpecializedMethod, Context.BoundMemberTy,
+            /*IsArrow=*/false, SpecializedMethod, Context.BoundMemberTy,
             MemberCall->getValueKind(), MemberCall->getObjectKind());
       }
       NewCall = SemaRef.BuildCallExpr(Scope, BestExpr, LParenLoc, ArgExprs,
@@ -7214,8 +7213,8 @@ ExprResult SemaOpenMP::ActOnOpenMPCall(ExprResult Call, Scope *Scope,
           FunctionDecl *NewCalleeFnDecl = NCE->getDirectCallee();
           QualType NewType = getASTContext().mergeFunctionTypes(
               CalleeFnType, NewCalleeFnDecl->getType(),
-              /* OfBlockPointer */ false,
-              /* Unqualified */ false, /* AllowCXX */ true);
+              /*OfBlockPointer=*/false,
+              /*Unqualified=*/false, /*AllowCXX=*/true);
           if (!NewType.isNull())
             break;
           // Don't use the call if the function type was not compatible.
@@ -9413,7 +9412,7 @@ static ExprResult widenIterationCount(unsigned Bits, Expr *E, Sema &SemaRef) {
   if (HasBits >= Bits)
     return ExprResult(E);
   // OK to convert to signed, because new type has more bits than old.
-  QualType NewType = C.getIntTypeForBitwidth(Bits, /* Signed */ true);
+  QualType NewType = C.getIntTypeForBitwidth(Bits, /*Signed=*/true);
   return SemaRef.PerformImplicitConversion(E, NewType, Sema::AA_Converting,
                                            true);
 }
@@ -9598,7 +9597,7 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
           }))
     return 0;
 
-  Built.clear(/* size */ NestedLoopCount);
+  Built.clear(/*size=*/NestedLoopCount);
 
   if (SemaRef.CurContext->isDependentContext())
     return NestedLoopCount;
@@ -9752,13 +9751,13 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
     LB = buildDeclRefExpr(SemaRef, LBDecl, VType, InitLoc);
     SemaRef.AddInitializerToDecl(LBDecl,
                                  SemaRef.ActOnIntegerConstant(InitLoc, 0).get(),
-                                 /*DirectInit*/ false);
+                                 /*DirectInit=*/false);
 
     // Upper bound variable, initialized with last iteration number.
     VarDecl *UBDecl = buildVarDecl(SemaRef, InitLoc, VType, ".omp.ub");
     UB = buildDeclRefExpr(SemaRef, UBDecl, VType, InitLoc);
     SemaRef.AddInitializerToDecl(UBDecl, LastIteration.get(),
-                                 /*DirectInit*/ false);
+                                 /*DirectInit=*/false);
 
     // A 32-bit variable-flag where runtime returns 1 for the last iteration.
     // This will be used to implement clause 'lastprivate'.
@@ -9767,7 +9766,7 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
     IL = buildDeclRefExpr(SemaRef, ILDecl, Int32Ty, InitLoc);
     SemaRef.AddInitializerToDecl(ILDecl,
                                  SemaRef.ActOnIntegerConstant(InitLoc, 0).get(),
-                                 /*DirectInit*/ false);
+                                 /*DirectInit=*/false);
 
     // Stride variable returned by runtime (we initialize it to 1 by default).
     VarDecl *STDecl =
@@ -9775,7 +9774,7 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
     ST = buildDeclRefExpr(SemaRef, STDecl, StrideVType, InitLoc);
     SemaRef.AddInitializerToDecl(STDecl,
                                  SemaRef.ActOnIntegerConstant(InitLoc, 1).get(),
-                                 /*DirectInit*/ false);
+                                 /*DirectInit=*/false);
 
     // Build expression: UB = min(UB, LastIteration)
     // It is necessary for CodeGen of directives with static scheduling.
@@ -9786,7 +9785,7 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
         LastIteration.get(), UB.get());
     EUB = SemaRef.BuildBinOp(CurScope, InitLoc, BO_Assign, UB.get(),
                              CondOp.get());
-    EUB = SemaRef.ActOnFinishFullExpr(EUB.get(), /*DiscardedValue*/ false);
+    EUB = SemaRef.ActOnFinishFullExpr(EUB.get(), /*DiscardedValue=*/false);
 
     // If we have a combined directive that combines 'distribute', 'for' or
     // 'simd' we need to be able to access the bounds of the schedule of the
@@ -9799,14 +9798,14 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
       CombLB = buildDeclRefExpr(SemaRef, CombLBDecl, VType, InitLoc);
       SemaRef.AddInitializerToDecl(
           CombLBDecl, SemaRef.ActOnIntegerConstant(InitLoc, 0).get(),
-          /*DirectInit*/ false);
+          /*DirectInit=*/false);
 
       // Upper bound variable, initialized with last iteration number.
       VarDecl *CombUBDecl =
           buildVarDecl(SemaRef, InitLoc, VType, ".omp.comb.ub");
       CombUB = buildDeclRefExpr(SemaRef, CombUBDecl, VType, InitLoc);
       SemaRef.AddInitializerToDecl(CombUBDecl, LastIteration.get(),
-                                   /*DirectInit*/ false);
+                                   /*DirectInit=*/false);
 
       ExprResult CombIsUBGreater = SemaRef.BuildBinOp(
           CurScope, InitLoc, BO_GT, CombUB.get(), LastIteration.get());
@@ -9816,7 +9815,7 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
       CombEUB = SemaRef.BuildBinOp(CurScope, InitLoc, BO_Assign, CombUB.get(),
                                    CombCondOp.get());
       CombEUB =
-          SemaRef.ActOnFinishFullExpr(CombEUB.get(), /*DiscardedValue*/ false);
+          SemaRef.ActOnFinishFullExpr(CombEUB.get(), /*DiscardedValue=*/false);
 
       const CapturedDecl *CD = cast<CapturedStmt>(AStmt)->getCapturedDecl();
       // We expect to have at least 2 more parameters than the 'parallel'
@@ -9852,7 +9851,7 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
                     ? LB.get()
                     : SemaRef.ActOnIntegerConstant(SourceLocation(), 0).get();
     Init = SemaRef.BuildBinOp(CurScope, InitLoc, BO_Assign, IV.get(), RHS);
-    Init = SemaRef.ActOnFinishFullExpr(Init.get(), /*DiscardedValue*/ false);
+    Init = SemaRef.ActOnFinishFullExpr(Init.get(), /*DiscardedValue=*/false);
 
     if (isOpenMPLoopBoundSharingDirective(DKind)) {
       Expr *CombRHS =
@@ -9865,7 +9864,7 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
       CombInit =
           SemaRef.BuildBinOp(CurScope, InitLoc, BO_Assign, IV.get(), CombRHS);
       CombInit =
-          SemaRef.ActOnFinishFullExpr(CombInit.get(), /*DiscardedValue*/ false);
+          SemaRef.ActOnFinishFullExpr(CombInit.get(), /*DiscardedValue=*/false);
     }
   }
 
@@ -9885,7 +9884,7 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
                         SemaRef.ActOnIntegerConstant(SourceLocation(), 1).get())
             .get();
     BoundUB =
-        SemaRef.ActOnFinishFullExpr(BoundUB, /*DiscardedValue*/ false).get();
+        SemaRef.ActOnFinishFullExpr(BoundUB, /*DiscardedValue=*/false).get();
   }
   ExprResult Cond =
       (isOpenMPWorksharingDirective(DKind) ||
@@ -9914,7 +9913,7 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
                   SemaRef.ActOnIntegerConstant(SourceLocation(), 1).get())
               .get();
       BoundCombUB =
-          SemaRef.ActOnFinishFullExpr(BoundCombUB, /*DiscardedValue*/ false)
+          SemaRef.ActOnFinishFullExpr(BoundCombUB, /*DiscardedValue=*/false)
               .get();
     }
     CombCond =
@@ -9929,7 +9928,7 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
   if (!Inc.isUsable())
     return 0;
   Inc = SemaRef.BuildBinOp(CurScope, IncLoc, BO_Assign, IV.get(), Inc.get());
-  Inc = SemaRef.ActOnFinishFullExpr(Inc.get(), /*DiscardedValue*/ false);
+  Inc = SemaRef.ActOnFinishFullExpr(Inc.get(), /*DiscardedValue=*/false);
   if (!Inc.isUsable())
     return 0;
 
@@ -9950,7 +9949,7 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
     NextLB =
         SemaRef.BuildBinOp(CurScope, IncLoc, BO_Assign, LB.get(), NextLB.get());
     NextLB =
-        SemaRef.ActOnFinishFullExpr(NextLB.get(), /*DiscardedValue*/ false);
+        SemaRef.ActOnFinishFullExpr(NextLB.get(), /*DiscardedValue=*/false);
     if (!NextLB.isUsable())
       return 0;
     // UB + ST
@@ -9961,7 +9960,7 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
     NextUB =
         SemaRef.BuildBinOp(CurScope, IncLoc, BO_Assign, UB.get(), NextUB.get());
     NextUB =
-        SemaRef.ActOnFinishFullExpr(NextUB.get(), /*DiscardedValue*/ false);
+        SemaRef.ActOnFinishFullExpr(NextUB.get(), /*DiscardedValue=*/false);
     if (!NextUB.isUsable())
       return 0;
     if (isOpenMPLoopBoundSharingDirective(DKind)) {
@@ -9973,7 +9972,7 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
       CombNextLB = SemaRef.BuildBinOp(CurScope, IncLoc, BO_Assign, CombLB.get(),
                                       CombNextLB.get());
       CombNextLB = SemaRef.ActOnFinishFullExpr(CombNextLB.get(),
-                                               /*DiscardedValue*/ false);
+                                               /*DiscardedValue=*/false);
       if (!CombNextLB.isUsable())
         return 0;
       // UB + ST
@@ -9985,7 +9984,7 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
       CombNextUB = SemaRef.BuildBinOp(CurScope, IncLoc, BO_Assign, CombUB.get(),
                                       CombNextUB.get());
       CombNextUB = SemaRef.ActOnFinishFullExpr(CombNextUB.get(),
-                                               /*DiscardedValue*/ false);
+                                               /*DiscardedValue=*/false);
       if (!CombNextUB.isUsable())
         return 0;
     }
@@ -10008,7 +10007,7 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
     DistInc = SemaRef.BuildBinOp(CurScope, DistIncLoc, BO_Assign, IV.get(),
                                  DistInc.get());
     DistInc =
-        SemaRef.ActOnFinishFullExpr(DistInc.get(), /*DiscardedValue*/ false);
+        SemaRef.ActOnFinishFullExpr(DistInc.get(), /*DiscardedValue=*/false);
     assert(DistInc.isUsable() && "distribute inc expr was not built");
 
     // Build expression: UB = min(UB, prevUB) for #for in composite or combined
@@ -10031,7 +10030,7 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
     PrevEUB = SemaRef.BuildBinOp(CurScope, DistIncLoc, BO_Assign, UB.get(),
                                  CondOp.get());
     PrevEUB =
-        SemaRef.ActOnFinishFullExpr(PrevEUB.get(), /*DiscardedValue*/ false);
+        SemaRef.ActOnFinishFullExpr(PrevEUB.get(), /*DiscardedValue=*/false);
 
     // Build IV <= PrevUB or IV < PrevUB + 1 for unsigned IV to be used in
     // parallel for is in combination with a distribute directive with
@@ -10045,7 +10044,7 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
                   SemaRef.ActOnIntegerConstant(SourceLocation(), 1).get())
               .get();
       BoundPrevUB =
-          SemaRef.ActOnFinishFullExpr(BoundPrevUB, /*DiscardedValue*/ false)
+          SemaRef.ActOnFinishFullExpr(BoundPrevUB, /*DiscardedValue=*/false)
               .get();
     }
     ParForInDistCond =
@@ -15317,7 +15316,7 @@ SemaOpenMP::PerformOpenMPImplicitIntegerConversion(SourceLocation Loc,
   class IntConvertDiagnoser : public Sema::ICEConvertDiagnoser {
   public:
     IntConvertDiagnoser()
-        : ICEConvertDiagnoser(/*AllowScopedEnumerations*/ false, false, true) {}
+        : ICEConvertDiagnoser(/*AllowScopedEnumerations=*/false, false, true) {}
     SemaDiagnosticBuilder diagnoseNotInt(Sema &S, SourceLocation Loc,
                                          QualType T) override {
       return S.Diag(Loc, diag::err_omp_not_integral) << T;
@@ -15540,7 +15539,7 @@ static bool findOMPAllocatorHandleT(Sema &S, SourceLocation Loc,
     }
     Res = S.PerformImplicitConversion(Res.get(), AllocatorHandleEnumTy,
                                       Sema::AA_Initializing,
-                                      /* AllowExplicit */ true);
+                                      /*AllowExplicit=*/true);
     if (!Res.isUsable()) {
       ErrorFound = true;
       break;
@@ -17643,7 +17642,7 @@ OMPClause *SemaOpenMP::ActOnOpenMPLastprivateClause(
     if (AssignmentOp.isInvalid())
       continue;
     AssignmentOp = SemaRef.ActOnFinishFullExpr(AssignmentOp.get(), ELoc,
-                                               /*DiscardedValue*/ false);
+                                               /*DiscardedValue=*/false);
     if (AssignmentOp.isInvalid())
       continue;
 
@@ -18360,7 +18359,7 @@ static bool actOnOMPReductionKindClause(
     // A list item that appears in a reduction clause must not be
     // const-qualified.
     if (rejectConstNotMutableType(S, D, Type, ClauseKind, ELoc,
-                                  /*AcceptIfMutable*/ false, ASE || OASE))
+                                  /*AcceptIfMutable=*/false, ASE || OASE))
       continue;
 
     OpenMPDirectiveKind CurrDir = Stack->getCurrentDirective();
@@ -18753,7 +18752,7 @@ static bool actOnOMPReductionKindClause(
       }
       if (ReductionOp.isUsable())
         ReductionOp = S.ActOnFinishFullExpr(ReductionOp.get(),
-                                            /*DiscardedValue*/ false);
+                                            /*DiscardedValue=*/false);
       if (!ReductionOp.isUsable())
         continue;
     }
@@ -19172,7 +19171,7 @@ OMPClause *SemaOpenMP::ActOnOpenMPLinearClause(
     ExprResult CalcStep = SemaRef.BuildBinOp(
         SemaRef.getCurScope(), StepLoc, BO_Assign, SaveRef.get(), StepExpr);
     CalcStep =
-        SemaRef.ActOnFinishFullExpr(CalcStep.get(), /*DiscardedValue*/ false);
+        SemaRef.ActOnFinishFullExpr(CalcStep.get(), /*DiscardedValue=*/false);
 
     // Warn about zero linear step (it would be probably better specified as
     // making corresponding variables 'const').
@@ -19261,7 +19260,7 @@ static bool FinishOpenMPLinearClause(OMPLinearClause &Clause, DeclRefExpr *IV,
     else
       Update = *CurPrivate;
     Update = SemaRef.ActOnFinishFullExpr(Update.get(), DE->getBeginLoc(),
-                                         /*DiscardedValue*/ false);
+                                         /*DiscardedValue=*/false);
 
     // Build final: Var = PrivCopy;
     ExprResult Final;
@@ -19272,7 +19271,7 @@ static bool FinishOpenMPLinearClause(OMPLinearClause &Clause, DeclRefExpr *IV,
     else
       Final = *CurPrivate;
     Final = SemaRef.ActOnFinishFullExpr(Final.get(), DE->getBeginLoc(),
-                                        /*DiscardedValue*/ false);
+                                        /*DiscardedValue=*/false);
 
     if (!Update.isUsable() || !Final.isUsable()) {
       Updates.push_back(nullptr);
@@ -19451,7 +19450,7 @@ OMPClause *SemaOpenMP::ActOnOpenMPCopyinClause(ArrayRef<Expr *> VarList,
       continue;
     AssignmentOp =
         SemaRef.ActOnFinishFullExpr(AssignmentOp.get(), DE->getExprLoc(),
-                                    /*DiscardedValue*/ false);
+                                    /*DiscardedValue=*/false);
     if (AssignmentOp.isInvalid())
       continue;
 
@@ -19560,7 +19559,7 @@ OMPClause *SemaOpenMP::ActOnOpenMPCopyprivateClause(ArrayRef<Expr *> VarList,
     if (AssignmentOp.isInvalid())
       continue;
     AssignmentOp = SemaRef.ActOnFinishFullExpr(AssignmentOp.get(), ELoc,
-                                               /*DiscardedValue*/ false);
+                                               /*DiscardedValue=*/false);
     if (AssignmentOp.isInvalid())
       continue;
 
@@ -20946,7 +20945,7 @@ static ExprResult buildImplicitMapper(Sema &S, QualType BaseType,
                                            BaseType, MapperId, Maps, nullptr);
   Scope *Scope = S.getScopeForContext(DCT);
   if (Scope)
-    S.PushOnScopeChains(DMD, Scope, /*AddToContext*/ false);
+    S.PushOnScopeChains(DMD, Scope, /*AddToContext=*/false);
   DCT->addDecl(DMD);
   DMD->setAccess(clang::AS_none);
   auto *VD = cast<DeclRefExpr>(MapperVarRef)->getDecl();

From bcbe9d6c776bc02bfe8f911607240af0d1775d79 Mon Sep 17 00:00:00 2001
From: Mital Ashok <mital@mitalashok.co.uk>
Date: Sun, 18 Aug 2024 17:07:47 +0100
Subject: [PATCH 303/441] [Clang] Do not allow `[[clang::lifetimebound]]` on
 explicit object member functions (#96113)

Previously, `[[clang::lifetimebound]]` applied to an explicit object
member function did nothing and was silently ignored.

Now issue the error diagnostic `'lifetimebound' attribute cannot be
applied; explicit object member function has no implicit object
parameter`
---
 clang/docs/ReleaseNotes.rst                      |  3 +++
 clang/include/clang/Basic/DiagnosticSemaKinds.td |  5 +++--
 clang/lib/Sema/SemaDecl.cpp                      | 11 +++++++++--
 clang/test/SemaCXX/attr-lifetimebound.cpp        |  3 ++-
 4 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index a4257ea1f48c11b..b78c0229de4898b 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -193,6 +193,9 @@ Attribute Changes in Clang
 - The ``hybrid_patchable`` attribute is now supported on ARM64EC targets. It can be used to specify
   that a function requires an additional x86-64 thunk, which may be patched at runtime.
 
+- ``[[clang::lifetimebound]]`` is now explicitly disallowed on explicit object member functions
+  where they were previously silently ignored.
+
 Improvements to Clang's diagnostics
 -----------------------------------
 
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 8a92973236ddbdb..94be9c874afe971 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -10066,8 +10066,9 @@ def warn_null_ret : Warning<
   InGroup<NonNull>;
 
 def err_lifetimebound_no_object_param : Error<
-  "'lifetimebound' attribute cannot be applied; %select{static |non-}0member "
-  "function has no implicit object parameter">;
+  "'lifetimebound' attribute cannot be applied; "
+  "%select{non-|static |explicit object }0"
+  "member function has no implicit object parameter">;
 def err_lifetimebound_ctor_dtor : Error<
   "'lifetimebound' attribute cannot be applied to a "
   "%select{constructor|destructor}0">;
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index 9c15b5fa571a394..503e93f92571371 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -6939,9 +6939,16 @@ static void checkAttributesAfterMerging(Sema &S, NamedDecl &ND) {
       // by applying it to the function type.
       if (const auto *A = ATL.getAttrAs<LifetimeBoundAttr>()) {
         const auto *MD = dyn_cast<CXXMethodDecl>(FD);
-        if (!MD || MD->isStatic()) {
+        int NoImplicitObjectError = -1;
+        if (!MD)
+          NoImplicitObjectError = 0;
+        else if (MD->isStatic())
+          NoImplicitObjectError = 1;
+        else if (MD->isExplicitObjectMemberFunction())
+          NoImplicitObjectError = 2;
+        if (NoImplicitObjectError != -1) {
           S.Diag(A->getLocation(), diag::err_lifetimebound_no_object_param)
-              << !MD << A->getRange();
+              << NoImplicitObjectError << A->getRange();
         } else if (isa<CXXConstructorDecl>(MD) || isa<CXXDestructorDecl>(MD)) {
           S.Diag(A->getLocation(), diag::err_lifetimebound_ctor_dtor)
               << isa<CXXDestructorDecl>(MD) << A->getRange();
diff --git a/clang/test/SemaCXX/attr-lifetimebound.cpp b/clang/test/SemaCXX/attr-lifetimebound.cpp
index 7db0a4d64d2596c..f4ca840cb276f9f 100644
--- a/clang/test/SemaCXX/attr-lifetimebound.cpp
+++ b/clang/test/SemaCXX/attr-lifetimebound.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -std=c++2a -verify %s
+// RUN: %clang_cc1 -std=c++23 -verify %s
 
 namespace usage_invalid {
   // FIXME: Should we diagnose a void return type?
@@ -9,6 +9,7 @@ namespace usage_invalid {
     A() [[clang::lifetimebound]]; // expected-error {{cannot be applied to a constructor}}
     ~A() [[clang::lifetimebound]]; // expected-error {{cannot be applied to a destructor}}
     static int *static_class_member() [[clang::lifetimebound]]; // expected-error {{static member function has no implicit object parameter}}
+    int *explicit_object(this A&) [[clang::lifetimebound]]; // expected-error {{explicit object member function has no implicit object parameter}}
     int not_function [[clang::lifetimebound]]; // expected-error {{only applies to parameters and implicit object parameters}}
     int [[clang::lifetimebound]] also_not_function; // expected-error {{cannot be applied to types}}
   };

From 0d150db214e2aa13a825b563c7238e1243d61db1 Mon Sep 17 00:00:00 2001
From: Jacques Pienaar <jpienaar@google.com>
Date: Sun, 18 Aug 2024 09:46:51 -0700
Subject: [PATCH 304/441] [llvm][clang] Move RewriterBuffer to ADT. (#99770)

These classes are not specific to clang and useful for other rewriter
tools (flagged in previous review).
---
 clang/include/clang/Rewrite/Core/DeltaTree.h  |  50 --
 .../include/clang/Rewrite/Core/HTMLRewrite.h  |  11 +-
 .../include/clang/Rewrite/Core/RewriteRope.h  | 223 --------
 clang/include/clang/Rewrite/Core/Rewriter.h   |  19 +-
 clang/lib/ARCMigrate/ARCMT.cpp                |   2 +-
 clang/lib/ARCMigrate/ObjCMT.cpp               |   1 +
 clang/lib/Frontend/Rewrite/FixItRewriter.cpp  |   3 +-
 clang/lib/Frontend/Rewrite/HTMLPrint.cpp      |   2 +
 clang/lib/Frontend/Rewrite/RewriteMacros.cpp  |   4 +-
 .../Frontend/Rewrite/RewriteModernObjC.cpp    |   3 +-
 clang/lib/Frontend/Rewrite/RewriteObjC.cpp    |   1 +
 clang/lib/Rewrite/CMakeLists.txt              |   2 -
 clang/lib/Rewrite/HTMLRewrite.cpp             |   1 +
 clang/lib/Rewrite/Rewriter.cpp                | 109 +---
 .../StaticAnalyzer/Core/HTMLDiagnostics.cpp   |   2 +
 clang/lib/Tooling/Core/Replacement.cpp        |   2 +-
 clang/unittests/Rewrite/CMakeLists.txt        |   1 -
 llvm/include/llvm/ADT/DeltaTree.h             |  50 ++
 .../include/llvm/ADT}/RewriteBuffer.h         |  40 +-
 llvm/include/llvm/ADT/RewriteRope.h           | 223 ++++++++
 llvm/lib/Support/CMakeLists.txt               |   3 +
 .../lib/Support}/DeltaTree.cpp                | 265 +++++----
 llvm/lib/Support/RewriteBuffer.cpp            | 107 ++++
 .../lib/Support}/RewriteRope.cpp              | 507 +++++++++---------
 llvm/unittests/ADT/CMakeLists.txt             |   1 +
 .../unittests/ADT}/RewriteBufferTest.cpp      |   5 +-
 26 files changed, 825 insertions(+), 812 deletions(-)
 delete mode 100644 clang/include/clang/Rewrite/Core/DeltaTree.h
 delete mode 100644 clang/include/clang/Rewrite/Core/RewriteRope.h
 create mode 100644 llvm/include/llvm/ADT/DeltaTree.h
 rename {clang/include/clang/Rewrite/Core => llvm/include/llvm/ADT}/RewriteBuffer.h (82%)
 create mode 100644 llvm/include/llvm/ADT/RewriteRope.h
 rename {clang/lib/Rewrite => llvm/lib/Support}/DeltaTree.cpp (68%)
 create mode 100644 llvm/lib/Support/RewriteBuffer.cpp
 rename {clang/lib/Rewrite => llvm/lib/Support}/RewriteRope.cpp (63%)
 rename {clang/unittests/Rewrite => llvm/unittests/ADT}/RewriteBufferTest.cpp (96%)

diff --git a/clang/include/clang/Rewrite/Core/DeltaTree.h b/clang/include/clang/Rewrite/Core/DeltaTree.h
deleted file mode 100644
index e566c92aaff91ad..000000000000000
--- a/clang/include/clang/Rewrite/Core/DeltaTree.h
+++ /dev/null
@@ -1,50 +0,0 @@
-//===- DeltaTree.h - B-Tree for Rewrite Delta tracking ----------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines the DeltaTree class.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_CLANG_REWRITE_CORE_DELTATREE_H
-#define LLVM_CLANG_REWRITE_CORE_DELTATREE_H
-
-namespace clang {
-
-  /// DeltaTree - a multiway search tree (BTree) structure with some fancy
-  /// features.  B-Trees are generally more memory and cache efficient than
-  /// binary trees, because they store multiple keys/values in each node.  This
-  /// implements a key/value mapping from index to delta, and allows fast lookup
-  /// on index.  However, an added (important) bonus is that it can also
-  /// efficiently tell us the full accumulated delta for a specific file offset
-  /// as well, without traversing the whole tree.
-  class DeltaTree {
-    void *Root;    // "DeltaTreeNode *"
-
-  public:
-    DeltaTree();
-
-    // Note: Currently we only support copying when the RHS is empty.
-    DeltaTree(const DeltaTree &RHS);
-
-    DeltaTree &operator=(const DeltaTree &) = delete;
-    ~DeltaTree();
-
-    /// getDeltaAt - Return the accumulated delta at the specified file offset.
-    /// This includes all insertions or delections that occurred *before* the
-    /// specified file index.
-    int getDeltaAt(unsigned FileIndex) const;
-
-    /// AddDelta - When a change is made that shifts around the text buffer,
-    /// this method is used to record that info.  It inserts a delta of 'Delta'
-    /// into the current DeltaTree at offset FileIndex.
-    void AddDelta(unsigned FileIndex, int Delta);
-  };
-
-} // namespace clang
-
-#endif // LLVM_CLANG_REWRITE_CORE_DELTATREE_H
diff --git a/clang/include/clang/Rewrite/Core/HTMLRewrite.h b/clang/include/clang/Rewrite/Core/HTMLRewrite.h
index eecf589632746c1..9edb514d566b9c1 100644
--- a/clang/include/clang/Rewrite/Core/HTMLRewrite.h
+++ b/clang/include/clang/Rewrite/Core/HTMLRewrite.h
@@ -17,10 +17,13 @@
 #include "clang/Basic/SourceLocation.h"
 #include <string>
 
+namespace llvm {
+class RewriteBuffer;
+} // namespace llvm
+
 namespace clang {
 
 class Rewriter;
-class RewriteBuffer;
 class Preprocessor;
 
 namespace html {
@@ -53,9 +56,9 @@ namespace html {
 
   /// HighlightRange - This is the same as the above method, but takes
   /// decomposed file locations.
-  void HighlightRange(RewriteBuffer &RB, unsigned B, unsigned E,
-                      const char *BufferStart,
-                      const char *StartTag, const char *EndTag);
+  void HighlightRange(llvm::RewriteBuffer &RB, unsigned B, unsigned E,
+                      const char *BufferStart, const char *StartTag,
+                      const char *EndTag);
 
   /// EscapeText - HTMLize a specified file so that special characters are
   /// are translated so that they are not interpreted as HTML tags.
diff --git a/clang/include/clang/Rewrite/Core/RewriteRope.h b/clang/include/clang/Rewrite/Core/RewriteRope.h
deleted file mode 100644
index 73e66e111f57434..000000000000000
--- a/clang/include/clang/Rewrite/Core/RewriteRope.h
+++ /dev/null
@@ -1,223 +0,0 @@
-//===- RewriteRope.h - Rope specialized for rewriter ------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-//  This file defines the RewriteRope class, which is a powerful string class.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_CLANG_REWRITE_CORE_REWRITEROPE_H
-#define LLVM_CLANG_REWRITE_CORE_REWRITEROPE_H
-
-#include "llvm/ADT/IntrusiveRefCntPtr.h"
-#include "llvm/ADT/StringRef.h"
-#include <cassert>
-#include <cstddef>
-#include <iterator>
-#include <utility>
-
-namespace clang {
-
-  //===--------------------------------------------------------------------===//
-  // RopeRefCountString Class
-  //===--------------------------------------------------------------------===//
-
-  /// RopeRefCountString - This struct is allocated with 'new char[]' from the
-  /// heap, and represents a reference counted chunk of string data.  When its
-  /// ref count drops to zero, it is delete[]'d.  This is primarily managed
-  /// through the RopePiece class below.
-  struct RopeRefCountString {
-    unsigned RefCount;
-    char Data[1];  //  Variable sized.
-
-    void Retain() { ++RefCount; }
-
-    void Release() {
-      assert(RefCount > 0 && "Reference count is already zero.");
-      if (--RefCount == 0)
-        delete [] (char*)this;
-    }
-  };
-
-  //===--------------------------------------------------------------------===//
-  // RopePiece Class
-  //===--------------------------------------------------------------------===//
-
-  /// RopePiece - This class represents a view into a RopeRefCountString object.
-  /// This allows references to string data to be efficiently chopped up and
-  /// moved around without having to push around the string data itself.
-  ///
-  /// For example, we could have a 1M RopePiece and want to insert something
-  /// into the middle of it.  To do this, we split it into two RopePiece objects
-  /// that both refer to the same underlying RopeRefCountString (just with
-  /// different offsets) which is a nice constant time operation.
-  struct RopePiece {
-    llvm::IntrusiveRefCntPtr<RopeRefCountString> StrData;
-    unsigned StartOffs = 0;
-    unsigned EndOffs = 0;
-
-    RopePiece() = default;
-    RopePiece(llvm::IntrusiveRefCntPtr<RopeRefCountString> Str, unsigned Start,
-              unsigned End)
-        : StrData(std::move(Str)), StartOffs(Start), EndOffs(End) {}
-
-    const char &operator[](unsigned Offset) const {
-      return StrData->Data[Offset+StartOffs];
-    }
-    char &operator[](unsigned Offset) {
-      return StrData->Data[Offset+StartOffs];
-    }
-
-    unsigned size() const { return EndOffs-StartOffs; }
-  };
-
-  //===--------------------------------------------------------------------===//
-  // RopePieceBTreeIterator Class
-  //===--------------------------------------------------------------------===//
-
-  /// RopePieceBTreeIterator - This class provides read-only forward iteration
-  /// over bytes that are in a RopePieceBTree.  This first iterates over bytes
-  /// in a RopePiece, then iterates over RopePiece's in a RopePieceBTreeLeaf,
-  /// then iterates over RopePieceBTreeLeaf's in a RopePieceBTree.
-  class RopePieceBTreeIterator {
-    /// CurNode - The current B+Tree node that we are inspecting.
-    const void /*RopePieceBTreeLeaf*/ *CurNode = nullptr;
-
-    /// CurPiece - The current RopePiece in the B+Tree node that we're
-    /// inspecting.
-    const RopePiece *CurPiece = nullptr;
-
-    /// CurChar - The current byte in the RopePiece we are pointing to.
-    unsigned CurChar = 0;
-
-  public:
-    using iterator_category = std::forward_iterator_tag;
-    using value_type = const char;
-    using difference_type = std::ptrdiff_t;
-    using pointer = value_type *;
-    using reference = value_type &;
-
-    RopePieceBTreeIterator() = default;
-    RopePieceBTreeIterator(const void /*RopePieceBTreeNode*/ *N);
-
-    char operator*() const {
-      return (*CurPiece)[CurChar];
-    }
-
-    bool operator==(const RopePieceBTreeIterator &RHS) const {
-      return CurPiece == RHS.CurPiece && CurChar == RHS.CurChar;
-    }
-    bool operator!=(const RopePieceBTreeIterator &RHS) const {
-      return !operator==(RHS);
-    }
-
-    RopePieceBTreeIterator& operator++() {   // Preincrement
-      if (CurChar+1 < CurPiece->size())
-        ++CurChar;
-      else
-        MoveToNextPiece();
-      return *this;
-    }
-
-    RopePieceBTreeIterator operator++(int) { // Postincrement
-      RopePieceBTreeIterator tmp = *this; ++*this; return tmp;
-    }
-
-    llvm::StringRef piece() const {
-      return llvm::StringRef(&(*CurPiece)[0], CurPiece->size());
-    }
-
-    void MoveToNextPiece();
-  };
-
-  //===--------------------------------------------------------------------===//
-  // RopePieceBTree Class
-  //===--------------------------------------------------------------------===//
-
-  class RopePieceBTree {
-    void /*RopePieceBTreeNode*/ *Root;
-
-  public:
-    RopePieceBTree();
-    RopePieceBTree(const RopePieceBTree &RHS);
-    RopePieceBTree &operator=(const RopePieceBTree &) = delete;
-    ~RopePieceBTree();
-
-    using iterator = RopePieceBTreeIterator;
-
-    iterator begin() const { return iterator(Root); }
-    iterator end() const { return iterator(); }
-    unsigned size() const;
-    unsigned empty() const { return size() == 0; }
-
-    void clear();
-
-    void insert(unsigned Offset, const RopePiece &R);
-
-    void erase(unsigned Offset, unsigned NumBytes);
-  };
-
-  //===--------------------------------------------------------------------===//
-  // RewriteRope Class
-  //===--------------------------------------------------------------------===//
-
-/// RewriteRope - A powerful string class.  This class supports extremely
-/// efficient insertions and deletions into the middle of it, even for
-/// ridiculously long strings.
-class RewriteRope {
-  RopePieceBTree Chunks;
-
-  /// We allocate space for string data out of a buffer of size AllocChunkSize.
-  /// This keeps track of how much space is left.
-  llvm::IntrusiveRefCntPtr<RopeRefCountString> AllocBuffer;
-  enum { AllocChunkSize = 4080 };
-  unsigned AllocOffs = AllocChunkSize;
-
-public:
-  RewriteRope() = default;
-  RewriteRope(const RewriteRope &RHS) : Chunks(RHS.Chunks) {}
-
-  // The copy assignment operator is defined as deleted pending further
-  // motivation.
-  RewriteRope &operator=(const RewriteRope &) = delete;
-
-  using iterator = RopePieceBTree::iterator;
-  using const_iterator = RopePieceBTree::iterator;
-
-  iterator begin() const { return Chunks.begin(); }
-  iterator end() const { return Chunks.end(); }
-  unsigned size() const { return Chunks.size(); }
-
-  void clear() {
-    Chunks.clear();
-  }
-
-  void assign(const char *Start, const char *End) {
-    clear();
-    if (Start != End)
-      Chunks.insert(0, MakeRopeString(Start, End));
-  }
-
-  void insert(unsigned Offset, const char *Start, const char *End) {
-    assert(Offset <= size() && "Invalid position to insert!");
-    if (Start == End) return;
-    Chunks.insert(Offset, MakeRopeString(Start, End));
-  }
-
-  void erase(unsigned Offset, unsigned NumBytes) {
-    assert(Offset+NumBytes <= size() && "Invalid region to erase!");
-    if (NumBytes == 0) return;
-    Chunks.erase(Offset, NumBytes);
-  }
-
-private:
-  RopePiece MakeRopeString(const char *Start, const char *End);
-};
-
-} // namespace clang
-
-#endif // LLVM_CLANG_REWRITE_CORE_REWRITEROPE_H
diff --git a/clang/include/clang/Rewrite/Core/Rewriter.h b/clang/include/clang/Rewrite/Core/Rewriter.h
index c89015e4055820c..4e96f6fcca919ce 100644
--- a/clang/include/clang/Rewrite/Core/Rewriter.h
+++ b/clang/include/clang/Rewrite/Core/Rewriter.h
@@ -16,7 +16,7 @@
 
 #include "clang/Basic/LLVM.h"
 #include "clang/Basic/SourceLocation.h"
-#include "clang/Rewrite/Core/RewriteBuffer.h"
+#include "llvm/ADT/RewriteBuffer.h"
 #include "llvm/ADT/StringRef.h"
 #include <map>
 #include <string>
@@ -32,7 +32,7 @@ class SourceManager;
 class Rewriter {
   SourceManager *SourceMgr = nullptr;
   const LangOptions *LangOpts = nullptr;
-  std::map<FileID, RewriteBuffer> RewriteBuffers;
+  std::map<FileID, llvm::RewriteBuffer> RewriteBuffers;
 
 public:
   struct RewriteOptions {
@@ -49,7 +49,7 @@ class Rewriter {
     ///
     /// FIXME: This sometimes corrupts the file's rewrite buffer due to
     /// incorrect indexing in the implementation (see the FIXME in
-    /// clang::RewriteBuffer::RemoveText).  Moreover, it's inefficient because
+    /// llvm::RewriteBuffer::RemoveText).  Moreover, it's inefficient because
     /// it must scan the buffer from the beginning to find the start of the
     /// line.  When feasible, it's better for the caller to check for a blank
     /// line and then, if found, expand the removal range to include it.
@@ -62,8 +62,9 @@ class Rewriter {
     RewriteOptions() {}
   };
 
-  using buffer_iterator = std::map<FileID, RewriteBuffer>::iterator;
-  using const_buffer_iterator = std::map<FileID, RewriteBuffer>::const_iterator;
+  using buffer_iterator = std::map<FileID, llvm::RewriteBuffer>::iterator;
+  using const_buffer_iterator =
+      std::map<FileID, llvm::RewriteBuffer>::const_iterator;
 
   explicit Rewriter() = default;
   explicit Rewriter(SourceManager &SM, const LangOptions &LO)
@@ -191,13 +192,13 @@ class Rewriter {
   /// buffer, and allows you to write on it directly.  This is useful if you
   /// want efficient low-level access to apis for scribbling on one specific
   /// FileID's buffer.
-  RewriteBuffer &getEditBuffer(FileID FID);
+  llvm::RewriteBuffer &getEditBuffer(FileID FID);
 
   /// getRewriteBufferFor - Return the rewrite buffer for the specified FileID.
   /// If no modification has been made to it, return null.
-  const RewriteBuffer *getRewriteBufferFor(FileID FID) const {
-    std::map<FileID, RewriteBuffer>::const_iterator I =
-      RewriteBuffers.find(FID);
+  const llvm::RewriteBuffer *getRewriteBufferFor(FileID FID) const {
+    std::map<FileID, llvm::RewriteBuffer>::const_iterator I =
+        RewriteBuffers.find(FID);
     return I == RewriteBuffers.end() ? nullptr : &I->second;
   }
 
diff --git a/clang/lib/ARCMigrate/ARCMT.cpp b/clang/lib/ARCMigrate/ARCMT.cpp
index 5835559bff6b720..1a8a200f2fc1949 100644
--- a/clang/lib/ARCMigrate/ARCMT.cpp
+++ b/clang/lib/ARCMigrate/ARCMT.cpp
@@ -596,7 +596,7 @@ bool MigrationProcess::applyTransform(TransformFn trans,
   for (Rewriter::buffer_iterator
         I = rewriter.buffer_begin(), E = rewriter.buffer_end(); I != E; ++I) {
     FileID FID = I->first;
-    RewriteBuffer &buf = I->second;
+    llvm::RewriteBuffer &buf = I->second;
     OptionalFileEntryRef file =
         Ctx.getSourceManager().getFileEntryRefForID(FID);
     assert(file);
diff --git a/clang/lib/ARCMigrate/ObjCMT.cpp b/clang/lib/ARCMigrate/ObjCMT.cpp
index 4357c8e3f09a550..c1bc7c762088f29 100644
--- a/clang/lib/ARCMigrate/ObjCMT.cpp
+++ b/clang/lib/ARCMigrate/ObjCMT.cpp
@@ -36,6 +36,7 @@
 using namespace clang;
 using namespace arcmt;
 using namespace ento;
+using llvm::RewriteBuffer;
 
 namespace {
 
diff --git a/clang/lib/Frontend/Rewrite/FixItRewriter.cpp b/clang/lib/Frontend/Rewrite/FixItRewriter.cpp
index 567bac576adb40a..44dfaf20eae73f0 100644
--- a/clang/lib/Frontend/Rewrite/FixItRewriter.cpp
+++ b/clang/lib/Frontend/Rewrite/FixItRewriter.cpp
@@ -21,8 +21,8 @@
 #include "clang/Edit/Commit.h"
 #include "clang/Edit/EditsReceiver.h"
 #include "clang/Frontend/FrontendDiagnostic.h"
-#include "clang/Rewrite/Core/RewriteBuffer.h"
 #include "clang/Rewrite/Core/Rewriter.h"
+#include "llvm/ADT/RewriteBuffer.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/raw_ostream.h"
@@ -33,6 +33,7 @@
 #include <utility>
 
 using namespace clang;
+using llvm::RewriteBuffer;
 
 FixItRewriter::FixItRewriter(DiagnosticsEngine &Diags, SourceManager &SourceMgr,
                              const LangOptions &LangOpts,
diff --git a/clang/lib/Frontend/Rewrite/HTMLPrint.cpp b/clang/lib/Frontend/Rewrite/HTMLPrint.cpp
index 69baa8f591088c9..5cce193ed1c4f30 100644
--- a/clang/lib/Frontend/Rewrite/HTMLPrint.cpp
+++ b/clang/lib/Frontend/Rewrite/HTMLPrint.cpp
@@ -20,8 +20,10 @@
 #include "clang/Rewrite/Core/HTMLRewrite.h"
 #include "clang/Rewrite/Core/Rewriter.h"
 #include "clang/Rewrite/Frontend/ASTConsumers.h"
+#include "llvm/ADT/RewriteBuffer.h"
 #include "llvm/Support/raw_ostream.h"
 using namespace clang;
+using llvm::RewriteBuffer;
 
 //===----------------------------------------------------------------------===//
 // Functional HTML pretty-printing.
diff --git a/clang/lib/Frontend/Rewrite/RewriteMacros.cpp b/clang/lib/Frontend/Rewrite/RewriteMacros.cpp
index 5701b271aff103b..b533cabe97f5e9e 100644
--- a/clang/lib/Frontend/Rewrite/RewriteMacros.cpp
+++ b/clang/lib/Frontend/Rewrite/RewriteMacros.cpp
@@ -11,16 +11,18 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Rewrite/Frontend/Rewriters.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Rewrite/Core/Rewriter.h"
+#include "clang/Rewrite/Frontend/Rewriters.h"
+#include "llvm/ADT/RewriteBuffer.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/raw_ostream.h"
 #include <cstdio>
 #include <memory>
 
 using namespace clang;
+using llvm::RewriteBuffer;
 
 /// isSameToken - Return true if the two specified tokens start have the same
 /// content.
diff --git a/clang/lib/Frontend/Rewrite/RewriteModernObjC.cpp b/clang/lib/Frontend/Rewrite/RewriteModernObjC.cpp
index 3849e4040b53a17..f618c536b5f3c63 100644
--- a/clang/lib/Frontend/Rewrite/RewriteModernObjC.cpp
+++ b/clang/lib/Frontend/Rewrite/RewriteModernObjC.cpp
@@ -10,7 +10,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Rewrite/Frontend/ASTConsumers.h"
 #include "clang/AST/AST.h"
 #include "clang/AST/ASTConsumer.h"
 #include "clang/AST/Attr.h"
@@ -23,6 +22,7 @@
 #include "clang/Config/config.h"
 #include "clang/Lex/Lexer.h"
 #include "clang/Rewrite/Core/Rewriter.h"
+#include "clang/Rewrite/Frontend/ASTConsumers.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
@@ -34,6 +34,7 @@
 #if CLANG_ENABLE_OBJC_REWRITER
 
 using namespace clang;
+using llvm::RewriteBuffer;
 using llvm::utostr;
 
 namespace {
diff --git a/clang/lib/Frontend/Rewrite/RewriteObjC.cpp b/clang/lib/Frontend/Rewrite/RewriteObjC.cpp
index bf5176a2b6fb2e3..9db6ddbf0b8908f 100644
--- a/clang/lib/Frontend/Rewrite/RewriteObjC.cpp
+++ b/clang/lib/Frontend/Rewrite/RewriteObjC.cpp
@@ -32,6 +32,7 @@
 #if CLANG_ENABLE_OBJC_REWRITER
 
 using namespace clang;
+using llvm::RewriteBuffer;
 using llvm::utostr;
 
 namespace {
diff --git a/clang/lib/Rewrite/CMakeLists.txt b/clang/lib/Rewrite/CMakeLists.txt
index 16550b1b710ef9f..93ecb3c835260fe 100644
--- a/clang/lib/Rewrite/CMakeLists.txt
+++ b/clang/lib/Rewrite/CMakeLists.txt
@@ -3,9 +3,7 @@ set(LLVM_LINK_COMPONENTS
   )
 
 add_clang_library(clangRewrite
-  DeltaTree.cpp
   HTMLRewrite.cpp
-  RewriteRope.cpp
   Rewriter.cpp
   TokenRewriter.cpp
 
diff --git a/clang/lib/Rewrite/HTMLRewrite.cpp b/clang/lib/Rewrite/HTMLRewrite.cpp
index a96ca0764ae73d5..c75835df2c98ee8 100644
--- a/clang/lib/Rewrite/HTMLRewrite.cpp
+++ b/clang/lib/Rewrite/HTMLRewrite.cpp
@@ -16,6 +16,7 @@
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Lex/TokenConcatenation.h"
 #include "clang/Rewrite/Core/Rewriter.h"
+#include "llvm/ADT/RewriteBuffer.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/MemoryBuffer.h"
diff --git a/clang/lib/Rewrite/Rewriter.cpp b/clang/lib/Rewrite/Rewriter.cpp
index 0e6ae365064463b..68cf797f97905bc 100644
--- a/clang/lib/Rewrite/Rewriter.cpp
+++ b/clang/lib/Rewrite/Rewriter.cpp
@@ -17,8 +17,8 @@
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Lex/Lexer.h"
-#include "clang/Rewrite/Core/RewriteBuffer.h"
-#include "clang/Rewrite/Core/RewriteRope.h"
+#include "llvm/ADT/RewriteBuffer.h"
+#include "llvm/ADT/RewriteRope.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Error.h"
@@ -29,113 +29,18 @@
 #include <utility>
 
 using namespace clang;
+using llvm::RewriteBuffer;
 
-raw_ostream &RewriteBuffer::write(raw_ostream &os) const {
-  // Walk RewriteRope chunks efficiently using MoveToNextPiece() instead of the
-  // character iterator.
-  for (RopePieceBTreeIterator I = begin(), E = end(); I != E;
-       I.MoveToNextPiece())
-    os << I.piece();
-  return os;
-}
+//===----------------------------------------------------------------------===//
+// Rewriter class
+//===----------------------------------------------------------------------===//
 
 /// Return true if this character is non-new-line whitespace:
 /// ' ', '\\t', '\\f', '\\v', '\\r'.
 static inline bool isWhitespaceExceptNL(unsigned char c) {
-  switch (c) {
-  case ' ':
-  case '\t':
-  case '\f':
-  case '\v':
-  case '\r':
-    return true;
-  default:
-    return false;
-  }
+  return c == ' ' || c == '\t' || c == '\f' || c == '\v' || c == '\r';
 }
 
-void RewriteBuffer::RemoveText(unsigned OrigOffset, unsigned Size,
-                               bool removeLineIfEmpty) {
-  // Nothing to remove, exit early.
-  if (Size == 0) return;
-
-  unsigned RealOffset = getMappedOffset(OrigOffset, true);
-  assert(RealOffset+Size <= Buffer.size() && "Invalid location");
-
-  // Remove the dead characters.
-  Buffer.erase(RealOffset, Size);
-
-  // Add a delta so that future changes are offset correctly.
-  AddReplaceDelta(OrigOffset, -Size);
-
-  if (removeLineIfEmpty) {
-    // Find the line that the remove occurred and if it is completely empty
-    // remove the line as well.
-
-    iterator curLineStart = begin();
-    unsigned curLineStartOffs = 0;
-    iterator posI = begin();
-    for (unsigned i = 0; i != RealOffset; ++i) {
-      if (*posI == '\n') {
-        curLineStart = posI;
-        ++curLineStart;
-        curLineStartOffs = i + 1;
-      }
-      ++posI;
-    }
-
-    unsigned lineSize = 0;
-    posI = curLineStart;
-    while (posI != end() && isWhitespaceExceptNL(*posI)) {
-      ++posI;
-      ++lineSize;
-    }
-    if (posI != end() && *posI == '\n') {
-      Buffer.erase(curLineStartOffs, lineSize + 1/* + '\n'*/);
-      // FIXME: Here, the offset of the start of the line is supposed to be
-      // expressed in terms of the original input not the "real" rewrite
-      // buffer.  How do we compute that reliably?  It might be tempting to use
-      // curLineStartOffs + OrigOffset - RealOffset, but that assumes the
-      // difference between the original and real offset is the same at the
-      // removed text and at the start of the line, but that's not true if
-      // edits were previously made earlier on the line.  This bug is also
-      // documented by a FIXME on the definition of
-      // clang::Rewriter::RewriteOptions::RemoveLineIfEmpty.  A reproducer for
-      // the implementation below is the test RemoveLineIfEmpty in
-      // clang/unittests/Rewrite/RewriteBufferTest.cpp.
-      AddReplaceDelta(curLineStartOffs, -(lineSize + 1/* + '\n'*/));
-    }
-  }
-}
-
-void RewriteBuffer::InsertText(unsigned OrigOffset, StringRef Str,
-                               bool InsertAfter) {
-  // Nothing to insert, exit early.
-  if (Str.empty()) return;
-
-  unsigned RealOffset = getMappedOffset(OrigOffset, InsertAfter);
-  Buffer.insert(RealOffset, Str.begin(), Str.end());
-
-  // Add a delta so that future changes are offset correctly.
-  AddInsertDelta(OrigOffset, Str.size());
-}
-
-/// ReplaceText - This method replaces a range of characters in the input
-/// buffer with a new string.  This is effectively a combined "remove+insert"
-/// operation.
-void RewriteBuffer::ReplaceText(unsigned OrigOffset, unsigned OrigLength,
-                                StringRef NewStr) {
-  unsigned RealOffset = getMappedOffset(OrigOffset, true);
-  Buffer.erase(RealOffset, OrigLength);
-  Buffer.insert(RealOffset, NewStr.begin(), NewStr.end());
-  if (OrigLength != NewStr.size())
-    AddReplaceDelta(OrigOffset, NewStr.size() - OrigLength);
-}
-
-//===----------------------------------------------------------------------===//
-// Rewriter class
-//===----------------------------------------------------------------------===//
-
 /// getRangeSize - Return the size in bytes of the specified range if they
 /// are in the same file.  If not, this returns -1.
 int Rewriter::getRangeSize(const CharSourceRange &Range,
diff --git a/clang/lib/StaticAnalyzer/Core/HTMLDiagnostics.cpp b/clang/lib/StaticAnalyzer/Core/HTMLDiagnostics.cpp
index fb5030d373c2f18..28d9de2973b01fa 100644
--- a/clang/lib/StaticAnalyzer/Core/HTMLDiagnostics.cpp
+++ b/clang/lib/StaticAnalyzer/Core/HTMLDiagnostics.cpp
@@ -27,6 +27,7 @@
 #include "clang/Rewrite/Core/Rewriter.h"
 #include "clang/StaticAnalyzer/Core/PathDiagnosticConsumers.h"
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/RewriteBuffer.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/Sequence.h"
 #include "llvm/ADT/SmallString.h"
@@ -52,6 +53,7 @@
 
 using namespace clang;
 using namespace ento;
+using llvm::RewriteBuffer;
 
 //===----------------------------------------------------------------------===//
 // Boilerplate.
diff --git a/clang/lib/Tooling/Core/Replacement.cpp b/clang/lib/Tooling/Core/Replacement.cpp
index 269f17a6db4cfc1..9acf00263b5a8ce 100644
--- a/clang/lib/Tooling/Core/Replacement.cpp
+++ b/clang/lib/Tooling/Core/Replacement.cpp
@@ -19,9 +19,9 @@
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Lex/Lexer.h"
-#include "clang/Rewrite/Core/RewriteBuffer.h"
 #include "clang/Rewrite/Core/Rewriter.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/ADT/RewriteBuffer.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Error.h"
diff --git a/clang/unittests/Rewrite/CMakeLists.txt b/clang/unittests/Rewrite/CMakeLists.txt
index 6594d8cc5897f34..3c5e2f8e5354bee 100644
--- a/clang/unittests/Rewrite/CMakeLists.txt
+++ b/clang/unittests/Rewrite/CMakeLists.txt
@@ -3,7 +3,6 @@ set(LLVM_LINK_COMPONENTS
   )
 
 add_clang_unittest(RewriteTests
-  RewriteBufferTest.cpp
   RewriterTest.cpp
   )
 clang_target_link_libraries(RewriteTests
diff --git a/llvm/include/llvm/ADT/DeltaTree.h b/llvm/include/llvm/ADT/DeltaTree.h
new file mode 100644
index 000000000000000..5db9d74d9d5296f
--- /dev/null
+++ b/llvm/include/llvm/ADT/DeltaTree.h
@@ -0,0 +1,50 @@
+//===- DeltaTree.h - B-Tree for Rewrite Delta tracking ----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the DeltaTree class.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_ADT_DELTATREE_H
+#define LLVM_ADT_DELTATREE_H
+
+namespace llvm {
+
+/// DeltaTree - a multiway search tree (BTree) structure with some fancy
+/// features.  B-Trees are generally more memory and cache efficient than
+/// binary trees, because they store multiple keys/values in each node.  This
+/// implements a key/value mapping from index to delta, and allows fast lookup
+/// on index.  However, an added (important) bonus is that it can also
+/// efficiently tell us the full accumulated delta for a specific file offset
+/// as well, without traversing the whole tree.
+class DeltaTree {
+  void *Root; // "DeltaTreeNode *"
+
+public:
+  DeltaTree();
+
+  // Note: Currently we only support copying when the RHS is empty.
+  DeltaTree(const DeltaTree &RHS);
+
+  DeltaTree &operator=(const DeltaTree &) = delete;
+  ~DeltaTree();
+
+  /// getDeltaAt - Return the accumulated delta at the specified file offset.
+  /// This includes all insertions or delections that occurred *before* the
+  /// specified file index.
+  int getDeltaAt(unsigned FileIndex) const;
+
+  /// AddDelta - When a change is made that shifts around the text buffer,
+  /// this method is used to record that info.  It inserts a delta of 'Delta'
+  /// into the current DeltaTree at offset FileIndex.
+  void AddDelta(unsigned FileIndex, int Delta);
+};
+
+} // namespace llvm
+
+#endif // LLVM_ADT_DELTATREE_H
diff --git a/clang/include/clang/Rewrite/Core/RewriteBuffer.h b/llvm/include/llvm/ADT/RewriteBuffer.h
similarity index 82%
rename from clang/include/clang/Rewrite/Core/RewriteBuffer.h
rename to llvm/include/llvm/ADT/RewriteBuffer.h
index b8f34174b715664..8197a06f882c1aa 100644
--- a/clang/include/clang/Rewrite/Core/RewriteBuffer.h
+++ b/llvm/include/llvm/ADT/RewriteBuffer.h
@@ -6,15 +6,20 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_CLANG_REWRITE_CORE_REWRITEBUFFER_H
-#define LLVM_CLANG_REWRITE_CORE_REWRITEBUFFER_H
+#ifndef LLVM_ADT_REWRITEBUFFER_H
+#define LLVM_ADT_REWRITEBUFFER_H
 
-#include "clang/Basic/LLVM.h"
-#include "clang/Rewrite/Core/DeltaTree.h"
-#include "clang/Rewrite/Core/RewriteRope.h"
+#include "llvm/ADT/DeltaTree.h"
+#include "llvm/ADT/RewriteRope.h"
 #include "llvm/ADT/StringRef.h"
 
 namespace clang {
+class Rewriter;
+} // namespace clang
+
+namespace llvm {
+
+class raw_ostream;
 
 /// RewriteBuffer - As code is rewritten, SourceBuffer's from the original
 /// input with modifications get a new RewriteBuffer associated with them.  The
@@ -23,7 +28,7 @@ namespace clang {
 /// RewriteBuffer.  For example, if text is inserted into the buffer, any
 /// locations after the insertion point have to be mapped.
 class RewriteBuffer {
-  friend class Rewriter;
+  friend class clang::Rewriter;
 
   /// Deltas - Keep track of all the deltas in the source code due to insertions
   /// and deletions.
@@ -43,9 +48,7 @@ class RewriteBuffer {
   void Initialize(const char *BufStart, const char *BufEnd) {
     Buffer.assign(BufStart, BufEnd);
   }
-  void Initialize(StringRef Input) {
-    Initialize(Input.begin(), Input.end());
-  }
+  void Initialize(StringRef Input) { Initialize(Input.begin(), Input.end()); }
 
   /// Write to \p Stream the result of applying all changes to the
   /// original buffer.
@@ -63,9 +66,7 @@ class RewriteBuffer {
   /// InsertText - Insert some text at the specified point, where the offset in
   /// the buffer is specified relative to the original SourceBuffer.  The
   /// text is inserted after the specified location.
-  void InsertText(unsigned OrigOffset, StringRef Str,
-                  bool InsertAfter = true);
-
+  void InsertText(unsigned OrigOffset, StringRef Str, bool InsertAfter = true);
 
   /// InsertTextBefore - Insert some text before the specified point, where the
   /// offset in the buffer is specified relative to the original
@@ -85,8 +86,7 @@ class RewriteBuffer {
   /// ReplaceText - This method replaces a range of characters in the input
   /// buffer with a new string.  This is effectively a combined "remove/insert"
   /// operation.
-  void ReplaceText(unsigned OrigOffset, unsigned OrigLength,
-                   StringRef NewStr);
+  void ReplaceText(unsigned OrigOffset, unsigned OrigLength, StringRef NewStr);
 
 private:
   /// getMappedOffset - Given an offset into the original SourceBuffer that this
@@ -95,23 +95,23 @@ class RewriteBuffer {
   /// position where text is inserted, the location returned will be after any
   /// inserted text at the position.
   unsigned getMappedOffset(unsigned OrigOffset,
-                           bool AfterInserts = false) const{
-    return Deltas.getDeltaAt(2*OrigOffset+AfterInserts)+OrigOffset;
+                           bool AfterInserts = false) const {
+    return Deltas.getDeltaAt(2 * OrigOffset + AfterInserts) + OrigOffset;
   }
 
   /// AddInsertDelta - When an insertion is made at a position, this
   /// method is used to record that information.
   void AddInsertDelta(unsigned OrigOffset, int Change) {
-    return Deltas.AddDelta(2*OrigOffset, Change);
+    return Deltas.AddDelta(2 * OrigOffset, Change);
   }
 
   /// AddReplaceDelta - When a replacement/deletion is made at a position, this
   /// method is used to record that information.
   void AddReplaceDelta(unsigned OrigOffset, int Change) {
-    return Deltas.AddDelta(2*OrigOffset+1, Change);
+    return Deltas.AddDelta(2 * OrigOffset + 1, Change);
   }
 };
 
-} // namespace clang
+} // namespace llvm
 
-#endif // LLVM_CLANG_REWRITE_CORE_REWRITEBUFFER_H
+#endif // LLVM_ADT_REWRITEBUFFER_H
diff --git a/llvm/include/llvm/ADT/RewriteRope.h b/llvm/include/llvm/ADT/RewriteRope.h
new file mode 100644
index 000000000000000..784f3c07eaa62bc
--- /dev/null
+++ b/llvm/include/llvm/ADT/RewriteRope.h
@@ -0,0 +1,223 @@
+//===- RewriteRope.h - Rope specialized for rewriter ------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file defines the RewriteRope class, which is a powerful string class.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_ADT_REWRITEROPE_H
+#define LLVM_ADT_REWRITEROPE_H
+
+#include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/ADT/StringRef.h"
+#include <cassert>
+#include <cstddef>
+#include <iterator>
+#include <utility>
+
+namespace llvm {
+
+//===--------------------------------------------------------------------===//
+// RopeRefCountString Class
+//===--------------------------------------------------------------------===//
+
+/// RopeRefCountString - This struct is allocated with 'new char[]' from the
+/// heap, and represents a reference counted chunk of string data.  When its
+/// ref count drops to zero, it is delete[]'d.  This is primarily managed
+/// through the RopePiece class below.
+struct RopeRefCountString {
+  unsigned RefCount;
+  char Data[1]; //  Variable sized.
+
+  void Retain() { ++RefCount; }
+
+  void Release() {
+    assert(RefCount > 0 && "Reference count is already zero.");
+    if (--RefCount == 0)
+      delete[] (char *)this;
+  }
+};
+
+//===--------------------------------------------------------------------===//
+// RopePiece Class
+//===--------------------------------------------------------------------===//
+
+/// RopePiece - This class represents a view into a RopeRefCountString object.
+/// This allows references to string data to be efficiently chopped up and
+/// moved around without having to push around the string data itself.
+///
+/// For example, we could have a 1M RopePiece and want to insert something
+/// into the middle of it.  To do this, we split it into two RopePiece objects
+/// that both refer to the same underlying RopeRefCountString (just with
+/// different offsets) which is a nice constant time operation.
+struct RopePiece {
+  llvm::IntrusiveRefCntPtr<RopeRefCountString> StrData;
+  unsigned StartOffs = 0;
+  unsigned EndOffs = 0;
+
+  RopePiece() = default;
+  RopePiece(llvm::IntrusiveRefCntPtr<RopeRefCountString> Str, unsigned Start,
+            unsigned End)
+      : StrData(std::move(Str)), StartOffs(Start), EndOffs(End) {}
+
+  const char &operator[](unsigned Offset) const {
+    return StrData->Data[Offset + StartOffs];
+  }
+  char &operator[](unsigned Offset) {
+    return StrData->Data[Offset + StartOffs];
+  }
+
+  unsigned size() const { return EndOffs - StartOffs; }
+};
+
+//===--------------------------------------------------------------------===//
+// RopePieceBTreeIterator Class
+//===--------------------------------------------------------------------===//
+
+/// RopePieceBTreeIterator - This class provides read-only forward iteration
+/// over bytes that are in a RopePieceBTree.  This first iterates over bytes
+/// in a RopePiece, then iterates over RopePiece's in a RopePieceBTreeLeaf,
+/// then iterates over RopePieceBTreeLeaf's in a RopePieceBTree.
+class RopePieceBTreeIterator {
+  /// CurNode - The current B+Tree node that we are inspecting.
+  const void /*RopePieceBTreeLeaf*/ *CurNode = nullptr;
+
+  /// CurPiece - The current RopePiece in the B+Tree node that we're
+  /// inspecting.
+  const RopePiece *CurPiece = nullptr;
+
+  /// CurChar - The current byte in the RopePiece we are pointing to.
+  unsigned CurChar = 0;
+
+public:
+  using iterator_category = std::forward_iterator_tag;
+  using value_type = const char;
+  using difference_type = std::ptrdiff_t;
+  using pointer = value_type *;
+  using reference = value_type &;
+
+  RopePieceBTreeIterator() = default;
+  RopePieceBTreeIterator(const void /*RopePieceBTreeNode*/ *N);
+
+  char operator*() const { return (*CurPiece)[CurChar]; }
+
+  bool operator==(const RopePieceBTreeIterator &RHS) const {
+    return CurPiece == RHS.CurPiece && CurChar == RHS.CurChar;
+  }
+  bool operator!=(const RopePieceBTreeIterator &RHS) const {
+    return !operator==(RHS);
+  }
+
+  RopePieceBTreeIterator &operator++() { // Preincrement
+    if (CurChar + 1 < CurPiece->size())
+      ++CurChar;
+    else
+      MoveToNextPiece();
+    return *this;
+  }
+
+  RopePieceBTreeIterator operator++(int) { // Postincrement
+    RopePieceBTreeIterator tmp = *this;
+    ++*this;
+    return tmp;
+  }
+
+  llvm::StringRef piece() const {
+    return llvm::StringRef(&(*CurPiece)[0], CurPiece->size());
+  }
+
+  void MoveToNextPiece();
+};
+
+//===--------------------------------------------------------------------===//
+// RopePieceBTree Class
+//===--------------------------------------------------------------------===//
+
+class RopePieceBTree {
+  void /*RopePieceBTreeNode*/ *Root;
+
+public:
+  RopePieceBTree();
+  RopePieceBTree(const RopePieceBTree &RHS);
+  RopePieceBTree &operator=(const RopePieceBTree &) = delete;
+  ~RopePieceBTree();
+
+  using iterator = RopePieceBTreeIterator;
+
+  iterator begin() const { return iterator(Root); }
+  iterator end() const { return iterator(); }
+  unsigned size() const;
+  unsigned empty() const { return size() == 0; }
+
+  void clear();
+
+  void insert(unsigned Offset, const RopePiece &R);
+
+  void erase(unsigned Offset, unsigned NumBytes);
+};
+
+//===--------------------------------------------------------------------===//
+// RewriteRope Class
+//===--------------------------------------------------------------------===//
+
+/// RewriteRope - A powerful string class.  This class supports extremely
+/// efficient insertions and deletions into the middle of it, even for
+/// ridiculously long strings.
+class RewriteRope {
+  RopePieceBTree Chunks;
+
+  /// We allocate space for string data out of a buffer of size AllocChunkSize.
+  /// This keeps track of how much space is left.
+  llvm::IntrusiveRefCntPtr<RopeRefCountString> AllocBuffer;
+  enum { AllocChunkSize = 4080 };
+  unsigned AllocOffs = AllocChunkSize;
+
+public:
+  RewriteRope() = default;
+  RewriteRope(const RewriteRope &RHS) : Chunks(RHS.Chunks) {}
+
+  // The copy assignment operator is defined as deleted pending further
+  // motivation.
+  RewriteRope &operator=(const RewriteRope &) = delete;
+
+  using iterator = RopePieceBTree::iterator;
+  using const_iterator = RopePieceBTree::iterator;
+
+  iterator begin() const { return Chunks.begin(); }
+  iterator end() const { return Chunks.end(); }
+  unsigned size() const { return Chunks.size(); }
+
+  void clear() { Chunks.clear(); }
+
+  void assign(const char *Start, const char *End) {
+    clear();
+    if (Start != End)
+      Chunks.insert(0, MakeRopeString(Start, End));
+  }
+
+  void insert(unsigned Offset, const char *Start, const char *End) {
+    assert(Offset <= size() && "Invalid position to insert!");
+    if (Start == End)
+      return;
+    Chunks.insert(Offset, MakeRopeString(Start, End));
+  }
+
+  void erase(unsigned Offset, unsigned NumBytes) {
+    assert(Offset + NumBytes <= size() && "Invalid region to erase!");
+    if (NumBytes == 0)
+      return;
+    Chunks.erase(Offset, NumBytes);
+  }
+
+private:
+  RopePiece MakeRopeString(const char *Start, const char *End);
+};
+
+} // namespace llvm
+
+#endif // LLVM_ADT_REWRITEROPE_H
diff --git a/llvm/lib/Support/CMakeLists.txt b/llvm/lib/Support/CMakeLists.txt
index f653379e3033494..a73ac54a01c5a51 100644
--- a/llvm/lib/Support/CMakeLists.txt
+++ b/llvm/lib/Support/CMakeLists.txt
@@ -168,6 +168,7 @@ add_llvm_component_library(LLVMSupport
   Debug.cpp
   DebugCounter.cpp
   DeltaAlgorithm.cpp
+  DeltaTree.cpp
   DivisionByConstantInfo.cpp
   DAGDeltaAlgorithm.cpp
   DJB.cpp
@@ -216,6 +217,8 @@ add_llvm_component_library(LLVMSupport
   PrettyStackTrace.cpp
   RandomNumberGenerator.cpp
   Regex.cpp
+  RewriteBuffer.cpp
+  RewriteRope.cpp
   RISCVAttributes.cpp
   RISCVAttributeParser.cpp
   RISCVISAUtils.cpp
diff --git a/clang/lib/Rewrite/DeltaTree.cpp b/llvm/lib/Support/DeltaTree.cpp
similarity index 68%
rename from clang/lib/Rewrite/DeltaTree.cpp
rename to llvm/lib/Support/DeltaTree.cpp
index 61467f84928f2aa..2eb642ffd1df6c9 100644
--- a/clang/lib/Rewrite/DeltaTree.cpp
+++ b/llvm/lib/Support/DeltaTree.cpp
@@ -10,13 +10,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Rewrite/Core/DeltaTree.h"
-#include "clang/Basic/LLVM.h"
+#include "llvm/ADT/DeltaTree.h"
 #include "llvm/Support/Casting.h"
 #include <cassert>
 #include <cstring>
 
-using namespace clang;
+using namespace llvm;
 
 /// The DeltaTree class is a multiway search tree (BTree) structure with some
 /// fancy features.  B-Trees are generally more memory and cache efficient
@@ -35,125 +34,125 @@ using namespace clang;
 
 namespace {
 
-  /// SourceDelta - As code in the original input buffer is added and deleted,
-  /// SourceDelta records are used to keep track of how the input SourceLocation
-  /// object is mapped into the output buffer.
-  struct SourceDelta {
-    unsigned FileLoc;
-    int Delta;
-
-    static SourceDelta get(unsigned Loc, int D) {
-      SourceDelta Delta;
-      Delta.FileLoc = Loc;
-      Delta.Delta = D;
-      return Delta;
-    }
+/// SourceDelta - As code in the original input buffer is added and deleted,
+/// SourceDelta records are used to keep track of how the input SourceLocation
+/// object is mapped into the output buffer.
+struct SourceDelta {
+  unsigned FileLoc;
+  int Delta;
+
+  static SourceDelta get(unsigned Loc, int D) {
+    SourceDelta Delta;
+    Delta.FileLoc = Loc;
+    Delta.Delta = D;
+    return Delta;
+  }
+};
+
+/// DeltaTreeNode - The common part of all nodes.
+///
+class DeltaTreeNode {
+public:
+  struct InsertResult {
+    DeltaTreeNode *LHS, *RHS;
+    SourceDelta Split;
   };
 
-  /// DeltaTreeNode - The common part of all nodes.
-  ///
-  class DeltaTreeNode {
-  public:
-    struct InsertResult {
-      DeltaTreeNode *LHS, *RHS;
-      SourceDelta Split;
-    };
-
-  private:
-    friend class DeltaTreeInteriorNode;
-
-    /// WidthFactor - This controls the number of K/V slots held in the BTree:
-    /// how wide it is.  Each level of the BTree is guaranteed to have at least
-    /// WidthFactor-1 K/V pairs (except the root) and may have at most
-    /// 2*WidthFactor-1 K/V pairs.
-    enum { WidthFactor = 8 };
-
-    /// Values - This tracks the SourceDelta's currently in this node.
-    SourceDelta Values[2*WidthFactor-1];
-
-    /// NumValuesUsed - This tracks the number of values this node currently
-    /// holds.
-    unsigned char NumValuesUsed = 0;
-
-    /// IsLeaf - This is true if this is a leaf of the btree.  If false, this is
-    /// an interior node, and is actually an instance of DeltaTreeInteriorNode.
-    bool IsLeaf;
-
-    /// FullDelta - This is the full delta of all the values in this node and
-    /// all children nodes.
-    int FullDelta = 0;
+private:
+  friend class DeltaTreeInteriorNode;
 
-  public:
-    DeltaTreeNode(bool isLeaf = true) : IsLeaf(isLeaf) {}
+  /// WidthFactor - This controls the number of K/V slots held in the BTree:
+  /// how wide it is.  Each level of the BTree is guaranteed to have at least
+  /// WidthFactor-1 K/V pairs (except the root) and may have at most
+  /// 2*WidthFactor-1 K/V pairs.
+  enum { WidthFactor = 8 };
 
-    bool isLeaf() const { return IsLeaf; }
-    int getFullDelta() const { return FullDelta; }
-    bool isFull() const { return NumValuesUsed == 2*WidthFactor-1; }
+  /// Values - This tracks the SourceDelta's currently in this node.
+  SourceDelta Values[2 * WidthFactor - 1];
 
-    unsigned getNumValuesUsed() const { return NumValuesUsed; }
+  /// NumValuesUsed - This tracks the number of values this node currently
+  /// holds.
+  unsigned char NumValuesUsed = 0;
 
-    const SourceDelta &getValue(unsigned i) const {
-      assert(i < NumValuesUsed && "Invalid value #");
-      return Values[i];
-    }
+  /// IsLeaf - This is true if this is a leaf of the btree.  If false, this is
+  /// an interior node, and is actually an instance of DeltaTreeInteriorNode.
+  bool IsLeaf;
 
-    SourceDelta &getValue(unsigned i) {
-      assert(i < NumValuesUsed && "Invalid value #");
-      return Values[i];
-    }
+  /// FullDelta - This is the full delta of all the values in this node and
+  /// all children nodes.
+  int FullDelta = 0;
 
-    /// DoInsertion - Do an insertion of the specified FileIndex/Delta pair into
-    /// this node.  If insertion is easy, do it and return false.  Otherwise,
-    /// split the node, populate InsertRes with info about the split, and return
-    /// true.
-    bool DoInsertion(unsigned FileIndex, int Delta, InsertResult *InsertRes);
+public:
+  DeltaTreeNode(bool isLeaf = true) : IsLeaf(isLeaf) {}
 
-    void DoSplit(InsertResult &InsertRes);
+  bool isLeaf() const { return IsLeaf; }
+  int getFullDelta() const { return FullDelta; }
+  bool isFull() const { return NumValuesUsed == 2 * WidthFactor - 1; }
 
+  unsigned getNumValuesUsed() const { return NumValuesUsed; }
 
-    /// RecomputeFullDeltaLocally - Recompute the FullDelta field by doing a
-    /// local walk over our contained deltas.
-    void RecomputeFullDeltaLocally();
+  const SourceDelta &getValue(unsigned i) const {
+    assert(i < NumValuesUsed && "Invalid value #");
+    return Values[i];
+  }
 
-    void Destroy();
-  };
+  SourceDelta &getValue(unsigned i) {
+    assert(i < NumValuesUsed && "Invalid value #");
+    return Values[i];
+  }
 
-  /// DeltaTreeInteriorNode - When isLeaf = false, a node has child pointers.
-  /// This class tracks them.
-  class DeltaTreeInteriorNode : public DeltaTreeNode {
-    friend class DeltaTreeNode;
+  /// DoInsertion - Do an insertion of the specified FileIndex/Delta pair into
+  /// this node.  If insertion is easy, do it and return false.  Otherwise,
+  /// split the node, populate InsertRes with info about the split, and return
+  /// true.
+  bool DoInsertion(unsigned FileIndex, int Delta, InsertResult *InsertRes);
 
-    DeltaTreeNode *Children[2*WidthFactor];
+  void DoSplit(InsertResult &InsertRes);
 
-    ~DeltaTreeInteriorNode() {
-      for (unsigned i = 0, e = NumValuesUsed+1; i != e; ++i)
-        Children[i]->Destroy();
-    }
+  /// RecomputeFullDeltaLocally - Recompute the FullDelta field by doing a
+  /// local walk over our contained deltas.
+  void RecomputeFullDeltaLocally();
 
-  public:
-    DeltaTreeInteriorNode() : DeltaTreeNode(false /*nonleaf*/) {}
+  void Destroy();
+};
 
-    DeltaTreeInteriorNode(const InsertResult &IR)
-        : DeltaTreeNode(false /*nonleaf*/) {
-      Children[0] = IR.LHS;
-      Children[1] = IR.RHS;
-      Values[0] = IR.Split;
-      FullDelta = IR.LHS->getFullDelta()+IR.RHS->getFullDelta()+IR.Split.Delta;
-      NumValuesUsed = 1;
-    }
+/// DeltaTreeInteriorNode - When isLeaf = false, a node has child pointers.
+/// This class tracks them.
+class DeltaTreeInteriorNode : public DeltaTreeNode {
+  friend class DeltaTreeNode;
 
-    const DeltaTreeNode *getChild(unsigned i) const {
-      assert(i < getNumValuesUsed()+1 && "Invalid child");
-      return Children[i];
-    }
+  DeltaTreeNode *Children[2 * WidthFactor];
 
-    DeltaTreeNode *getChild(unsigned i) {
-      assert(i < getNumValuesUsed()+1 && "Invalid child");
-      return Children[i];
-    }
+  ~DeltaTreeInteriorNode() {
+    for (unsigned i = 0, e = NumValuesUsed + 1; i != e; ++i)
+      Children[i]->Destroy();
+  }
 
-    static bool classof(const DeltaTreeNode *N) { return !N->isLeaf(); }
-  };
+public:
+  DeltaTreeInteriorNode() : DeltaTreeNode(false /*nonleaf*/) {}
+
+  DeltaTreeInteriorNode(const InsertResult &IR)
+      : DeltaTreeNode(false /*nonleaf*/) {
+    Children[0] = IR.LHS;
+    Children[1] = IR.RHS;
+    Values[0] = IR.Split;
+    FullDelta =
+        IR.LHS->getFullDelta() + IR.RHS->getFullDelta() + IR.Split.Delta;
+    NumValuesUsed = 1;
+  }
+
+  const DeltaTreeNode *getChild(unsigned i) const {
+    assert(i < getNumValuesUsed() + 1 && "Invalid child");
+    return Children[i];
+  }
+
+  DeltaTreeNode *getChild(unsigned i) {
+    assert(i < getNumValuesUsed() + 1 && "Invalid child");
+    return Children[i];
+  }
+
+  static bool classof(const DeltaTreeNode *N) { return !N->isLeaf(); }
+};
 
 } // namespace
 
@@ -172,7 +171,7 @@ void DeltaTreeNode::RecomputeFullDeltaLocally() {
   for (unsigned i = 0, e = getNumValuesUsed(); i != e; ++i)
     NewFullDelta += Values[i].Delta;
   if (auto *IN = dyn_cast<DeltaTreeInteriorNode>(this))
-    for (unsigned i = 0, e = getNumValuesUsed()+1; i != e; ++i)
+    for (unsigned i = 0, e = getNumValuesUsed() + 1; i != e; ++i)
       NewFullDelta += IN->getChild(i)->getFullDelta();
   FullDelta = NewFullDelta;
 }
@@ -209,7 +208,7 @@ bool DeltaTreeNode::DoInsertion(unsigned FileIndex, int Delta,
       // For an insertion into a non-full leaf node, just insert the value in
       // its sorted position.  This requires moving later values over.
       if (i != e)
-        memmove(&Values[i+1], &Values[i], sizeof(Values[0])*(e-i));
+        memmove(&Values[i + 1], &Values[i], sizeof(Values[0]) * (e - i));
       Values[i] = SourceDelta::get(FileIndex, Delta);
       ++NumValuesUsed;
       return false;
@@ -239,13 +238,13 @@ bool DeltaTreeNode::DoInsertion(unsigned FileIndex, int Delta,
     // into ourself by moving all the later values/children down, then inserting
     // the new one.
     if (i != e)
-      memmove(&IN->Children[i+2], &IN->Children[i+1],
-              (e-i)*sizeof(IN->Children[0]));
+      memmove(&IN->Children[i + 2], &IN->Children[i + 1],
+              (e - i) * sizeof(IN->Children[0]));
     IN->Children[i] = InsertRes->LHS;
-    IN->Children[i+1] = InsertRes->RHS;
+    IN->Children[i + 1] = InsertRes->RHS;
 
     if (e != i)
-      memmove(&Values[i+1], &Values[i], (e-i)*sizeof(Values[0]));
+      memmove(&Values[i + 1], &Values[i], (e - i) * sizeof(Values[0]));
     Values[i] = InsertRes->Split;
     ++NumValuesUsed;
     return false;
@@ -272,20 +271,21 @@ bool DeltaTreeNode::DoInsertion(unsigned FileIndex, int Delta,
   // SubRHS/SubSplit into.  Find out where to insert SubSplit.
 
   // Find the insertion point, the first delta whose index is >SubSplit.FileLoc.
-  i = 0; e = InsertSide->getNumValuesUsed();
+  i = 0;
+  e = InsertSide->getNumValuesUsed();
   while (i != e && SubSplit.FileLoc > InsertSide->getValue(i).FileLoc)
     ++i;
 
   // Now we know that i is the place to insert the split value into.  Insert it
   // and the child right after it.
   if (i != e)
-    memmove(&InsertSide->Children[i+2], &InsertSide->Children[i+1],
-            (e-i)*sizeof(IN->Children[0]));
-  InsertSide->Children[i+1] = SubRHS;
+    memmove(&InsertSide->Children[i + 2], &InsertSide->Children[i + 1],
+            (e - i) * sizeof(IN->Children[0]));
+  InsertSide->Children[i + 1] = SubRHS;
 
   if (e != i)
-    memmove(&InsertSide->Values[i+1], &InsertSide->Values[i],
-            (e-i)*sizeof(Values[0]));
+    memmove(&InsertSide->Values[i + 1], &InsertSide->Values[i],
+            (e - i) * sizeof(Values[0]));
   InsertSide->Values[i] = SubSplit;
   ++InsertSide->NumValuesUsed;
   InsertSide->FullDelta += SubSplit.Delta + SubRHS->getFullDelta();
@@ -310,7 +310,7 @@ void DeltaTreeNode::DoSplit(InsertResult &InsertRes) {
     // into the new node.
     DeltaTreeInteriorNode *New = new DeltaTreeInteriorNode();
     memcpy(&New->Children[0], &IN->Children[WidthFactor],
-           WidthFactor*sizeof(IN->Children[0]));
+           WidthFactor * sizeof(IN->Children[0]));
     NewNode = New;
   } else {
     // Just create the new leaf node.
@@ -319,10 +319,10 @@ void DeltaTreeNode::DoSplit(InsertResult &InsertRes) {
 
   // Move over the last 'WidthFactor-1' values from here to NewNode.
   memcpy(&NewNode->Values[0], &Values[WidthFactor],
-         (WidthFactor-1)*sizeof(Values[0]));
+         (WidthFactor - 1) * sizeof(Values[0]));
 
   // Decrease the number of values in the two nodes.
-  NewNode->NumValuesUsed = NumValuesUsed = WidthFactor-1;
+  NewNode->NumValuesUsed = NumValuesUsed = WidthFactor - 1;
 
   // Recompute the two nodes' full delta.
   NewNode->RecomputeFullDeltaLocally();
@@ -330,14 +330,14 @@ void DeltaTreeNode::DoSplit(InsertResult &InsertRes) {
 
   InsertRes.LHS = this;
   InsertRes.RHS = NewNode;
-  InsertRes.Split = Values[WidthFactor-1];
+  InsertRes.Split = Values[WidthFactor - 1];
 }
 
 //===----------------------------------------------------------------------===//
 //                        DeltaTree Implementation
 //===----------------------------------------------------------------------===//
 
-//#define VERIFY_TREE
+// #define VERIFY_TREE
 
 #ifdef VERIFY_TREE
 /// VerifyTree - Walk the btree performing assertions on various properties to
@@ -350,7 +350,7 @@ static void VerifyTree(const DeltaTreeNode *N) {
     int FullDelta = 0;
     for (unsigned i = 0, e = N->getNumValuesUsed(); i != e; ++i) {
       if (i)
-        assert(N->getValue(i-1).FileLoc < N->getValue(i).FileLoc);
+        assert(N->getValue(i - 1).FileLoc < N->getValue(i).FileLoc);
       FullDelta += N->getValue(i).Delta;
     }
     assert(FullDelta == N->getFullDelta());
@@ -364,16 +364,16 @@ static void VerifyTree(const DeltaTreeNode *N) {
     const SourceDelta &IVal = N->getValue(i);
     const DeltaTreeNode *IChild = IN->getChild(i);
     if (i)
-      assert(IN->getValue(i-1).FileLoc < IVal.FileLoc);
+      assert(IN->getValue(i - 1).FileLoc < IVal.FileLoc);
     FullDelta += IVal.Delta;
     FullDelta += IChild->getFullDelta();
 
     // The largest value in child #i should be smaller than FileLoc.
-    assert(IChild->getValue(IChild->getNumValuesUsed()-1).FileLoc <
+    assert(IChild->getValue(IChild->getNumValuesUsed() - 1).FileLoc <
            IVal.FileLoc);
 
     // The smallest value in child #i+1 should be larger than FileLoc.
-    assert(IN->getChild(i+1)->getValue(0).FileLoc > IVal.FileLoc);
+    assert(IN->getChild(i + 1)->getValue(0).FileLoc > IVal.FileLoc);
     VerifyTree(IChild);
   }
 
@@ -381,15 +381,11 @@ static void VerifyTree(const DeltaTreeNode *N) {
 
   assert(FullDelta == N->getFullDelta());
 }
-#endif  // VERIFY_TREE
+#endif // VERIFY_TREE
 
-static DeltaTreeNode *getRoot(void *Root) {
-  return (DeltaTreeNode*)Root;
-}
+static DeltaTreeNode *getRoot(void *Root) { return (DeltaTreeNode *)Root; }
 
-DeltaTree::DeltaTree() {
-  Root = new DeltaTreeNode();
-}
+DeltaTree::DeltaTree() { Root = new DeltaTreeNode(); }
 
 DeltaTree::DeltaTree(const DeltaTree &RHS) {
   // Currently we only support copying when the RHS is empty.
@@ -398,9 +394,7 @@ DeltaTree::DeltaTree(const DeltaTree &RHS) {
   Root = new DeltaTreeNode();
 }
 
-DeltaTree::~DeltaTree() {
-  getRoot(Root)->Destroy();
-}
+DeltaTree::~DeltaTree() { getRoot(Root)->Destroy(); }
 
 /// getDeltaAt - Return the accumulated delta at the specified file offset.
 /// This includes all insertions or delections that occurred *before* the
@@ -428,7 +422,8 @@ int DeltaTree::getDeltaAt(unsigned FileIndex) const {
     // If we have an interior node, include information about children and
     // recurse.  Otherwise, if we have a leaf, we're done.
     const auto *IN = dyn_cast<DeltaTreeInteriorNode>(Node);
-    if (!IN) return Result;
+    if (!IN)
+      return Result;
 
     // Include any children to the left of the values we skipped, all of
     // their deltas should be included as well.
@@ -440,7 +435,7 @@ int DeltaTree::getDeltaAt(unsigned FileIndex) const {
     // partial results.
     if (NumValsGreater != Node->getNumValuesUsed() &&
         Node->getValue(NumValsGreater).FileLoc == FileIndex)
-      return Result+IN->getChild(NumValsGreater)->getFullDelta();
+      return Result + IN->getChild(NumValsGreater)->getFullDelta();
 
     // Otherwise, traverse down the tree.  The selected subtree may be
     // partially included in the range.
diff --git a/llvm/lib/Support/RewriteBuffer.cpp b/llvm/lib/Support/RewriteBuffer.cpp
new file mode 100644
index 000000000000000..cc6ba7e97b1a7fd
--- /dev/null
+++ b/llvm/lib/Support/RewriteBuffer.cpp
@@ -0,0 +1,107 @@
+//===- RewriteBuffer.h - Buffer rewriting interface -----------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/RewriteBuffer.h"
+#include "llvm/Support/raw_ostream.h"
+
+using namespace llvm;
+
+raw_ostream &RewriteBuffer::write(raw_ostream &Stream) const {
+  // Walk RewriteRope chunks efficiently using MoveToNextPiece() instead of the
+  // character iterator.
+  for (RopePieceBTreeIterator I = begin(), E = end(); I != E;
+       I.MoveToNextPiece())
+    Stream << I.piece();
+  return Stream;
+}
+
+/// Return true if this character is non-new-line whitespace:
+/// ' ', '\\t', '\\f', '\\v', '\\r'.
+static inline bool isWhitespaceExceptNL(unsigned char c) {
+  return c == ' ' || c == '\t' || c == '\f' || c == '\v' || c == '\r';
+}
+
+void RewriteBuffer::RemoveText(unsigned OrigOffset, unsigned Size,
+                               bool removeLineIfEmpty) {
+  // Nothing to remove, exit early.
+  if (Size == 0)
+    return;
+
+  unsigned RealOffset = getMappedOffset(OrigOffset, true);
+  assert(RealOffset + Size <= Buffer.size() && "Invalid location");
+
+  // Remove the dead characters.
+  Buffer.erase(RealOffset, Size);
+
+  // Add a delta so that future changes are offset correctly.
+  AddReplaceDelta(OrigOffset, -Size);
+
+  if (removeLineIfEmpty) {
+    // Find the line that the remove occurred and if it is completely empty
+    // remove the line as well.
+
+    iterator curLineStart = begin();
+    unsigned curLineStartOffs = 0;
+    iterator posI = begin();
+    for (unsigned i = 0; i != RealOffset; ++i) {
+      if (*posI == '\n') {
+        curLineStart = posI;
+        ++curLineStart;
+        curLineStartOffs = i + 1;
+      }
+      ++posI;
+    }
+
+    unsigned lineSize = 0;
+    posI = curLineStart;
+    while (posI != end() && isWhitespaceExceptNL(*posI)) {
+      ++posI;
+      ++lineSize;
+    }
+    if (posI != end() && *posI == '\n') {
+      Buffer.erase(curLineStartOffs, lineSize + 1 /* + '\n'*/);
+      // FIXME: Here, the offset of the start of the line is supposed to be
+      // expressed in terms of the original input not the "real" rewrite
+      // buffer.  How do we compute that reliably?  It might be tempting to use
+      // curLineStartOffs + OrigOffset - RealOffset, but that assumes the
+      // difference between the original and real offset is the same at the
+      // removed text and at the start of the line, but that's not true if
+      // edits were previously made earlier on the line.  This bug is also
+      // documented by a FIXME on the definition of
+      // clang::Rewriter::RewriteOptions::RemoveLineIfEmpty.  A reproducer for
+      // the implementation below is the test RemoveLineIfEmpty in
+      // clang/unittests/Rewrite/RewriteBufferTest.cpp.
+      AddReplaceDelta(curLineStartOffs, -(lineSize + 1 /* + '\n'*/));
+    }
+  }
+}
+
+void RewriteBuffer::InsertText(unsigned OrigOffset, StringRef Str,
+                               bool InsertAfter) {
+  // Nothing to insert, exit early.
+  if (Str.empty())
+    return;
+
+  unsigned RealOffset = getMappedOffset(OrigOffset, InsertAfter);
+  Buffer.insert(RealOffset, Str.begin(), Str.end());
+
+  // Add a delta so that future changes are offset correctly.
+  AddInsertDelta(OrigOffset, Str.size());
+}
+
+/// ReplaceText - This method replaces a range of characters in the input
+/// buffer with a new string.  This is effectively a combined "remove+insert"
+/// operation.
+void RewriteBuffer::ReplaceText(unsigned OrigOffset, unsigned OrigLength,
+                                StringRef NewStr) {
+  unsigned RealOffset = getMappedOffset(OrigOffset, true);
+  Buffer.erase(RealOffset, OrigLength);
+  Buffer.insert(RealOffset, NewStr.begin(), NewStr.end());
+  if (OrigLength != NewStr.size())
+    AddReplaceDelta(OrigOffset, NewStr.size() - OrigLength);
+}
diff --git a/clang/lib/Rewrite/RewriteRope.cpp b/llvm/lib/Support/RewriteRope.cpp
similarity index 63%
rename from clang/lib/Rewrite/RewriteRope.cpp
rename to llvm/lib/Support/RewriteRope.cpp
index 980d0f01e277c4d..7440aa6ab501e78 100644
--- a/clang/lib/Rewrite/RewriteRope.cpp
+++ b/llvm/lib/Support/RewriteRope.cpp
@@ -10,14 +10,13 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Rewrite/Core/RewriteRope.h"
-#include "clang/Basic/LLVM.h"
+#include "llvm/ADT/RewriteRope.h"
 #include "llvm/Support/Casting.h"
 #include <algorithm>
 #include <cassert>
 #include <cstring>
 
-using namespace clang;
+using namespace llvm;
 
 /// RewriteRope is a "strong" string class, designed to make insertions and
 /// deletions in the middle of the string nearly constant time (really, they are
@@ -68,162 +67,160 @@ namespace {
 // RopePieceBTreeNode Class
 //===----------------------------------------------------------------------===//
 
-  /// RopePieceBTreeNode - Common base class of RopePieceBTreeLeaf and
-  /// RopePieceBTreeInterior.  This provides some 'virtual' dispatching methods
-  /// and a flag that determines which subclass the instance is.  Also
-  /// important, this node knows the full extend of the node, including any
-  /// children that it has.  This allows efficient skipping over entire subtrees
-  /// when looking for an offset in the BTree.
-  class RopePieceBTreeNode {
-  protected:
-    /// WidthFactor - This controls the number of K/V slots held in the BTree:
-    /// how wide it is.  Each level of the BTree is guaranteed to have at least
-    /// 'WidthFactor' elements in it (either ropepieces or children), (except
-    /// the root, which may have less) and may have at most 2*WidthFactor
-    /// elements.
-    enum { WidthFactor = 8 };
-
-    /// Size - This is the number of bytes of file this node (including any
-    /// potential children) covers.
-    unsigned Size = 0;
-
-    /// IsLeaf - True if this is an instance of RopePieceBTreeLeaf, false if it
-    /// is an instance of RopePieceBTreeInterior.
-    bool IsLeaf;
-
-    RopePieceBTreeNode(bool isLeaf) : IsLeaf(isLeaf) {}
-    ~RopePieceBTreeNode() = default;
-
-  public:
-    bool isLeaf() const { return IsLeaf; }
-    unsigned size() const { return Size; }
-
-    void Destroy();
-
-    /// split - Split the range containing the specified offset so that we are
-    /// guaranteed that there is a place to do an insertion at the specified
-    /// offset.  The offset is relative, so "0" is the start of the node.
-    ///
-    /// If there is no space in this subtree for the extra piece, the extra tree
-    /// node is returned and must be inserted into a parent.
-    RopePieceBTreeNode *split(unsigned Offset);
-
-    /// insert - Insert the specified ropepiece into this tree node at the
-    /// specified offset.  The offset is relative, so "0" is the start of the
-    /// node.
-    ///
-    /// If there is no space in this subtree for the extra piece, the extra tree
-    /// node is returned and must be inserted into a parent.
-    RopePieceBTreeNode *insert(unsigned Offset, const RopePiece &R);
-
-    /// erase - Remove NumBytes from this node at the specified offset.  We are
-    /// guaranteed that there is a split at Offset.
-    void erase(unsigned Offset, unsigned NumBytes);
-  };
+/// RopePieceBTreeNode - Common base class of RopePieceBTreeLeaf and
+/// RopePieceBTreeInterior.  This provides some 'virtual' dispatching methods
+/// and a flag that determines which subclass the instance is.  Also
+/// important, this node knows the full extend of the node, including any
+/// children that it has.  This allows efficient skipping over entire subtrees
+/// when looking for an offset in the BTree.
+class RopePieceBTreeNode {
+protected:
+  /// WidthFactor - This controls the number of K/V slots held in the BTree:
+  /// how wide it is.  Each level of the BTree is guaranteed to have at least
+  /// 'WidthFactor' elements in it (either ropepieces or children), (except
+  /// the root, which may have less) and may have at most 2*WidthFactor
+  /// elements.
+  enum { WidthFactor = 8 };
+
+  /// Size - This is the number of bytes of file this node (including any
+  /// potential children) covers.
+  unsigned Size = 0;
+
+  /// IsLeaf - True if this is an instance of RopePieceBTreeLeaf, false if it
+  /// is an instance of RopePieceBTreeInterior.
+  bool IsLeaf;
+
+  RopePieceBTreeNode(bool isLeaf) : IsLeaf(isLeaf) {}
+  ~RopePieceBTreeNode() = default;
+
+public:
+  bool isLeaf() const { return IsLeaf; }
+  unsigned size() const { return Size; }
+
+  void Destroy();
+
+  /// split - Split the range containing the specified offset so that we are
+  /// guaranteed that there is a place to do an insertion at the specified
+  /// offset.  The offset is relative, so "0" is the start of the node.
+  ///
+  /// If there is no space in this subtree for the extra piece, the extra tree
+  /// node is returned and must be inserted into a parent.
+  RopePieceBTreeNode *split(unsigned Offset);
+
+  /// insert - Insert the specified ropepiece into this tree node at the
+  /// specified offset.  The offset is relative, so "0" is the start of the
+  /// node.
+  ///
+  /// If there is no space in this subtree for the extra piece, the extra tree
+  /// node is returned and must be inserted into a parent.
+  RopePieceBTreeNode *insert(unsigned Offset, const RopePiece &R);
+
+  /// erase - Remove NumBytes from this node at the specified offset.  We are
+  /// guaranteed that there is a split at Offset.
+  void erase(unsigned Offset, unsigned NumBytes);
+};
 
 //===----------------------------------------------------------------------===//
 // RopePieceBTreeLeaf Class
 //===----------------------------------------------------------------------===//
 
-  /// RopePieceBTreeLeaf - Directly manages up to '2*WidthFactor' RopePiece
-  /// nodes.  This directly represents a chunk of the string with those
-  /// RopePieces concatenated.  Since this is a B+Tree, all values (in this case
-  /// instances of RopePiece) are stored in leaves like this.  To make iteration
-  /// over the leaves efficient, they maintain a singly linked list through the
-  /// NextLeaf field.  This allows the B+Tree forward iterator to be constant
-  /// time for all increments.
-  class RopePieceBTreeLeaf : public RopePieceBTreeNode {
-    /// NumPieces - This holds the number of rope pieces currently active in the
-    /// Pieces array.
-    unsigned char NumPieces = 0;
-
-    /// Pieces - This tracks the file chunks currently in this leaf.
-    RopePiece Pieces[2*WidthFactor];
-
-    /// NextLeaf - This is a pointer to the next leaf in the tree, allowing
-    /// efficient in-order forward iteration of the tree without traversal.
-    RopePieceBTreeLeaf **PrevLeaf = nullptr;
-    RopePieceBTreeLeaf *NextLeaf = nullptr;
-
-  public:
-    RopePieceBTreeLeaf() : RopePieceBTreeNode(true) {}
-
-    ~RopePieceBTreeLeaf() {
-      if (PrevLeaf || NextLeaf)
-        removeFromLeafInOrder();
-      clear();
-    }
+/// RopePieceBTreeLeaf - Directly manages up to '2*WidthFactor' RopePiece
+/// nodes.  This directly represents a chunk of the string with those
+/// RopePieces concatenated.  Since this is a B+Tree, all values (in this case
+/// instances of RopePiece) are stored in leaves like this.  To make iteration
+/// over the leaves efficient, they maintain a singly linked list through the
+/// NextLeaf field.  This allows the B+Tree forward iterator to be constant
+/// time for all increments.
+class RopePieceBTreeLeaf : public RopePieceBTreeNode {
+  /// NumPieces - This holds the number of rope pieces currently active in the
+  /// Pieces array.
+  unsigned char NumPieces = 0;
+
+  /// Pieces - This tracks the file chunks currently in this leaf.
+  RopePiece Pieces[2 * WidthFactor];
+
+  /// NextLeaf - This is a pointer to the next leaf in the tree, allowing
+  /// efficient in-order forward iteration of the tree without traversal.
+  RopePieceBTreeLeaf **PrevLeaf = nullptr;
+  RopePieceBTreeLeaf *NextLeaf = nullptr;
+
+public:
+  RopePieceBTreeLeaf() : RopePieceBTreeNode(true) {}
+
+  ~RopePieceBTreeLeaf() {
+    if (PrevLeaf || NextLeaf)
+      removeFromLeafInOrder();
+    clear();
+  }
 
-    bool isFull() const { return NumPieces == 2*WidthFactor; }
+  bool isFull() const { return NumPieces == 2 * WidthFactor; }
 
-    /// clear - Remove all rope pieces from this leaf.
-    void clear() {
-      while (NumPieces)
-        Pieces[--NumPieces] = RopePiece();
-      Size = 0;
-    }
+  /// clear - Remove all rope pieces from this leaf.
+  void clear() {
+    while (NumPieces)
+      Pieces[--NumPieces] = RopePiece();
+    Size = 0;
+  }
 
-    unsigned getNumPieces() const { return NumPieces; }
+  unsigned getNumPieces() const { return NumPieces; }
 
-    const RopePiece &getPiece(unsigned i) const {
-      assert(i < getNumPieces() && "Invalid piece ID");
-      return Pieces[i];
-    }
+  const RopePiece &getPiece(unsigned i) const {
+    assert(i < getNumPieces() && "Invalid piece ID");
+    return Pieces[i];
+  }
 
-    const RopePieceBTreeLeaf *getNextLeafInOrder() const { return NextLeaf; }
+  const RopePieceBTreeLeaf *getNextLeafInOrder() const { return NextLeaf; }
 
-    void insertAfterLeafInOrder(RopePieceBTreeLeaf *Node) {
-      assert(!PrevLeaf && !NextLeaf && "Already in ordering");
+  void insertAfterLeafInOrder(RopePieceBTreeLeaf *Node) {
+    assert(!PrevLeaf && !NextLeaf && "Already in ordering");
 
-      NextLeaf = Node->NextLeaf;
-      if (NextLeaf)
-        NextLeaf->PrevLeaf = &NextLeaf;
-      PrevLeaf = &Node->NextLeaf;
-      Node->NextLeaf = this;
-    }
+    NextLeaf = Node->NextLeaf;
+    if (NextLeaf)
+      NextLeaf->PrevLeaf = &NextLeaf;
+    PrevLeaf = &Node->NextLeaf;
+    Node->NextLeaf = this;
+  }
 
-    void removeFromLeafInOrder() {
-      if (PrevLeaf) {
-        *PrevLeaf = NextLeaf;
-        if (NextLeaf)
-          NextLeaf->PrevLeaf = PrevLeaf;
-      } else if (NextLeaf) {
-        NextLeaf->PrevLeaf = nullptr;
-      }
+  void removeFromLeafInOrder() {
+    if (PrevLeaf) {
+      *PrevLeaf = NextLeaf;
+      if (NextLeaf)
+        NextLeaf->PrevLeaf = PrevLeaf;
+    } else if (NextLeaf) {
+      NextLeaf->PrevLeaf = nullptr;
     }
+  }
 
-    /// FullRecomputeSizeLocally - This method recomputes the 'Size' field by
-    /// summing the size of all RopePieces.
-    void FullRecomputeSizeLocally() {
-      Size = 0;
-      for (unsigned i = 0, e = getNumPieces(); i != e; ++i)
-        Size += getPiece(i).size();
-    }
+  /// FullRecomputeSizeLocally - This method recomputes the 'Size' field by
+  /// summing the size of all RopePieces.
+  void FullRecomputeSizeLocally() {
+    Size = 0;
+    for (unsigned i = 0, e = getNumPieces(); i != e; ++i)
+      Size += getPiece(i).size();
+  }
 
-    /// split - Split the range containing the specified offset so that we are
-    /// guaranteed that there is a place to do an insertion at the specified
-    /// offset.  The offset is relative, so "0" is the start of the node.
-    ///
-    /// If there is no space in this subtree for the extra piece, the extra tree
-    /// node is returned and must be inserted into a parent.
-    RopePieceBTreeNode *split(unsigned Offset);
-
-    /// insert - Insert the specified ropepiece into this tree node at the
-    /// specified offset.  The offset is relative, so "0" is the start of the
-    /// node.
-    ///
-    /// If there is no space in this subtree for the extra piece, the extra tree
-    /// node is returned and must be inserted into a parent.
-    RopePieceBTreeNode *insert(unsigned Offset, const RopePiece &R);
-
-    /// erase - Remove NumBytes from this node at the specified offset.  We are
-    /// guaranteed that there is a split at Offset.
-    void erase(unsigned Offset, unsigned NumBytes);
-
-    static bool classof(const RopePieceBTreeNode *N) {
-      return N->isLeaf();
-    }
-  };
+  /// split - Split the range containing the specified offset so that we are
+  /// guaranteed that there is a place to do an insertion at the specified
+  /// offset.  The offset is relative, so "0" is the start of the node.
+  ///
+  /// If there is no space in this subtree for the extra piece, the extra tree
+  /// node is returned and must be inserted into a parent.
+  RopePieceBTreeNode *split(unsigned Offset);
+
+  /// insert - Insert the specified ropepiece into this tree node at the
+  /// specified offset.  The offset is relative, so "0" is the start of the
+  /// node.
+  ///
+  /// If there is no space in this subtree for the extra piece, the extra tree
+  /// node is returned and must be inserted into a parent.
+  RopePieceBTreeNode *insert(unsigned Offset, const RopePiece &R);
+
+  /// erase - Remove NumBytes from this node at the specified offset.  We are
+  /// guaranteed that there is a split at Offset.
+  void erase(unsigned Offset, unsigned NumBytes);
+
+  static bool classof(const RopePieceBTreeNode *N) { return N->isLeaf(); }
+};
 
 } // namespace
 
@@ -244,7 +241,7 @@ RopePieceBTreeNode *RopePieceBTreeLeaf::split(unsigned Offset) {
   // Find the piece that this offset lands in.
   unsigned PieceOffs = 0;
   unsigned i = 0;
-  while (Offset >= PieceOffs+Pieces[i].size()) {
+  while (Offset >= PieceOffs + Pieces[i].size()) {
     PieceOffs += Pieces[i].size();
     ++i;
   }
@@ -256,13 +253,13 @@ RopePieceBTreeNode *RopePieceBTreeLeaf::split(unsigned Offset) {
 
   // Otherwise, we need to split piece 'i' at Offset-PieceOffs.  Convert Offset
   // to being Piece relative.
-  unsigned IntraPieceOffset = Offset-PieceOffs;
+  unsigned IntraPieceOffset = Offset - PieceOffs;
 
   // We do this by shrinking the RopePiece and then doing an insert of the tail.
-  RopePiece Tail(Pieces[i].StrData, Pieces[i].StartOffs+IntraPieceOffset,
+  RopePiece Tail(Pieces[i].StrData, Pieces[i].StartOffs + IntraPieceOffset,
                  Pieces[i].EndOffs);
   Size -= Pieces[i].size();
-  Pieces[i].EndOffs = Pieces[i].StartOffs+IntraPieceOffset;
+  Pieces[i].EndOffs = Pieces[i].StartOffs + IntraPieceOffset;
   Size += Pieces[i].size();
 
   return insert(Offset, Tail);
@@ -293,7 +290,7 @@ RopePieceBTreeNode *RopePieceBTreeLeaf::insert(unsigned Offset,
     // For an insertion into a non-full leaf node, just insert the value in
     // its sorted position.  This requires moving later values over.
     for (; i != e; --e)
-      Pieces[e] = Pieces[e-1];
+      Pieces[e] = Pieces[e - 1];
     Pieces[i] = R;
     ++NumPieces;
     Size += R.size();
@@ -309,10 +306,10 @@ RopePieceBTreeNode *RopePieceBTreeLeaf::insert(unsigned Offset,
   RopePieceBTreeLeaf *NewNode = new RopePieceBTreeLeaf();
 
   // Move over the last 'WidthFactor' values from here to NewNode.
-  std::copy(&Pieces[WidthFactor], &Pieces[2*WidthFactor],
+  std::copy(&Pieces[WidthFactor], &Pieces[2 * WidthFactor],
             &NewNode->Pieces[0]);
   // Replace old pieces with null RopePieces to drop refcounts.
-  std::fill(&Pieces[WidthFactor], &Pieces[2*WidthFactor], RopePiece());
+  std::fill(&Pieces[WidthFactor], &Pieces[2 * WidthFactor], RopePiece());
 
   // Decrease the number of values in the two nodes.
   NewNode->NumPieces = NumPieces = WidthFactor;
@@ -347,33 +344,34 @@ void RopePieceBTreeLeaf::erase(unsigned Offset, unsigned NumBytes) {
 
   // Figure out how many pieces completely cover 'NumBytes'.  We want to remove
   // all of them.
-  for (; Offset+NumBytes > PieceOffs+getPiece(i).size(); ++i)
+  for (; Offset + NumBytes > PieceOffs + getPiece(i).size(); ++i)
     PieceOffs += getPiece(i).size();
 
   // If we exactly include the last one, include it in the region to delete.
-  if (Offset+NumBytes == PieceOffs+getPiece(i).size()) {
+  if (Offset + NumBytes == PieceOffs + getPiece(i).size()) {
     PieceOffs += getPiece(i).size();
     ++i;
   }
 
   // If we completely cover some RopePieces, erase them now.
   if (i != StartPiece) {
-    unsigned NumDeleted = i-StartPiece;
+    unsigned NumDeleted = i - StartPiece;
     for (; i != getNumPieces(); ++i)
-      Pieces[i-NumDeleted] = Pieces[i];
+      Pieces[i - NumDeleted] = Pieces[i];
 
     // Drop references to dead rope pieces.
-    std::fill(&Pieces[getNumPieces()-NumDeleted], &Pieces[getNumPieces()],
+    std::fill(&Pieces[getNumPieces() - NumDeleted], &Pieces[getNumPieces()],
               RopePiece());
     NumPieces -= NumDeleted;
 
-    unsigned CoverBytes = PieceOffs-Offset;
+    unsigned CoverBytes = PieceOffs - Offset;
     NumBytes -= CoverBytes;
     Size -= CoverBytes;
   }
 
   // If we completely removed some stuff, we could be done.
-  if (NumBytes == 0) return;
+  if (NumBytes == 0)
+    return;
 
   // Okay, now might be erasing part of some Piece.  If this is the case, then
   // move the start point of the piece.
@@ -390,81 +388,79 @@ void RopePieceBTreeLeaf::erase(unsigned Offset, unsigned NumBytes) {
 
 namespace {
 
-  /// RopePieceBTreeInterior - This represents an interior node in the B+Tree,
-  /// which holds up to 2*WidthFactor pointers to child nodes.
-  class RopePieceBTreeInterior : public RopePieceBTreeNode {
-    /// NumChildren - This holds the number of children currently active in the
-    /// Children array.
-    unsigned char NumChildren = 0;
+/// RopePieceBTreeInterior - This represents an interior node in the B+Tree,
+/// which holds up to 2*WidthFactor pointers to child nodes.
+class RopePieceBTreeInterior : public RopePieceBTreeNode {
+  /// NumChildren - This holds the number of children currently active in the
+  /// Children array.
+  unsigned char NumChildren = 0;
 
-    RopePieceBTreeNode *Children[2*WidthFactor];
+  RopePieceBTreeNode *Children[2 * WidthFactor];
 
-  public:
-    RopePieceBTreeInterior() : RopePieceBTreeNode(false) {}
+public:
+  RopePieceBTreeInterior() : RopePieceBTreeNode(false) {}
 
-    RopePieceBTreeInterior(RopePieceBTreeNode *LHS, RopePieceBTreeNode *RHS)
-        : RopePieceBTreeNode(false) {
-      Children[0] = LHS;
-      Children[1] = RHS;
-      NumChildren = 2;
-      Size = LHS->size() + RHS->size();
-    }
+  RopePieceBTreeInterior(RopePieceBTreeNode *LHS, RopePieceBTreeNode *RHS)
+      : RopePieceBTreeNode(false) {
+    Children[0] = LHS;
+    Children[1] = RHS;
+    NumChildren = 2;
+    Size = LHS->size() + RHS->size();
+  }
 
-    ~RopePieceBTreeInterior() {
-      for (unsigned i = 0, e = getNumChildren(); i != e; ++i)
-        Children[i]->Destroy();
-    }
+  ~RopePieceBTreeInterior() {
+    for (unsigned i = 0, e = getNumChildren(); i != e; ++i)
+      Children[i]->Destroy();
+  }
 
-    bool isFull() const { return NumChildren == 2*WidthFactor; }
+  bool isFull() const { return NumChildren == 2 * WidthFactor; }
 
-    unsigned getNumChildren() const { return NumChildren; }
+  unsigned getNumChildren() const { return NumChildren; }
 
-    const RopePieceBTreeNode *getChild(unsigned i) const {
-      assert(i < NumChildren && "invalid child #");
-      return Children[i];
-    }
+  const RopePieceBTreeNode *getChild(unsigned i) const {
+    assert(i < NumChildren && "invalid child #");
+    return Children[i];
+  }
 
-    RopePieceBTreeNode *getChild(unsigned i) {
-      assert(i < NumChildren && "invalid child #");
-      return Children[i];
-    }
+  RopePieceBTreeNode *getChild(unsigned i) {
+    assert(i < NumChildren && "invalid child #");
+    return Children[i];
+  }
 
-    /// FullRecomputeSizeLocally - Recompute the Size field of this node by
-    /// summing up the sizes of the child nodes.
-    void FullRecomputeSizeLocally() {
-      Size = 0;
-      for (unsigned i = 0, e = getNumChildren(); i != e; ++i)
-        Size += getChild(i)->size();
-    }
+  /// FullRecomputeSizeLocally - Recompute the Size field of this node by
+  /// summing up the sizes of the child nodes.
+  void FullRecomputeSizeLocally() {
+    Size = 0;
+    for (unsigned i = 0, e = getNumChildren(); i != e; ++i)
+      Size += getChild(i)->size();
+  }
 
-    /// split - Split the range containing the specified offset so that we are
-    /// guaranteed that there is a place to do an insertion at the specified
-    /// offset.  The offset is relative, so "0" is the start of the node.
-    ///
-    /// If there is no space in this subtree for the extra piece, the extra tree
-    /// node is returned and must be inserted into a parent.
-    RopePieceBTreeNode *split(unsigned Offset);
-
-    /// insert - Insert the specified ropepiece into this tree node at the
-    /// specified offset.  The offset is relative, so "0" is the start of the
-    /// node.
-    ///
-    /// If there is no space in this subtree for the extra piece, the extra tree
-    /// node is returned and must be inserted into a parent.
-    RopePieceBTreeNode *insert(unsigned Offset, const RopePiece &R);
-
-    /// HandleChildPiece - A child propagated an insertion result up to us.
-    /// Insert the new child, and/or propagate the result further up the tree.
-    RopePieceBTreeNode *HandleChildPiece(unsigned i, RopePieceBTreeNode *RHS);
-
-    /// erase - Remove NumBytes from this node at the specified offset.  We are
-    /// guaranteed that there is a split at Offset.
-    void erase(unsigned Offset, unsigned NumBytes);
-
-    static bool classof(const RopePieceBTreeNode *N) {
-      return !N->isLeaf();
-    }
-  };
+  /// split - Split the range containing the specified offset so that we are
+  /// guaranteed that there is a place to do an insertion at the specified
+  /// offset.  The offset is relative, so "0" is the start of the node.
+  ///
+  /// If there is no space in this subtree for the extra piece, the extra tree
+  /// node is returned and must be inserted into a parent.
+  RopePieceBTreeNode *split(unsigned Offset);
+
+  /// insert - Insert the specified ropepiece into this tree node at the
+  /// specified offset.  The offset is relative, so "0" is the start of the
+  /// node.
+  ///
+  /// If there is no space in this subtree for the extra piece, the extra tree
+  /// node is returned and must be inserted into a parent.
+  RopePieceBTreeNode *insert(unsigned Offset, const RopePiece &R);
+
+  /// HandleChildPiece - A child propagated an insertion result up to us.
+  /// Insert the new child, and/or propagate the result further up the tree.
+  RopePieceBTreeNode *HandleChildPiece(unsigned i, RopePieceBTreeNode *RHS);
+
+  /// erase - Remove NumBytes from this node at the specified offset.  We are
+  /// guaranteed that there is a split at Offset.
+  void erase(unsigned Offset, unsigned NumBytes);
+
+  static bool classof(const RopePieceBTreeNode *N) { return !N->isLeaf(); }
+};
 
 } // namespace
 
@@ -481,7 +477,7 @@ RopePieceBTreeNode *RopePieceBTreeInterior::split(unsigned Offset) {
 
   unsigned ChildOffset = 0;
   unsigned i = 0;
-  for (; Offset >= ChildOffset+getChild(i)->size(); ++i)
+  for (; Offset >= ChildOffset + getChild(i)->size(); ++i)
     ChildOffset += getChild(i)->size();
 
   // If already split there, we're done.
@@ -489,7 +485,7 @@ RopePieceBTreeNode *RopePieceBTreeInterior::split(unsigned Offset) {
     return nullptr;
 
   // Otherwise, recursively split the child.
-  if (RopePieceBTreeNode *RHS = getChild(i)->split(Offset-ChildOffset))
+  if (RopePieceBTreeNode *RHS = getChild(i)->split(Offset - ChildOffset))
     return HandleChildPiece(i, RHS);
   return nullptr; // Done!
 }
@@ -509,17 +505,17 @@ RopePieceBTreeNode *RopePieceBTreeInterior::insert(unsigned Offset,
   unsigned ChildOffs = 0;
   if (Offset == size()) {
     // Fastpath for a common case.  Insert at end of last child.
-    i = e-1;
-    ChildOffs = size()-getChild(i)->size();
+    i = e - 1;
+    ChildOffs = size() - getChild(i)->size();
   } else {
-    for (; Offset > ChildOffs+getChild(i)->size(); ++i)
+    for (; Offset > ChildOffs + getChild(i)->size(); ++i)
       ChildOffs += getChild(i)->size();
   }
 
   Size += R.size();
 
   // Insert at the end of this child.
-  if (RopePieceBTreeNode *RHS = getChild(i)->insert(Offset-ChildOffs, R))
+  if (RopePieceBTreeNode *RHS = getChild(i)->insert(Offset - ChildOffs, R))
     return HandleChildPiece(i, RHS);
 
   return nullptr;
@@ -534,9 +530,9 @@ RopePieceBTreeInterior::HandleChildPiece(unsigned i, RopePieceBTreeNode *RHS) {
   if (!isFull()) {
     // Insert RHS after child 'i'.
     if (i + 1 != getNumChildren())
-      memmove(&Children[i+2], &Children[i+1],
-              (getNumChildren()-i-1)*sizeof(Children[0]));
-    Children[i+1] = RHS;
+      memmove(&Children[i + 2], &Children[i + 1],
+              (getNumChildren() - i - 1) * sizeof(Children[0]));
+    Children[i + 1] = RHS;
     ++NumChildren;
     return nullptr;
   }
@@ -549,7 +545,7 @@ RopePieceBTreeInterior::HandleChildPiece(unsigned i, RopePieceBTreeNode *RHS) {
 
   // Move over the last 'WidthFactor' values from here to NewNode.
   memcpy(&NewNode->Children[0], &Children[WidthFactor],
-         WidthFactor*sizeof(Children[0]));
+         WidthFactor * sizeof(Children[0]));
 
   // Decrease the number of values in the two nodes.
   NewNode->NumChildren = NumChildren = WidthFactor;
@@ -559,7 +555,7 @@ RopePieceBTreeInterior::HandleChildPiece(unsigned i, RopePieceBTreeNode *RHS) {
   if (i < WidthFactor)
     this->HandleChildPiece(i, RHS);
   else
-    NewNode->HandleChildPiece(i-WidthFactor, RHS);
+    NewNode->HandleChildPiece(i - WidthFactor, RHS);
 
   // Recompute the two nodes' size.
   NewNode->FullRecomputeSizeLocally();
@@ -585,7 +581,7 @@ void RopePieceBTreeInterior::erase(unsigned Offset, unsigned NumBytes) {
 
     // If we are deleting something contained entirely in the child, pass on the
     // request.
-    if (Offset+NumBytes < CurChild->size()) {
+    if (Offset + NumBytes < CurChild->size()) {
       CurChild->erase(Offset, NumBytes);
       return;
     }
@@ -593,7 +589,7 @@ void RopePieceBTreeInterior::erase(unsigned Offset, unsigned NumBytes) {
     // If this deletion request starts somewhere in the middle of the child, it
     // must be deleting to the end of the child.
     if (Offset) {
-      unsigned BytesFromChild = CurChild->size()-Offset;
+      unsigned BytesFromChild = CurChild->size() - Offset;
       CurChild->erase(Offset, BytesFromChild);
       NumBytes -= BytesFromChild;
       // Start at the beginning of the next child.
@@ -608,8 +604,8 @@ void RopePieceBTreeInterior::erase(unsigned Offset, unsigned NumBytes) {
     CurChild->Destroy();
     --NumChildren;
     if (i != getNumChildren())
-      memmove(&Children[i], &Children[i+1],
-              (getNumChildren()-i)*sizeof(Children[0]));
+      memmove(&Children[i], &Children[i + 1],
+              (getNumChildren() - i) * sizeof(Children[0]));
   }
 }
 
@@ -654,7 +650,7 @@ RopePieceBTreeNode *RopePieceBTreeNode::insert(unsigned Offset,
 /// erase - Remove NumBytes from this node at the specified offset.  We are
 /// guaranteed that there is a split at Offset.
 void RopePieceBTreeNode::erase(unsigned Offset, unsigned NumBytes) {
-  assert(Offset+NumBytes <= size() && "Invalid offset to erase!");
+  assert(Offset + NumBytes <= size() && "Invalid offset to erase!");
   if (auto *Leaf = dyn_cast<RopePieceBTreeLeaf>(this))
     return Leaf->erase(Offset, NumBytes);
   return cast<RopePieceBTreeInterior>(this)->erase(Offset, NumBytes);
@@ -665,7 +661,7 @@ void RopePieceBTreeNode::erase(unsigned Offset, unsigned NumBytes) {
 //===----------------------------------------------------------------------===//
 
 static const RopePieceBTreeLeaf *getCN(const void *P) {
-  return static_cast<const RopePieceBTreeLeaf*>(P);
+  return static_cast<const RopePieceBTreeLeaf *>(P);
 }
 
 // begin iterator.
@@ -686,13 +682,14 @@ RopePieceBTreeIterator::RopePieceBTreeIterator(const void *n) {
 
   if (CurNode)
     CurPiece = &getCN(CurNode)->getPiece(0);
-  else  // Empty tree, this is an end() iterator.
+  else // Empty tree, this is an end() iterator.
     CurPiece = nullptr;
   CurChar = 0;
 }
 
 void RopePieceBTreeIterator::MoveToNextPiece() {
-  if (CurPiece != &getCN(CurNode)->getPiece(getCN(CurNode)->getNumPieces()-1)) {
+  if (CurPiece !=
+      &getCN(CurNode)->getPiece(getCN(CurNode)->getNumPieces() - 1)) {
     CurChar = 0;
     ++CurPiece;
     return;
@@ -715,25 +712,19 @@ void RopePieceBTreeIterator::MoveToNextPiece() {
 //===----------------------------------------------------------------------===//
 
 static RopePieceBTreeNode *getRoot(void *P) {
-  return static_cast<RopePieceBTreeNode*>(P);
+  return static_cast<RopePieceBTreeNode *>(P);
 }
 
-RopePieceBTree::RopePieceBTree() {
-  Root = new RopePieceBTreeLeaf();
-}
+RopePieceBTree::RopePieceBTree() { Root = new RopePieceBTreeLeaf(); }
 
 RopePieceBTree::RopePieceBTree(const RopePieceBTree &RHS) {
   assert(RHS.empty() && "Can't copy non-empty tree yet");
   Root = new RopePieceBTreeLeaf();
 }
 
-RopePieceBTree::~RopePieceBTree() {
-  getRoot(Root)->Destroy();
-}
+RopePieceBTree::~RopePieceBTree() { getRoot(Root)->Destroy(); }
 
-unsigned RopePieceBTree::size() const {
-  return getRoot(Root)->size();
-}
+unsigned RopePieceBTree::size() const { return getRoot(Root)->size(); }
 
 void RopePieceBTree::clear() {
   if (auto *Leaf = dyn_cast<RopePieceBTreeLeaf>(getRoot(Root)))
@@ -772,24 +763,24 @@ void RopePieceBTree::erase(unsigned Offset, unsigned NumBytes) {
 /// the AllocBuffer object to aggregate requests for small strings into one
 /// allocation instead of doing tons of tiny allocations.
 RopePiece RewriteRope::MakeRopeString(const char *Start, const char *End) {
-  unsigned Len = End-Start;
+  unsigned Len = End - Start;
   assert(Len && "Zero length RopePiece is invalid!");
 
   // If we have space for this string in the current alloc buffer, use it.
-  if (AllocOffs+Len <= AllocChunkSize) {
-    memcpy(AllocBuffer->Data+AllocOffs, Start, Len);
+  if (AllocOffs + Len <= AllocChunkSize) {
+    memcpy(AllocBuffer->Data + AllocOffs, Start, Len);
     AllocOffs += Len;
-    return RopePiece(AllocBuffer, AllocOffs-Len, AllocOffs);
+    return RopePiece(AllocBuffer, AllocOffs - Len, AllocOffs);
   }
 
   // If we don't have enough room because this specific allocation is huge,
   // just allocate a new rope piece for it alone.
   if (Len > AllocChunkSize) {
-    unsigned Size = End-Start+sizeof(RopeRefCountString)-1;
+    unsigned Size = End - Start + sizeof(RopeRefCountString) - 1;
     auto *Res = reinterpret_cast<RopeRefCountString *>(new char[Size]);
     Res->RefCount = 0;
-    memcpy(Res->Data, Start, End-Start);
-    return RopePiece(Res, 0, End-Start);
+    memcpy(Res->Data, Start, End - Start);
+    return RopePiece(Res, 0, End - Start);
   }
 
   // Otherwise, this was a small request but we just don't have space for it
diff --git a/llvm/unittests/ADT/CMakeLists.txt b/llvm/unittests/ADT/CMakeLists.txt
index f48d840a105952f..745e4d9fb74a4a6 100644
--- a/llvm/unittests/ADT/CMakeLists.txt
+++ b/llvm/unittests/ADT/CMakeLists.txt
@@ -63,6 +63,7 @@ add_llvm_unittest(ADTTests
   PostOrderIteratorTest.cpp
   PriorityWorklistTest.cpp
   RangeAdapterTest.cpp
+  RewriteBufferTest.cpp
   SCCIteratorTest.cpp
   STLExtrasTest.cpp
   STLForwardCompatTest.cpp
diff --git a/clang/unittests/Rewrite/RewriteBufferTest.cpp b/llvm/unittests/ADT/RewriteBufferTest.cpp
similarity index 96%
rename from clang/unittests/Rewrite/RewriteBufferTest.cpp
rename to llvm/unittests/ADT/RewriteBufferTest.cpp
index bf29c0ff928304e..3b367daa7806e7a 100644
--- a/clang/unittests/Rewrite/RewriteBufferTest.cpp
+++ b/llvm/unittests/ADT/RewriteBufferTest.cpp
@@ -6,11 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Rewrite/Core/RewriteBuffer.h"
+#include "llvm/ADT/RewriteBuffer.h"
 #include "gtest/gtest.h"
 
 using namespace llvm;
-using namespace clang;
 
 namespace {
 
@@ -32,7 +31,7 @@ static void tagRange(unsigned Offset, unsigned Len, StringRef tagName,
   raw_string_ostream(EndTag) << "</" << tagName << '>';
 
   Buf.InsertTextAfter(Offset, BeginTag);
-  Buf.InsertTextBefore(Offset+Len, EndTag);
+  Buf.InsertTextBefore(Offset + Len, EndTag);
 }
 
 TEST(RewriteBuffer, TagRanges) {

From a28c20244242e6f4faad3e8ac673cb8f972c9d7a Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Sun, 18 Aug 2024 16:47:14 +0000
Subject: [PATCH 305/441] [gn build] Port 0d150db214e2

---
 llvm/utils/gn/secondary/clang/lib/Rewrite/BUILD.gn       | 2 --
 llvm/utils/gn/secondary/clang/unittests/Rewrite/BUILD.gn | 5 +----
 llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn        | 3 +++
 llvm/utils/gn/secondary/llvm/unittests/ADT/BUILD.gn      | 1 +
 4 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/llvm/utils/gn/secondary/clang/lib/Rewrite/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/Rewrite/BUILD.gn
index 2ea3cdd393bda50..586b33f5df9daf7 100644
--- a/llvm/utils/gn/secondary/clang/lib/Rewrite/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/lib/Rewrite/BUILD.gn
@@ -7,9 +7,7 @@ static_library("Rewrite") {
     "//llvm/lib/Support",
   ]
   sources = [
-    "DeltaTree.cpp",
     "HTMLRewrite.cpp",
-    "RewriteRope.cpp",
     "Rewriter.cpp",
     "TokenRewriter.cpp",
   ]
diff --git a/llvm/utils/gn/secondary/clang/unittests/Rewrite/BUILD.gn b/llvm/utils/gn/secondary/clang/unittests/Rewrite/BUILD.gn
index 52003f8a8db0fa4..88eeff60d05d406 100644
--- a/llvm/utils/gn/secondary/clang/unittests/Rewrite/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/unittests/Rewrite/BUILD.gn
@@ -9,8 +9,5 @@ unittest("RewriteTests") {
     "//clang/lib/Tooling",
     "//llvm/lib/Support",
   ]
-  sources = [
-    "RewriteBufferTest.cpp",
-    "RewriterTest.cpp",
-  ]
+  sources = [ "RewriterTest.cpp" ]
 }
diff --git a/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn
index d6b6b71a1e59b3b..d10a6e4be3770c1 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn
@@ -73,6 +73,7 @@ static_library("Support") {
     "Debug.cpp",
     "DebugCounter.cpp",
     "DeltaAlgorithm.cpp",
+    "DeltaTree.cpp",
     "DivisionByConstantInfo.cpp",
     "DynamicAPInt.cpp",
     "ELFAttributeParser.cpp",
@@ -123,6 +124,8 @@ static_library("Support") {
     "RWMutex.cpp",
     "RandomNumberGenerator.cpp",
     "Regex.cpp",
+    "RewriteBuffer.cpp",
+    "RewriteRope.cpp",
     "SHA1.cpp",
     "SHA256.cpp",
     "ScaledNumber.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/unittests/ADT/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/ADT/BUILD.gn
index c081bc03c66957e..c27faaaecf30d16 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/ADT/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/ADT/BUILD.gn
@@ -71,6 +71,7 @@ unittest("ADTTests") {
     "PostOrderIteratorTest.cpp",
     "PriorityWorklistTest.cpp",
     "RangeAdapterTest.cpp",
+    "RewriteBufferTest.cpp",
     "SCCIteratorTest.cpp",
     "STLExtrasTest.cpp",
     "STLForwardCompatTest.cpp",

From caa1070b1c880627b223dc3884860a64e4b510c7 Mon Sep 17 00:00:00 2001
From: "Yaxun (Sam) Liu" <yaxun.liu@amd.com>
Date: Sun, 18 Aug 2024 13:29:30 -0400
Subject: [PATCH 306/441] [HIP] search fatbin symbols for libs passed by -l
 (#104638)

For -fgpu-rdc linking, clang needs to collect undefined fatbin symbols
and resolve them to the embedded fatbin.

This has been done for object files and archive files passed as input
files to clang.

However, the same action is not performed for archive files passed
through -l options, which causes missing symbols.

This patch adds that.
---
 clang/lib/Driver/ToolChains/HIPUtility.cpp | 75 ++++++++++++++++++++--
 clang/test/Driver/hip-toolchain-rdc.hip    | 23 +++++++
 2 files changed, 93 insertions(+), 5 deletions(-)

diff --git a/clang/lib/Driver/ToolChains/HIPUtility.cpp b/clang/lib/Driver/ToolChains/HIPUtility.cpp
index f32a23f111e4bf9..1b707376dea819c 100644
--- a/clang/lib/Driver/ToolChains/HIPUtility.cpp
+++ b/clang/lib/Driver/ToolChains/HIPUtility.cpp
@@ -52,13 +52,16 @@ static std::string normalizeForBundler(const llvm::Triple &T,
 // input object or archive files.
 class HIPUndefinedFatBinSymbols {
 public:
-  HIPUndefinedFatBinSymbols(const Compilation &C)
-      : C(C), DiagID(C.getDriver().getDiags().getCustomDiagID(
-                  DiagnosticsEngine::Error,
-                  "Error collecting HIP undefined fatbin symbols: %0")),
+  HIPUndefinedFatBinSymbols(const Compilation &C,
+                            const llvm::opt::ArgList &Args_)
+      : C(C), Args(Args_),
+        DiagID(C.getDriver().getDiags().getCustomDiagID(
+            DiagnosticsEngine::Error,
+            "Error collecting HIP undefined fatbin symbols: %0")),
         Quiet(C.getArgs().hasArg(options::OPT__HASH_HASH_HASH)),
         Verbose(C.getArgs().hasArg(options::OPT_v)) {
     populateSymbols();
+    processStaticLibraries();
     if (Verbose) {
       for (const auto &Name : FatBinSymbols)
         llvm::errs() << "Found undefined HIP fatbin symbol: " << Name << "\n";
@@ -76,8 +79,70 @@ class HIPUndefinedFatBinSymbols {
     return GPUBinHandleSymbols;
   }
 
+  // Collect symbols from static libraries specified by -l options.
+  void processStaticLibraries() {
+    llvm::SmallVector<llvm::StringRef, 16> LibNames;
+    llvm::SmallVector<llvm::StringRef, 16> LibPaths;
+    llvm::SmallVector<llvm::StringRef, 16> ExactLibNames;
+    llvm::Triple Triple(C.getDriver().getTargetTriple());
+    bool IsMSVC = Triple.isWindowsMSVCEnvironment();
+    llvm::StringRef Ext = IsMSVC ? ".lib" : ".a";
+
+    for (const auto *Arg : Args.filtered(options::OPT_l)) {
+      llvm::StringRef Value = Arg->getValue();
+      if (Value.starts_with(":"))
+        ExactLibNames.push_back(Value.drop_front());
+      else
+        LibNames.push_back(Value);
+    }
+    for (const auto *Arg : Args.filtered(options::OPT_L)) {
+      auto Path = Arg->getValue();
+      LibPaths.push_back(Path);
+      if (Verbose)
+        llvm::errs() << "HIP fatbin symbol search uses library path:  " << Path
+                     << "\n";
+    }
+
+    auto ProcessLib = [&](llvm::StringRef LibName, bool IsExact) {
+      llvm::SmallString<256> FullLibName(
+          IsExact  ? Twine(LibName).str()
+          : IsMSVC ? (Twine(LibName) + Ext).str()
+                   : (Twine("lib") + LibName + Ext).str());
+
+      bool Found = false;
+      for (const auto Path : LibPaths) {
+        llvm::SmallString<256> FullPath = Path;
+        llvm::sys::path::append(FullPath, FullLibName);
+
+        if (llvm::sys::fs::exists(FullPath)) {
+          if (Verbose)
+            llvm::errs() << "HIP fatbin symbol search found library: "
+                         << FullPath << "\n";
+          auto BufferOrErr = llvm::MemoryBuffer::getFile(FullPath);
+          if (!BufferOrErr) {
+            errorHandler(llvm::errorCodeToError(BufferOrErr.getError()));
+            continue;
+          }
+          processInput(BufferOrErr.get()->getMemBufferRef());
+          Found = true;
+          break;
+        }
+      }
+      if (!Found && Verbose)
+        llvm::errs() << "HIP fatbin symbol search could not find library: "
+                     << FullLibName << "\n";
+    };
+
+    for (const auto LibName : ExactLibNames)
+      ProcessLib(LibName, true);
+
+    for (const auto LibName : LibNames)
+      ProcessLib(LibName, false);
+  }
+
 private:
   const Compilation &C;
+  const llvm::opt::ArgList &Args;
   unsigned DiagID;
   bool Quiet;
   bool Verbose;
@@ -301,7 +366,7 @@ void HIP::constructGenerateObjFileFromHIPFatBinary(
   auto HostTriple =
       C.getSingleOffloadToolChain<Action::OFK_Host>()->getTriple();
 
-  HIPUndefinedFatBinSymbols Symbols(C);
+  HIPUndefinedFatBinSymbols Symbols(C, Args);
 
   std::string PrimaryHipFatbinSymbol;
   std::string PrimaryGpuBinHandleSymbol;
diff --git a/clang/test/Driver/hip-toolchain-rdc.hip b/clang/test/Driver/hip-toolchain-rdc.hip
index 7e6697a0e254f6c..ec79bf06afb92c2 100644
--- a/clang/test/Driver/hip-toolchain-rdc.hip
+++ b/clang/test/Driver/hip-toolchain-rdc.hip
@@ -20,6 +20,29 @@
 // RUN:   %S/Inputs/hip_multiple_inputs/b.hip \
 // RUN: 2>&1 | FileCheck -check-prefixes=CHECK,MSVC %s
 
+// Test fatbin symbol search for -l libraries.
+
+// RUN: touch librdctest.a rdctest2.bin
+// RUN: %clang -### --target=x86_64-linux-gnu -v \
+// RUN:   -fgpu-rdc -nogpuinc -nogpulib \
+// RUN:   --no-offload-new-driver -L. -lrdctest -l:rdctest2.bin \
+// RUN:   %s \
+// RUN: 2>&1 | FileCheck -check-prefixes=LIB,LNX-LIB %s
+// RUN: rm librdctest.a rdctest2.bin
+
+// RUN: touch rdctest.lib rdctest2.bin
+// RUN: %clang -### --target=x86_64-pc-windows-msvc -v \
+// RUN:   -fgpu-rdc -nogpuinc -nogpulib \
+// RUN:   --no-offload-new-driver -L. -lrdctest -l:rdctest2.bin \
+// RUN:   %s \
+// RUN: 2>&1 | FileCheck -check-prefixes=LIB,MSVC-LIB %s
+// RUN: rm rdctest.lib rdctest2.bin
+
+// LIB: HIP fatbin symbol search uses library path: .
+// LIB: HIP fatbin symbol search found library: .{{/|\\}}rdctest2.bin
+// LNX-LIB: HIP fatbin symbol search found library: .{{/|\\}}librdctest.a
+// MSVC-LIB: HIP fatbin symbol search found library: .{{/|\\}}rdctest.lib
+
 // check HIP fatbin and gpubin handle symbols and code object alignment in dumped llvm-mc input
 // CHECK: Found undefined HIP fatbin symbol: __hip_fatbin_[[ID1:[0-9a-f]+]]
 // CHECK: Found undefined HIP fatbin symbol: __hip_fatbin_[[ID2:[0-9a-f]+]]

From 5c13f9aea280f1d499be7eca45b85371b0240c48 Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Sun, 18 Aug 2024 12:50:15 -0500
Subject: [PATCH 307/441] [libc] Add single threaded kernel attributes to
 AMDGPU startup utility (#104651)

Summary:
I fixed the errors here recently so I can actually use these. This
shouldn't impact much, just should hopefully make the code generated
slightly better.
---
 libc/startup/gpu/amdgpu/start.cpp | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/libc/startup/gpu/amdgpu/start.cpp b/libc/startup/gpu/amdgpu/start.cpp
index 5aaa7e938d27924..f09541b0d558081 100644
--- a/libc/startup/gpu/amdgpu/start.cpp
+++ b/libc/startup/gpu/amdgpu/start.cpp
@@ -41,7 +41,9 @@ static void call_fini_array_callbacks() {
 
 } // namespace LIBC_NAMESPACE_DECL
 
-extern "C" [[gnu::visibility("protected"), clang::amdgpu_kernel]] void
+extern "C" [[gnu::visibility("protected"), clang::amdgpu_kernel,
+             clang::amdgpu_flat_work_group_size(1, 1),
+             clang::amdgpu_max_num_work_groups(1)]] void
 _begin(int argc, char **argv, char **env) {
   __atomic_store_n(&LIBC_NAMESPACE::app.env_ptr,
                    reinterpret_cast<uintptr_t *>(env), __ATOMIC_RELAXED);
@@ -60,7 +62,9 @@ _start(int argc, char **argv, char **envp, int *ret) {
   __atomic_fetch_or(ret, main(argc, argv, envp), __ATOMIC_RELAXED);
 }
 
-extern "C" [[gnu::visibility("protected"), clang::amdgpu_kernel]] void
+extern "C" [[gnu::visibility("protected"), clang::amdgpu_kernel,
+             clang::amdgpu_flat_work_group_size(1, 1),
+             clang::amdgpu_max_num_work_groups(1)]] void
 _end(int retval) {
   // Only a single thread should call `exit` here, the rest should gracefully
   // return from the kernel. This is so only one thread calls the destructors

From deb6b45c32687275a6d4e24326ffc9700f8ae52c Mon Sep 17 00:00:00 2001
From: jameshu15869 <55058507+jameshu15869@users.noreply.github.com>
Date: Sun, 18 Aug 2024 13:50:30 -0400
Subject: [PATCH 308/441] [libc][gpu] Add Atan2 Benchmarks (#104708)

This PR adds benchmarking for `atan2()`, `__nv_atan2()`, and
`__ocml_atan2_f64()` using the same setup as `sin()`. This PR also adds
support for throughout bencmarking for functions with 2 inputs.
---
 libc/benchmarks/gpu/LibcGpuBenchmark.cpp      |  4 +-
 libc/benchmarks/gpu/LibcGpuBenchmark.h        | 21 +++++++--
 libc/benchmarks/gpu/src/math/CMakeLists.txt   | 19 ++++++++
 .../gpu/src/math/atan2_benchmark.cpp          | 47 +++++++++++++++++++
 libc/benchmarks/gpu/timing/amdgpu/timing.h    | 25 ++++++++++
 libc/benchmarks/gpu/timing/nvptx/timing.h     | 26 ++++++++++
 6 files changed, 137 insertions(+), 5 deletions(-)
 create mode 100644 libc/benchmarks/gpu/src/math/atan2_benchmark.cpp

diff --git a/libc/benchmarks/gpu/LibcGpuBenchmark.cpp b/libc/benchmarks/gpu/LibcGpuBenchmark.cpp
index f237e2ea1b9545e..920c5b206b0fefa 100644
--- a/libc/benchmarks/gpu/LibcGpuBenchmark.cpp
+++ b/libc/benchmarks/gpu/LibcGpuBenchmark.cpp
@@ -115,7 +115,7 @@ void print_results(Benchmark *b) {
   cpp::atomic_thread_fence(cpp::MemoryOrder::RELEASE);
 
   LIBC_NAMESPACE::printf(
-      "%-20s |%8ld |%8ld |%8ld |%11d |%14ld %2s |%9ld |%9d |\n",
+      "%-24s |%8ld |%8ld |%8ld |%11d |%14ld %2s |%9ld |%9d |\n",
       b->get_test_name().data(), result.cycles, result.min, result.max,
       result.total_iterations, result.total_time, time_unit,
       static_cast<uint64_t>(result.standard_deviation), num_threads);
@@ -127,7 +127,7 @@ void print_header() {
                          benchmarks[0]->get_suite_name().data());
   LIBC_NAMESPACE::printf("%s", RESET);
   cpp::string titles =
-      "Benchmark            |  Cycles |     Min |     Max | "
+      "Benchmark                |  Cycles |     Min |     Max | "
       "Iterations | Time / Iteration |   Stddev |  Threads |\n";
   LIBC_NAMESPACE::printf(titles.data());
 
diff --git a/libc/benchmarks/gpu/LibcGpuBenchmark.h b/libc/benchmarks/gpu/LibcGpuBenchmark.h
index 830e6f9e89a7439..f2cfbfbfdcdf0db 100644
--- a/libc/benchmarks/gpu/LibcGpuBenchmark.h
+++ b/libc/benchmarks/gpu/LibcGpuBenchmark.h
@@ -146,10 +146,8 @@ template <typename T> class MathPerf {
       cpp::numeric_limits<StorageType>::max();
 
 public:
-  typedef T Func(T);
-
   template <size_t N = 1>
-  static uint64_t run_throughput_in_range(Func f, int min_exp, int max_exp) {
+  static uint64_t run_throughput_in_range(T f(T), int min_exp, int max_exp) {
     cpp::array<T, N> inputs;
     for (size_t i = 0; i < N; ++i)
       inputs[i] = get_rand_input<T>(min_exp, max_exp);
@@ -158,6 +156,23 @@ template <typename T> class MathPerf {
 
     return total_time / N;
   }
+
+  // Throughput benchmarking for functions that take 2 inputs.
+  template <size_t N = 1>
+  static uint64_t run_throughput_in_range(T f(T, T), int arg1_min_exp,
+                                          int arg1_max_exp, int arg2_min_exp,
+                                          int arg2_max_exp) {
+    cpp::array<T, N> inputs1;
+    cpp::array<T, N> inputs2;
+    for (size_t i = 0; i < N; ++i) {
+      inputs1[i] = get_rand_input<T>(arg1_min_exp, arg1_max_exp);
+      inputs2[i] = get_rand_input<T>(arg2_min_exp, arg2_max_exp);
+    }
+
+    uint64_t total_time = LIBC_NAMESPACE::throughput(f, inputs1, inputs2);
+
+    return total_time / N;
+  }
 };
 
 } // namespace benchmarks
diff --git a/libc/benchmarks/gpu/src/math/CMakeLists.txt b/libc/benchmarks/gpu/src/math/CMakeLists.txt
index 335da5ad71cf882..6870c024490166d 100644
--- a/libc/benchmarks/gpu/src/math/CMakeLists.txt
+++ b/libc/benchmarks/gpu/src/math/CMakeLists.txt
@@ -43,3 +43,22 @@ add_benchmark(
   LOADER_ARGS
     --threads 64
 )
+
+add_benchmark(
+  atan2_benchmark
+  SUITE
+    libc-gpu-math-benchmarks
+  SRCS
+    atan2_benchmark.cpp
+  DEPENDS
+    libc.src.math.atan2
+    libc.src.stdlib.srand
+    libc.src.stdlib.rand
+    libc.src.__support.FPUtil.fp_bits
+    libc.src.__support.CPP.bit
+    libc.src.__support.CPP.array
+  COMPILE_OPTIONS
+    ${math_benchmark_flags}
+  LOADER_ARGS
+    --threads 64
+)
diff --git a/libc/benchmarks/gpu/src/math/atan2_benchmark.cpp b/libc/benchmarks/gpu/src/math/atan2_benchmark.cpp
new file mode 100644
index 000000000000000..3bb5b0cc6788ca1
--- /dev/null
+++ b/libc/benchmarks/gpu/src/math/atan2_benchmark.cpp
@@ -0,0 +1,47 @@
+#include "benchmarks/gpu/LibcGpuBenchmark.h"
+
+#include "src/math/atan2.h"
+#include "src/stdlib/rand.h"
+
+#ifdef NVPTX_MATH_FOUND
+#include "src/math/nvptx/declarations.h"
+#endif
+
+#ifdef AMDGPU_MATH_FOUND
+#include "src/math/amdgpu/declarations.h"
+#endif
+
+#define BM_TWO_RANDOM_INPUT(T, Func, MIN_EXP, MAX_EXP, N)                      \
+  []() {                                                                       \
+    return LIBC_NAMESPACE::benchmarks::MathPerf<T>::run_throughput_in_range<   \
+        N>(Func, MIN_EXP, MAX_EXP, MIN_EXP, MAX_EXP);                          \
+  }
+
+#define BENCH(T, Name, Func, MIN_EXP, MAX_EXP)                                 \
+  SINGLE_WAVE_BENCHMARK(LlvmLibcAtan2GpuBenchmark, Name##_1,                   \
+                        BM_TWO_RANDOM_INPUT(T, Func, MIN_EXP, MAX_EXP, 1));    \
+  SINGLE_WAVE_BENCHMARK(LlvmLibcAtan2GpuBenchmark, Name##_128,                 \
+                        BM_TWO_RANDOM_INPUT(T, Func, MIN_EXP, MAX_EXP, 128));  \
+  SINGLE_WAVE_BENCHMARK(LlvmLibcAtan2GpuBenchmark, Name##_1024,                \
+                        BM_TWO_RANDOM_INPUT(T, Func, MIN_EXP, MAX_EXP, 1024)); \
+  SINGLE_WAVE_BENCHMARK(LlvmLibcAtan2GpuBenchmark, Name##_4096,                \
+                        BM_TWO_RANDOM_INPUT(T, Func, MIN_EXP, MAX_EXP, 4096))
+
+BENCH(double, Atan2, LIBC_NAMESPACE::atan2, -1023, 1023);
+BENCH(double, Atan2TwoPi, LIBC_NAMESPACE::atan2, -10, 3);
+BENCH(double, Atan2TwoPow30, LIBC_NAMESPACE::atan2, 0, 30);
+BENCH(double, Atan2Large, LIBC_NAMESPACE::atan2, 30, 1000);
+
+#ifdef NVPTX_MATH_FOUND
+BENCH(double, NvAtan2, LIBC_NAMESPACE::__nv_atan2, -1023, 1023);
+BENCH(double, NvAtan2TwoPi, LIBC_NAMESPACE::__nv_atan2, -10, 3);
+BENCH(double, NvAtan2TwoPow30, LIBC_NAMESPACE::__nv_atan2, 0, 30);
+BENCH(double, NvAtan2Large, LIBC_NAMESPACE::__nv_atan2, 30, 1000);
+#endif
+
+#ifdef AMDGPU_MATH_FOUND
+BENCH(double, AmdAtan2, LIBC_NAMESPACE::__ocml_atan2_f64, -1023, 1023);
+BENCH(double, AmdAtan2TwoPi, LIBC_NAMESPACE::__ocml_atan2_f64, -10, 3);
+BENCH(double, AmdAtan2TwoPow30, LIBC_NAMESPACE::__ocml_atan2_f64, 0, 30);
+BENCH(double, AmdAtan2Large, LIBC_NAMESPACE::__ocml_atan2_f64, 30, 1000);
+#endif
diff --git a/libc/benchmarks/gpu/timing/amdgpu/timing.h b/libc/benchmarks/gpu/timing/amdgpu/timing.h
index d5c3df27b7de605..4cf7e9838add344 100644
--- a/libc/benchmarks/gpu/timing/amdgpu/timing.h
+++ b/libc/benchmarks/gpu/timing/amdgpu/timing.h
@@ -130,6 +130,31 @@ throughput(F f, const cpp::array<T, N> &inputs) {
   return stop - start;
 }
 
+// Provides throughput benchmarking for 2 arguments (e.g. atan2())
+template <typename F, typename T, size_t N>
+[[gnu::noinline]] static LIBC_INLINE uint64_t throughput(
+    F f, const cpp::array<T, N> &inputs1, const cpp::array<T, N> &inputs2) {
+  asm("" ::"v"(&inputs1), "v"(&inputs2));
+
+  gpu::memory_fence();
+  uint64_t start = gpu::processor_clock();
+
+  asm("" ::"s"(start));
+
+  for (size_t i = 0; i < inputs1.size(); i++) {
+    auto result = f(inputs1[i], inputs2[i]);
+
+    asm("" ::"v"(result));
+  }
+
+  uint64_t stop = gpu::processor_clock();
+  asm("" ::"s"(stop));
+  gpu::memory_fence();
+
+  // Return the time elapsed.
+  return stop - start;
+}
+
 } // namespace LIBC_NAMESPACE_DECL
 
 #endif // LLVM_LIBC_UTILS_GPU_TIMING_AMDGPU
diff --git a/libc/benchmarks/gpu/timing/nvptx/timing.h b/libc/benchmarks/gpu/timing/nvptx/timing.h
index 637986abd9092da..ece7d9a6c5396c2 100644
--- a/libc/benchmarks/gpu/timing/nvptx/timing.h
+++ b/libc/benchmarks/gpu/timing/nvptx/timing.h
@@ -121,6 +121,32 @@ throughput(F f, const cpp::array<T, N> &inputs) {
   // Return the time elapsed.
   return stop - start;
 }
+
+// Provides throughput benchmarking for 2 arguments (e.g. atan2())
+template <typename F, typename T, size_t N>
+[[gnu::noinline]] static LIBC_INLINE uint64_t throughput(
+    F f, const cpp::array<T, N> &inputs1, const cpp::array<T, N> &inputs2) {
+  asm("" ::"r"(&inputs1), "r"(&inputs2));
+
+  gpu::memory_fence();
+  uint64_t start = gpu::processor_clock();
+
+  asm("" ::"llr"(start));
+
+  uint64_t result;
+  for (size_t i = 0; i < inputs1.size(); i++) {
+    result = f(inputs1[i], inputs2[i]);
+    asm("" ::"r"(result));
+  }
+
+  uint64_t stop = gpu::processor_clock();
+  gpu::memory_fence();
+  asm("" ::"r"(stop));
+  volatile auto output = result;
+
+  // Return the time elapsed.
+  return stop - start;
+}
 } // namespace LIBC_NAMESPACE_DECL
 
 #endif // LLVM_LIBC_UTILS_GPU_TIMING_NVPTX

From 79d021116d8434d28e179660d31f4bd6cdbccd62 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Sun, 18 Aug 2024 11:14:59 -0700
Subject: [PATCH 309/441] [MC] Add test that requires multiple relaxation steps

This can catch the code size regression
due to the reverted be5a845e4c29aadb513ae6e5e2879dccf37efdbb
("[MC] Compute fragment offsets eagerly").
---
 llvm/test/MC/ELF/relax-branch.s | 398 ++++++++++++++++++++++++++++++++
 1 file changed, 398 insertions(+)
 create mode 100644 llvm/test/MC/ELF/relax-branch.s

diff --git a/llvm/test/MC/ELF/relax-branch.s b/llvm/test/MC/ELF/relax-branch.s
new file mode 100644
index 000000000000000..5abf8355f987627
--- /dev/null
+++ b/llvm/test/MC/ELF/relax-branch.s
@@ -0,0 +1,398 @@
+## A larger branch relaxation test that requires multiple relaxation steps.
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t
+# RUN: llvm-objdump -d %t | FileCheck %s
+
+## Regression test that we use jump short instead of jump near
+## (see the reverted be5a845e4c29aadb513ae6e5e2879dccf37efdbb).
+# CHECK:      4a9: eb 00                         jmp     0x4ab
+# CHECK:      5a3: eb 00                         jmp     0x5a5
+# CHECK-NEXT: 5a5: eb 00                         jmp     0x5a7
+# CHECK-NEXT: 5a7: eb 00                         jmp     0x5a9
+# CHECK-NEXT: 5a9: eb 00                         jmp     0x5ab
+# CHECK-NEXT: 5ab: e9 fb fe ff ff                jmp     0x4ab
+# CHECK-NEXT: 5b0: eb 00                         jmp     0x5b2
+
+	pushq	%rbp
+	movq	%rsp, %rbp
+	movq	%rcx, (%rbp)
+	movq	%r8, (%rbp)
+	movl	$0, (%rbp)
+	movl	$0, (%rbp)
+	movl	$0, (%rbp)
+	cmpq	$0, (%rbp)
+	je	.LBB0_2
+	movl	$0, (%rbp)
+	jmp	.LBB0_30
+.LBB0_2:
+	jmp	.LBB0_3
+.LBB0_3:
+	movq	(%rbp), %rax
+	movsbl	(%rax), %ecx
+	xorl	%eax, %eax
+	cmpl	%ecx, %eax
+	jne	.LBB0_29
+	jmp	.LBB0_5
+.LBB0_5:
+	movb	$0, %al
+	callq	sqlite3_stepPLT
+	movl	%eax, (%rbp)
+	cmpq	$0, (%rbp)
+	je	.LBB0_23
+	movl	$100, %eax
+	cmpl	(%rbp), %eax
+	sete	%al
+	andb	$1, %al
+	movzbl	%al, %eax
+	cmpl	(%rbp), %eax
+	jne	.LBB0_23
+	cmpl	$0, (%rbp)
+	je	.LBB0_23
+	movq	(%rbp), %rax
+	movl	(%rax), %eax
+	andl	$256, %eax
+	cmpl	$0, %eax
+	je	.LBB0_23
+	cmpl	$0, (%rbp)
+	je	.LBB0_22
+	movl	$0, (%rbp)
+.LBB0_11:
+	cmpl	$0, (%rbp)
+	je	.LBB0_13
+	movl	(%rbp), %esi
+	xorl	%edi, %edi
+	movb	$0, %al
+	callq	sqlite3_free_tablePLT
+	jmp	.LBB0_11
+.LBB0_13:
+	movq	(%rbp), %rax
+	movslq	(%rbp), %rcx
+	cmpb	$0, (%rax,%rcx)
+	je	.LBB0_15
+	movq	(%rbp), %rax
+	movb	$1, 8(%rax)
+.LBB0_15:
+	movq	(%rbp), %rax
+	movslq	(%rbp), %rcx
+	addq	%rcx, %rax
+	movq	%rax, (%rip)
+	movl	$0, (%rbp)
+.LBB0_16:
+	movl	(%rbp), %eax
+	cmpl	(%rbp), %eax
+	jge	.LBB0_21
+	movl	$0, (%rbp)
+	movl	$0, (%rbp)
+	movq	(%rbp), %rdi
+	movb	$0, %al
+	callq	sqlite3_free_tablePLT
+	movq	(%rip), %rax
+	movslq	(%rbp), %rcx
+	movl	$5, %edx
+	cmpq	%rdx, (%rax,%rcx,)
+	je	.LBB0_19
+	movq	(%rbp), %rax
+	movb	$1, 8(%rax)
+	jmp	.LBB0_30
+.LBB0_19:
+	jmp	.LBB0_20
+.LBB0_20:
+	movl	(%rbp), %eax
+	addl	$1, %eax
+	movl	%eax, (%rbp)
+	jmp	.LBB0_16
+.LBB0_21:
+	jmp	.LBB0_22
+.LBB0_22:
+	jmp	.LBB0_23
+.LBB0_23:
+	cmpl	$0, (%rbp)
+	je	.LBB0_27
+	movb	$0, %al
+	callq	sqlite3_stepPLT
+	movl	%eax, (%rbp)
+	movl	$0, (%rbp)
+	cmpl	$0, (%rbp)
+	je	.LBB0_26
+	jmp	.LBB0_28
+.LBB0_26:
+	jmp	.LBB0_27
+.LBB0_27:
+	jmp	.LBB0_5
+.LBB0_28:
+	jmp	.LBB0_3
+.LBB0_29:
+	jmp	.LBB0_30
+.LBB0_30:
+	movl	(%rbp), %eax
+	addq	$96, %rsp
+	popq	%rbp
+	retq
+	.p2align	4, 0x90
+	movq	%rsp, %rbp
+	subq	$48, %rsp
+	movq	%rdi, (%rbp)
+	movl	%esi, (%rbp)
+	movl	%edx, (%rbp)
+	cmpq	$0, sqlite3_get_table_res_00
+	movq	(%rbp), %rax
+	jmp	.LBB1_8
+	movq	$0, (%rbp)
+	movq	(%rbp), %rdi
+	leaq	sqlite3_get_table_zSql0, %rsi
+	leaq	sqlite3_get_table_cb0, %rdx
+	leaq	sqlite3_get_table_res0, %rcx
+	leaq	(%rip), %r8
+	callq	sqlite3_exec
+	movl	%eax, (%rip)
+	movslq	(%rbp), %rax
+	movq	%rax, (%rbp)
+	movl	(%rip), %eax
+	andl	$5, %eax
+	cmpl	$0, %eax
+	je	.LBB1_4
+	movl	(%rip), %ecx
+	movq	(%rbp), %rax
+	movl	%ecx, (%rax)
+	movl	(rip), %eax
+	movl	%eax, (%rbp)
+	jmp	.LBB1_8
+.LBB1_4:
+	movsbl	(%rip), %edi
+	movb	$0, %al
+	callq	sqlite3_free_tablePLT
+	cmpl	$0, (%rip)
+	je	.LBB1_6
+	movq	(%rip), %rax
+	movsbl	(%rax), %edi
+	movb	$0, %al
+	callq	sqlite3_free_tablePLT
+.LBB1_6:
+	movq	(rip), %rdi
+	movslq	(%rbp), %rsi
+	shlq	$0, %rsi
+	addq	$1, %rsi
+	movb	$0, %al
+	callq	sqlite3_mallocPLT
+	movq	%rax, (%rbp)
+	cmpq	$0, (%rbp)
+	je	.LBB1_8
+	movl	$1, (%rbp)
+.LBB1_8:
+	movl	(%rbp), %eax
+	addq	$48, %rsp
+	.p2align	4, 0x90
+	movq	(%rbp), %rax
+	movl	28(%rax), %eax
+	movq	(%rbp), %rcx
+	addl	16(%rcx), %eax
+	cmpl	$0, %eax
+	movq	(%rbp), %rax
+	movl	16(%rax), %ecx
+	addl	(%rbp), %ecx
+	addl	$1, %ecx
+	movq	(%rbp), %rax
+	movl	%ecx, 16(%rax)
+	jmp	.LBB2_14
+	movq	(%rbp), %rax
+	cmpl	$0, 20(%rax)
+	je	.LBB2_10
+	movl	$0, (%rbp)
+	movq	(%rbp), %rax
+	movl	$0, 24(%rax)
+.LBB2_4:
+	movl	(%rbp), %eax
+	cmpl	(%rbp), %eax
+	jge	.LBB2_7
+	movq	(%rbp), %rax
+	movslq	(%rbp), %rcx
+	movsbl	(%rax,%rcx), %esi
+	leaq	.str, %rdi
+	movb	$0, %al
+	callq	sqlite3_mprintf@PLT
+	movq	%rax, -64(%rbp)
+	movl	-56(%rbp), %eax
+	addl	$1, %eax
+	movl	%eax, -56(%rbp)
+	jmp	.LBB2_4
+.LBB2_7:
+	cmpq	$0, -64(%rbp)
+	je	.LBB2_9
+	jmp	.LBB2_9
+.LBB2_9:
+	movq	-64(%rbp), %rdx
+	movq	-48(%rbp), %rax
+	movq	(%rax), %rax
+	movq	-48(%rbp), %rsi
+	movl	28(%rsi), %ecx
+	movl	%ecx, %edi
+	addl	$1, %edi
+	movl	%edi, 28(%rsi)
+	movslq	%ecx, %rcx
+	movq	%rdx, (%rax,%rcx,8)
+	jmp	.LBB2_13
+.LBB2_10:
+	movq	-48(%rbp), %rax
+	movslq	-20(%rbp), %rcx
+	cmpq	%rcx, %rax
+	je	.LBB2_12
+	movq	-48(%rbp), %rax
+	movq	8(%rax), %rdi
+	movb	$0, %al
+	callq	sqlite3_free_table@PLT
+	leaq	.L.str, %rdi
+	movb	$0, %al
+	callq	sqlite3_mprintf@PLT
+	movq	%rax, %rcx
+	movq	-48(%rbp), %rax
+	movq	%rcx, 8(%rax)
+	movq	-48(%rbp), %rax
+	movl	$1, 32(%rax)
+	movl	$1, -4(%rbp)
+	jmp	.LBB2_15
+.LBB2_12:
+	jmp	.LBB2_13
+.LBB2_13:
+	movl	$0, -4(%rbp)
+	jmp	.LBB2_15
+.LBB2_14:
+.LBB2_15:
+	movl	-4(%rbp), %eax
+	addq	$64, %rsp
+	popq	%rbp
+	.p2align	4, 0x90
+sqlite3_complete:
+	je	.LBB3_54
+	je	.LBB3_25
+	je	.LBB3_25
+	je	.LBB3_25
+	jmp	.LBB3_28
+	movl	$42, %eax
+	cmpq	%rax, -160
+	je	.LBB3_5
+	jmp	.LBB3_53
+.LBB3_5:
+	jmp	.LBB3_6
+.LBB3_6:
+	movq	-160, %rax
+	movsbl	0, %ecx
+	xorl	%eax, %eax
+	movb	$1, %al
+	movl	$42, %ecx
+	cmpq	%rcx, -160
+	movb	%al, -340
+	jne	.LBB3_9
+	movq	-160, %rax
+	movsbl	10, %eax
+	cmpl	$0, %eax
+	setne	%al
+	movb	%al, -340
+.LBB3_9:
+	movb	-340, %al
+	movb	%al, -330
+	movb	-330, %al
+	testb	$1, %al
+	jne	.LBB3_11
+	jmp	.LBB3_14
+.LBB3_11:
+	movq	-160, %rax
+	movsbl	0, %eax
+	cmpl	$0, %eax
+	jne	.LBB3_13
+	movl	$0, -40
+	jmp	.LBB3_54
+.LBB3_13:
+	jmp	.LBB3_6
+.LBB3_14:
+	jmp	.LBB3_53
+	movq	-160, %rax
+	movsbl	10, %eax
+	cmpl	$45, %eax
+	je	.LBB3_17
+	movb	$2, -170
+	jmp	.LBB3_53
+.LBB3_17:
+	jmp	.LBB3_18
+.LBB3_18:
+	movq	-160, %rax
+	movsbl	0, %ecx
+	xorl	%eax, %eax
+	cmpl	$0, %ecx
+	movb	%al, -350
+	jmp	.LBB3_53
+.LBB3_25:
+	movl	$0, -40
+	jmp	.LBB3_54
+	jmp	.LBB3_53
+.LBB3_28:
+	je	.LBB3_52
+	jmp	.LBB3_52
+	jmp	.LBB3_31
+.LBB3_31:
+	movq	-160, %rax
+	movslq	-280, %rcx
+	movsbl	0, %ecx
+	movb	-360, %al
+	andb	$1, %cl
+	movzbl	%cl, %ecx
+	movl	%ecx, -240
+	testb	$1, %al
+	jne	.LBB3_36
+	jmp	.LBB3_51
+.LBB3_36:
+	movq	-160, %rax
+	movsbl	0, %eax
+	orl	$32, %eax
+	cmpl	$0, -280
+	je	.LBB3_40
+	movq	-160, %rdi
+	leaq	.L.str, %rsi
+	movl	$7, %edx
+	movb	$0, %al
+	callq	sqlite3StrNICmp@PLT
+	cmpl	$0, %eax
+	je	.LBB3_40
+	jmp	.LBB3_48
+.LBB3_40:
+	cmpl	$0, -280
+	je	.LBB3_43
+	movq	-160, %rdi
+	leaq	.L.str, %rsi
+	movl	$4, %edx
+	movb	$0, %al
+	callq	sqlite3StrNICmp@PLT
+	cmpl	$0, %eax
+	je	.LBB3_43
+	movb	$5, -170
+	jmp	.LBB3_47
+.LBB3_43:
+	cmpl	$0, -280
+	je	.LBB3_46
+	movq	-160, %rdi
+	leaq	.L.str, %rsi
+	movl	$9, %edx
+	movb	$0, %al
+	callq	sqlite3StrNICmp@PLT
+	cmpl	$0, %eax
+	je	.LBB3_46
+	movl	-280, %ecx
+	subl	$1, %ecx
+	movq	-160, %rax
+	movslq	%ecx, %rcx
+	addq	%rcx, %rax
+	movq	%rax, -160
+.LBB3_46:
+	jmp	.LBB3_47
+.LBB3_47:
+	jmp	.LBB3_48
+.LBB3_48:
+	jmp	.LBB3_49
+.LBB3_49:
+	jmp	.LBB3_50
+.LBB3_50:
+	jmp	.LBB3_31
+.LBB3_51:
+	jmp	.LBB3_52
+.LBB3_52:
+.LBB3_53:
+.LBB3_54:
+.L.str:

From bf5cd4220d20d0ee5533d55f463612fbe2980071 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Sun, 18 Aug 2024 11:24:44 -0700
Subject: [PATCH 310/441] [MIPS] Remove expensive LLVM_DEBUG relocation dump

The input is usually ordered by offset, so inspecting the output is
sufficient. The super expensive relocation dump is not conventional.
---
 .../Mips/MCTargetDesc/MipsELFObjectWriter.cpp | 32 ++-----------------
 1 file changed, 2 insertions(+), 30 deletions(-)

diff --git a/llvm/lib/Target/Mips/MCTargetDesc/MipsELFObjectWriter.cpp b/llvm/lib/Target/Mips/MCTargetDesc/MipsELFObjectWriter.cpp
index 4d6a00c14a3575c..1f047020d96c80e 100644
--- a/llvm/lib/Target/Mips/MCTargetDesc/MipsELFObjectWriter.cpp
+++ b/llvm/lib/Target/Mips/MCTargetDesc/MipsELFObjectWriter.cpp
@@ -47,13 +47,6 @@ struct MipsRelocationEntry {
   }
 };
 
-#ifndef NDEBUG
-raw_ostream &operator<<(raw_ostream &OS, const MipsRelocationEntry &RHS) {
-  RHS.print(OS);
-  return OS;
-}
-#endif
-
 class MipsELFObjectWriter : public MCELFObjectTargetWriter {
 public:
   MipsELFObjectWriter(uint8_t OSABI, bool HasRelocationAddend, bool Is64);
@@ -115,17 +108,11 @@ static InputIt find_best(InputIt First, InputIt Last, UnaryPredicate Predicate,
   for (InputIt I = First; I != Last; ++I) {
     unsigned Matched = Predicate(*I);
     if (Matched != FindBest_NoMatch) {
-      LLVM_DEBUG(dbgs() << std::distance(First, I) << " is a match (";
-                 I->print(dbgs()); dbgs() << ")\n");
-      if (Best == Last || BetterThan(*I, *Best)) {
-        LLVM_DEBUG(dbgs() << ".. and it beats the last one\n");
+      if (Best == Last || BetterThan(*I, *Best))
         Best = I;
-      }
     }
-    if (Matched == FindBest_PerfectMatch) {
-      LLVM_DEBUG(dbgs() << ".. and it is unbeatable\n");
+    if (Matched == FindBest_PerfectMatch)
       break;
-    }
   }
 
   return Best;
@@ -201,15 +188,6 @@ static bool compareMatchingRelocs(const MipsRelocationEntry &Candidate,
   return PreviousBest.Matched && !Candidate.Matched;
 }
 
-#ifndef NDEBUG
-/// Print all the relocations.
-template <class Container>
-static void dumpRelocs(const char *Prefix, const Container &Relocs) {
-  for (const auto &R : Relocs)
-    dbgs() << Prefix << R << "\n";
-}
-#endif
-
 MipsELFObjectWriter::MipsELFObjectWriter(uint8_t OSABI,
                                          bool HasRelocationAddend, bool Is64)
     : MCELFObjectTargetWriter(Is64, OSABI, ELF::EM_MIPS, HasRelocationAddend) {}
@@ -448,8 +426,6 @@ void MipsELFObjectWriter::sortRelocs(const MCAssembler &Asm,
   std::list<MipsRelocationEntry> Sorted;
   std::list<ELFRelocationEntry> Remainder;
 
-  LLVM_DEBUG(dumpRelocs("R: ", Relocs));
-
   // Separate the movable relocations (AHL relocations using the high bits) from
   // the immobile relocations (everything else). This does not preserve high/low
   // matches that already existed in the input.
@@ -459,8 +435,6 @@ void MipsELFObjectWriter::sortRelocs(const MCAssembler &Asm,
                });
 
   for (auto &R : Remainder) {
-    LLVM_DEBUG(dbgs() << "Matching: " << R << "\n");
-
     unsigned MatchingType = getMatchingLoType(R);
     assert(MatchingType != ELF::R_MIPS_NONE &&
            "Wrong list for reloc that doesn't need a match");
@@ -494,8 +468,6 @@ void MipsELFObjectWriter::sortRelocs(const MCAssembler &Asm,
     Sorted.insert(InsertionPoint, R)->Matched = true;
   }
 
-  LLVM_DEBUG(dumpRelocs("S: ", Sorted));
-
   assert(Relocs.size() == Sorted.size() && "Some relocs were not consumed");
 
   // Overwrite the original vector with the sorted elements.

From 3496245ed3d0b4d24444260da77dcdb93512fb5a Mon Sep 17 00:00:00 2001
From: Owen Pan <owenpiano@gmail.com>
Date: Sun, 18 Aug 2024 12:02:15 -0700
Subject: [PATCH 311/441] [clang-format] Change GNU style language standard to
 LS_Latest (#104669)

Fixes #104655.
---
 clang/lib/Format/Format.cpp                   | 1 -
 clang/unittests/Format/TokenAnnotatorTest.cpp | 9 +++++++++
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/clang/lib/Format/Format.cpp b/clang/lib/Format/Format.cpp
index a795d869a3cb66f..97fac41cdd30088 100644
--- a/clang/lib/Format/Format.cpp
+++ b/clang/lib/Format/Format.cpp
@@ -1905,7 +1905,6 @@ FormatStyle getGNUStyle() {
   Style.Cpp11BracedListStyle = false;
   Style.FixNamespaceComments = false;
   Style.SpaceBeforeParens = FormatStyle::SBPO_Always;
-  Style.Standard = FormatStyle::LS_Cpp03;
   return Style;
 }
 
diff --git a/clang/unittests/Format/TokenAnnotatorTest.cpp b/clang/unittests/Format/TokenAnnotatorTest.cpp
index 256bbd49c424fa7..99798de43e70ffe 100644
--- a/clang/unittests/Format/TokenAnnotatorTest.cpp
+++ b/clang/unittests/Format/TokenAnnotatorTest.cpp
@@ -3360,6 +3360,15 @@ TEST_F(TokenAnnotatorTest, TypenameMacro) {
   EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_Unknown);
 }
 
+TEST_F(TokenAnnotatorTest, GNULanguageStandard) {
+  auto Style = getGNUStyle();
+  EXPECT_EQ(Style.Standard, FormatStyle::LS_Latest);
+
+  auto Tokens = annotate("return 1 <=> 2;", Style);
+  ASSERT_EQ(Tokens.size(), 6u);
+  EXPECT_TOKEN(Tokens[2], tok::spaceship, TT_BinaryOperator);
+}
+
 } // namespace
 } // namespace format
 } // namespace clang

From 0224d8355d84b3eee2390cd13bb0b749ba40b0db Mon Sep 17 00:00:00 2001
From: Frederic Cambus <fred@statdns.com>
Date: Sun, 18 Aug 2024 23:28:56 +0200
Subject: [PATCH 312/441] Update Clang version from 19 to 20 in scan-build.1.

Similar to D110763.
---
 clang/tools/scan-build/man/scan-build.1 | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/clang/tools/scan-build/man/scan-build.1 b/clang/tools/scan-build/man/scan-build.1
index e2b37f6062dbc0b..b29e9ca91477f65 100644
--- a/clang/tools/scan-build/man/scan-build.1
+++ b/clang/tools/scan-build/man/scan-build.1
@@ -2,9 +2,9 @@
 .\" See https://llvm.org/LICENSE.txt for license information.
 .\" SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 .\" $Id$
-.Dd Feb 10, 2024
+.Dd Aug 18, 2024
 .Dt SCAN-BUILD 1
-.Os "clang" "19"
+.Os "clang" "20"
 .Sh NAME
 .Nm scan-build
 .Nd Clang static analyzer

From 70f3863b5f30e856278f399b068a30bc4d5d16c2 Mon Sep 17 00:00:00 2001
From: Noah Goldstein <goldstein.w.n@gmail.com>
Date: Tue, 13 Aug 2024 21:18:45 +0800
Subject: [PATCH 313/441] [DAG][PatternMatch] Add support for matchers with
 flags; NFC

Add support for matching with `SDNodeFlags` i.e `add` with `nuw`.

This patch adds helpers for `or disjoint` or `zext nneg` with the same
names as we have in IR/PatternMatch api.

Closes #103060
---
 llvm/include/llvm/CodeGen/SDPatternMatch.h    | 62 ++++++++++++++++---
 llvm/include/llvm/CodeGen/SelectionDAGNodes.h | 14 +++++
 .../CodeGen/SelectionDAGPatternMatchTest.cpp  | 25 +++++++-
 3 files changed, 90 insertions(+), 11 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/SDPatternMatch.h b/llvm/include/llvm/CodeGen/SDPatternMatch.h
index b1aa87ca2d3e134..92efff93f60f89b 100644
--- a/llvm/include/llvm/CodeGen/SDPatternMatch.h
+++ b/llvm/include/llvm/CodeGen/SDPatternMatch.h
@@ -514,19 +514,28 @@ struct BinaryOpc_match {
   unsigned Opcode;
   LHS_P LHS;
   RHS_P RHS;
-
-  BinaryOpc_match(unsigned Opc, const LHS_P &L, const RHS_P &R)
-      : Opcode(Opc), LHS(L), RHS(R) {}
+  std::optional<SDNodeFlags> Flags;
+  BinaryOpc_match(unsigned Opc, const LHS_P &L, const RHS_P &R,
+                  std::optional<SDNodeFlags> Flgs = std::nullopt)
+      : Opcode(Opc), LHS(L), RHS(R), Flags(Flgs) {}
 
   template <typename MatchContext>
   bool match(const MatchContext &Ctx, SDValue N) {
     if (sd_context_match(N, Ctx, m_Opc(Opcode))) {
       EffectiveOperands<ExcludeChain> EO(N, Ctx);
       assert(EO.Size == 2);
-      return (LHS.match(Ctx, N->getOperand(EO.FirstIndex)) &&
-              RHS.match(Ctx, N->getOperand(EO.FirstIndex + 1))) ||
-             (Commutable && LHS.match(Ctx, N->getOperand(EO.FirstIndex + 1)) &&
-              RHS.match(Ctx, N->getOperand(EO.FirstIndex)));
+      if (!((LHS.match(Ctx, N->getOperand(EO.FirstIndex)) &&
+             RHS.match(Ctx, N->getOperand(EO.FirstIndex + 1))) ||
+            (Commutable && LHS.match(Ctx, N->getOperand(EO.FirstIndex + 1)) &&
+             RHS.match(Ctx, N->getOperand(EO.FirstIndex)))))
+        return false;
+
+      if (!Flags.has_value())
+        return true;
+
+      SDNodeFlags TmpFlags = *Flags;
+      TmpFlags.intersectWith(N->getFlags());
+      return TmpFlags == *Flags;
     }
 
     return false;
@@ -581,6 +590,19 @@ inline BinaryOpc_match<LHS, RHS, true> m_Or(const LHS &L, const RHS &R) {
   return BinaryOpc_match<LHS, RHS, true>(ISD::OR, L, R);
 }
 
+template <typename LHS, typename RHS>
+inline BinaryOpc_match<LHS, RHS, true> m_DisjointOr(const LHS &L,
+                                                    const RHS &R) {
+  SDNodeFlags Flags;
+  Flags.setDisjoint(true);
+  return BinaryOpc_match<LHS, RHS, true>(ISD::OR, L, R, Flags);
+}
+
+template <typename LHS, typename RHS>
+inline auto m_AddLike(const LHS &L, const RHS &R) {
+  return m_AnyOf(m_Add(L, R), m_DisjointOr(L, R));
+}
+
 template <typename LHS, typename RHS>
 inline BinaryOpc_match<LHS, RHS, true> m_Xor(const LHS &L, const RHS &R) {
   return BinaryOpc_match<LHS, RHS, true>(ISD::XOR, L, R);
@@ -667,15 +689,24 @@ inline BinaryOpc_match<LHS, RHS> m_FRem(const LHS &L, const RHS &R) {
 template <typename Opnd_P, bool ExcludeChain = false> struct UnaryOpc_match {
   unsigned Opcode;
   Opnd_P Opnd;
-
-  UnaryOpc_match(unsigned Opc, const Opnd_P &Op) : Opcode(Opc), Opnd(Op) {}
+  std::optional<SDNodeFlags> Flags;
+  UnaryOpc_match(unsigned Opc, const Opnd_P &Op,
+                 std::optional<SDNodeFlags> Flgs = std::nullopt)
+      : Opcode(Opc), Opnd(Op), Flags(Flgs) {}
 
   template <typename MatchContext>
   bool match(const MatchContext &Ctx, SDValue N) {
     if (sd_context_match(N, Ctx, m_Opc(Opcode))) {
       EffectiveOperands<ExcludeChain> EO(N, Ctx);
       assert(EO.Size == 1);
-      return Opnd.match(Ctx, N->getOperand(EO.FirstIndex));
+      if (!Opnd.match(Ctx, N->getOperand(EO.FirstIndex)))
+        return false;
+      if (!Flags.has_value())
+        return true;
+
+      SDNodeFlags TmpFlags = *Flags;
+      TmpFlags.intersectWith(N->getFlags());
+      return TmpFlags == *Flags;
     }
 
     return false;
@@ -701,6 +732,13 @@ template <typename Opnd> inline UnaryOpc_match<Opnd> m_ZExt(const Opnd &Op) {
   return UnaryOpc_match<Opnd>(ISD::ZERO_EXTEND, Op);
 }
 
+template <typename Opnd>
+inline UnaryOpc_match<Opnd> m_NNegZExt(const Opnd &Op) {
+  SDNodeFlags Flags;
+  Flags.setNonNeg(true);
+  return UnaryOpc_match<Opnd>(ISD::ZERO_EXTEND, Op, Flags);
+}
+
 template <typename Opnd> inline auto m_SExt(const Opnd &Op) {
   return UnaryOpc_match<Opnd>(ISD::SIGN_EXTEND, Op);
 }
@@ -725,6 +763,10 @@ template <typename Opnd> inline auto m_SExtOrSelf(const Opnd &Op) {
   return m_AnyOf(m_SExt(Op), Op);
 }
 
+template <typename Opnd> inline auto m_SExtLike(const Opnd &Op) {
+  return m_AnyOf(m_SExt(Op), m_NNegZExt(Op));
+}
+
 /// Match a aext or identity
 /// Allows to peek through optional extensions
 template <typename Opnd>
diff --git a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
index 2f36c2e86b1c3a1..88549d9c9a28580 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
@@ -452,6 +452,20 @@ struct SDNodeFlags {
   bool hasNoFPExcept() const { return NoFPExcept; }
   bool hasUnpredictable() const { return Unpredictable; }
 
+  bool operator==(const SDNodeFlags &Other) const {
+    return NoUnsignedWrap == Other.NoUnsignedWrap &&
+           NoSignedWrap == Other.NoSignedWrap && Exact == Other.Exact &&
+           Disjoint == Other.Disjoint && NonNeg == Other.NonNeg &&
+           NoNaNs == Other.NoNaNs && NoInfs == Other.NoInfs &&
+           NoSignedZeros == Other.NoSignedZeros &&
+           AllowReciprocal == Other.AllowReciprocal &&
+           AllowContract == Other.AllowContract &&
+           ApproximateFuncs == Other.ApproximateFuncs &&
+           AllowReassociation == Other.AllowReassociation &&
+           NoFPExcept == Other.NoFPExcept &&
+           Unpredictable == Other.Unpredictable;
+  }
+
   /// Clear any flags in this flag set that aren't also set in Flags. All
   /// flags will be cleared if Flags are undefined.
   void intersectWith(const SDNodeFlags Flags) {
diff --git a/llvm/unittests/CodeGen/SelectionDAGPatternMatchTest.cpp b/llvm/unittests/CodeGen/SelectionDAGPatternMatchTest.cpp
index c04fc5621ab4992..e66584b81bba256 100644
--- a/llvm/unittests/CodeGen/SelectionDAGPatternMatchTest.cpp
+++ b/llvm/unittests/CodeGen/SelectionDAGPatternMatchTest.cpp
@@ -185,6 +185,7 @@ TEST_F(SelectionDAGPatternMatchTest, matchBinaryOp) {
   SDValue Op0 = DAG->getCopyFromReg(DAG->getEntryNode(), DL, 1, Int32VT);
   SDValue Op1 = DAG->getCopyFromReg(DAG->getEntryNode(), DL, 2, Int32VT);
   SDValue Op2 = DAG->getCopyFromReg(DAG->getEntryNode(), DL, 3, Float32VT);
+  SDValue Op3 = DAG->getCopyFromReg(DAG->getEntryNode(), DL, 8, Int32VT);
 
   SDValue Add = DAG->getNode(ISD::ADD, DL, Int32VT, Op0, Op1);
   SDValue Sub = DAG->getNode(ISD::SUB, DL, Int32VT, Add, Op0);
@@ -192,6 +193,9 @@ TEST_F(SelectionDAGPatternMatchTest, matchBinaryOp) {
   SDValue And = DAG->getNode(ISD::AND, DL, Int32VT, Op0, Op1);
   SDValue Xor = DAG->getNode(ISD::XOR, DL, Int32VT, Op1, Op0);
   SDValue Or  = DAG->getNode(ISD::OR, DL, Int32VT, Op0, Op1);
+  SDNodeFlags DisFlags;
+  DisFlags.setDisjoint(true);
+  SDValue DisOr = DAG->getNode(ISD::OR, DL, Int32VT, Op0, Op3, DisFlags);
   SDValue SMax = DAG->getNode(ISD::SMAX, DL, Int32VT, Op0, Op1);
   SDValue SMin = DAG->getNode(ISD::SMIN, DL, Int32VT, Op1, Op0);
   SDValue UMax = DAG->getNode(ISD::UMAX, DL, Int32VT, Op0, Op1);
@@ -205,6 +209,7 @@ TEST_F(SelectionDAGPatternMatchTest, matchBinaryOp) {
   EXPECT_TRUE(sd_match(Sub, m_Sub(m_Value(), m_Value())));
   EXPECT_TRUE(sd_match(Add, m_c_BinOp(ISD::ADD, m_Value(), m_Value())));
   EXPECT_TRUE(sd_match(Add, m_Add(m_Value(), m_Value())));
+  EXPECT_TRUE(sd_match(Add, m_AddLike(m_Value(), m_Value())));
   EXPECT_TRUE(sd_match(
       Mul, m_Mul(m_OneUse(m_Opc(ISD::SUB)), m_NUses<2>(m_Specific(Add)))));
   EXPECT_TRUE(
@@ -217,6 +222,12 @@ TEST_F(SelectionDAGPatternMatchTest, matchBinaryOp) {
   EXPECT_TRUE(sd_match(Xor, m_Xor(m_Value(), m_Value())));
   EXPECT_TRUE(sd_match(Or, m_c_BinOp(ISD::OR, m_Value(), m_Value())));
   EXPECT_TRUE(sd_match(Or, m_Or(m_Value(), m_Value())));
+  EXPECT_FALSE(sd_match(Or, m_DisjointOr(m_Value(), m_Value())));
+
+  EXPECT_TRUE(sd_match(DisOr, m_Or(m_Value(), m_Value())));
+  EXPECT_TRUE(sd_match(DisOr, m_DisjointOr(m_Value(), m_Value())));
+  EXPECT_FALSE(sd_match(DisOr, m_Add(m_Value(), m_Value())));
+  EXPECT_TRUE(sd_match(DisOr, m_AddLike(m_Value(), m_Value())));
 
   EXPECT_TRUE(sd_match(SMax, m_c_BinOp(ISD::SMAX, m_Value(), m_Value())));
   EXPECT_TRUE(sd_match(SMax, m_SMax(m_Value(), m_Value())));
@@ -242,9 +253,14 @@ TEST_F(SelectionDAGPatternMatchTest, matchUnaryOp) {
 
   SDValue Op0 = DAG->getCopyFromReg(DAG->getEntryNode(), DL, 1, Int32VT);
   SDValue Op1 = DAG->getCopyFromReg(DAG->getEntryNode(), DL, 1, Int64VT);
-  SDValue Op2 = DAG->getCopyFromReg(DAG->getEntryNode(), DL, 1, FloatVT);
+  SDValue Op2 = DAG->getCopyFromReg(DAG->getEntryNode(), DL, 1, FloatVT);  
+  SDValue Op3 = DAG->getCopyFromReg(DAG->getEntryNode(), DL, 3, Int32VT);
 
   SDValue ZExt = DAG->getNode(ISD::ZERO_EXTEND, DL, Int64VT, Op0);
+  SDNodeFlags NNegFlags;
+  NNegFlags.setNonNeg(true);
+  SDValue ZExtNNeg =
+      DAG->getNode(ISD::ZERO_EXTEND, DL, Int64VT, Op3, NNegFlags);
   SDValue SExt = DAG->getNode(ISD::SIGN_EXTEND, DL, Int64VT, Op0);
   SDValue Trunc = DAG->getNode(ISD::TRUNCATE, DL, Int32VT, Op1);
 
@@ -260,6 +276,13 @@ TEST_F(SelectionDAGPatternMatchTest, matchUnaryOp) {
   using namespace SDPatternMatch;
   EXPECT_TRUE(sd_match(ZExt, m_UnaryOp(ISD::ZERO_EXTEND, m_Value())));
   EXPECT_TRUE(sd_match(SExt, m_SExt(m_Value())));
+  EXPECT_TRUE(sd_match(SExt, m_SExtLike(m_Value())));
+  ASSERT_TRUE(ZExtNNeg->getFlags().hasNonNeg());
+  EXPECT_FALSE(sd_match(ZExtNNeg, m_SExt(m_Value())));
+  EXPECT_TRUE(sd_match(ZExtNNeg, m_NNegZExt(m_Value())));
+  EXPECT_FALSE(sd_match(ZExt, m_NNegZExt(m_Value())));
+  EXPECT_TRUE(sd_match(ZExtNNeg, m_SExtLike(m_Value())));
+  EXPECT_FALSE(sd_match(ZExt, m_SExtLike(m_Value())));
   EXPECT_TRUE(sd_match(Trunc, m_Trunc(m_Specific(Op1))));
 
   EXPECT_TRUE(sd_match(Neg, m_Neg(m_Value())));

From c6e16a49ef41261b01aabc27f4b806af6cc81a20 Mon Sep 17 00:00:00 2001
From: Noah Goldstein <goldstein.w.n@gmail.com>
Date: Fri, 2 Aug 2024 02:10:31 +0800
Subject: [PATCH 314/441] [TLI] Add support for inferring attr
 `cold`/`noreturn` on `std::terminate` and `__cxa_throw`

These functions are both inherently on the error path so `cold` seems
appropriate. `noreturn` is definitional.

Closes #101622
---
 .../llvm/Analysis/TargetLibraryInfo.def        | 10 ++++++++++
 llvm/lib/Transforms/Utils/BuildLibCalls.cpp    | 18 ++++++++++++++++++
 .../Transforms/InferFunctionAttrs/annotate.ll  |  8 ++++++++
 .../tools/llvm-tli-checker/ps4-tli-check.yaml  | 16 ++++++++++++----
 .../Analysis/TargetLibraryInfoTest.cpp         |  2 ++
 5 files changed, 50 insertions(+), 4 deletions(-)

diff --git a/llvm/include/llvm/Analysis/TargetLibraryInfo.def b/llvm/include/llvm/Analysis/TargetLibraryInfo.def
index e1cb1e5c557eae9..5914324b286c05d 100644
--- a/llvm/include/llvm/Analysis/TargetLibraryInfo.def
+++ b/llvm/include/llvm/Analysis/TargetLibraryInfo.def
@@ -517,6 +517,16 @@ TLI_DEFINE_ENUM_INTERNAL(Exit)
 TLI_DEFINE_STRING_INTERNAL("_Exit")
 TLI_DEFINE_SIG_INTERNAL(Void, Int)
 
+/// void std::terminate();
+TLI_DEFINE_ENUM_INTERNAL(terminate)
+TLI_DEFINE_STRING_INTERNAL("_ZSt9terminatev")
+TLI_DEFINE_SIG_INTERNAL(Void)
+
+/// void __cxa_throw(void *, void *, void (*)(void *));
+TLI_DEFINE_ENUM_INTERNAL(cxa_throw)
+TLI_DEFINE_STRING_INTERNAL("__cxa_throw")
+TLI_DEFINE_SIG_INTERNAL(Void, Ptr, Ptr, Ptr)
+
 /// void __cxa_guard_abort(guard_t *guard);
 /// guard_t is int64_t in Itanium ABI or int32_t on ARM eabi.
 TLI_DEFINE_ENUM_INTERNAL(cxa_guard_abort)
diff --git a/llvm/lib/Transforms/Utils/BuildLibCalls.cpp b/llvm/lib/Transforms/Utils/BuildLibCalls.cpp
index c23d53f7111d214..b0da19813f0a4b6 100644
--- a/llvm/lib/Transforms/Utils/BuildLibCalls.cpp
+++ b/llvm/lib/Transforms/Utils/BuildLibCalls.cpp
@@ -50,6 +50,7 @@ STATISTIC(NumNoUndef, "Number of function returns inferred as noundef returns");
 STATISTIC(NumReturnedArg, "Number of arguments inferred as returned");
 STATISTIC(NumWillReturn, "Number of functions inferred as willreturn");
 STATISTIC(NumCold, "Number of functions inferred as cold");
+STATISTIC(NumNoReturn, "Number of functions inferred as no return");
 
 static bool setDoesNotAccessMemory(Function &F) {
   if (F.doesNotAccessMemory())
@@ -67,6 +68,14 @@ static bool setIsCold(Function &F) {
   return true;
 }
 
+static bool setNoReturn(Function &F) {
+  if (F.hasFnAttribute(Attribute::NoReturn))
+    return false;
+  F.addFnAttr(Attribute::NoReturn);
+  ++NumNoReturn;
+  return true;
+}
+
 static bool setOnlyAccessesInaccessibleMemory(Function &F) {
   if (F.onlyAccessesInaccessibleMemory())
     return false;
@@ -1103,6 +1112,15 @@ bool llvm::inferNonMandatoryLibFuncAttrs(Function &F,
   case LibFunc_abort:
     Changed |= setIsCold(F);
     break;
+  case LibFunc_terminate:
+    Changed |= setIsCold(F);
+    Changed |= setNoReturn(F);
+    break;
+  case LibFunc_cxa_throw:
+    Changed |= setIsCold(F);
+    Changed |= setNoReturn(F);
+    // Don't add `nofree` on `__cxa_throw`
+    return Changed;
   // int __nvvm_reflect(const char *)
   case LibFunc_nvvm_reflect:
     Changed |= setRetAndArgsNoUndef(F);
diff --git a/llvm/test/Transforms/InferFunctionAttrs/annotate.ll b/llvm/test/Transforms/InferFunctionAttrs/annotate.ll
index 3b914dc29ca41c4..bc0d7a509e1f5d2 100644
--- a/llvm/test/Transforms/InferFunctionAttrs/annotate.ll
+++ b/llvm/test/Transforms/InferFunctionAttrs/annotate.ll
@@ -1100,6 +1100,12 @@ declare i64 @write(i32, ptr, i64)
 ; CHECK: declare void @abort() [[NOFREE_COLD:#[0-9]+]]
 declare void @abort()
 
+; CHECK: declare void @__cxa_throw(ptr, ptr, ptr) [[COLD_NORETURN:#[0-9]+]]
+declare void @__cxa_throw(ptr, ptr, ptr)
+
+; CHECK: declare void @_ZSt9terminatev() [[NOFREE_COLD_NORETURN:#[0-9]+]]
+declare void @_ZSt9terminatev()
+
 ; memset_pattern{4,8,16} aren't available everywhere.
 ; CHECK-DARWIN: declare void @memset_pattern4(ptr nocapture writeonly, ptr nocapture readonly, i64) [[ARGMEMONLY_NOFREE_NOUNWIND_WILLRETURN]]
 declare void @memset_pattern4(ptr, ptr, i64)
@@ -1126,6 +1132,8 @@ declare void @memset_pattern16(ptr, ptr, i64)
 ; CHECK-DAG: attributes [[INACCESSIBLEMEMORARGMEMONLY_NOUNWIND_WILLRETURN_ALLOCKIND_REALLOC_ALLOCSIZE1_FAMILY_MALLOC]] = { mustprogress nounwind willreturn allockind("realloc") allocsize(1) memory(argmem: readwrite, inaccessiblemem: readwrite) "alloc-family"="malloc" }
 ; CHECK-DAG: attributes [[INACCESSIBLEMEMORARGONLY_NOFREE_NOUNWIND_WILLRETURN_FAMILY_MALLOC]] = { mustprogress nofree nounwind willreturn memory(argmem: readwrite, inaccessiblemem: readwrite) "alloc-family"="malloc" }
 ; CHECK-DAG: attributes [[NOFREE_COLD]] = { cold nofree }
+; CHECK-DAG: attributes [[NOFREE_COLD_NORETURN]] = { cold nofree noreturn }
+; CHECK-DAG: attributes [[COLD_NORETURN]] = { cold noreturn }
 
 ; CHECK-NVPTX-DAG: attributes [[NOFREE_NOUNWIND_READNONE]] = { nofree nosync nounwind memory(none) }
 
diff --git a/llvm/test/tools/llvm-tli-checker/ps4-tli-check.yaml b/llvm/test/tools/llvm-tli-checker/ps4-tli-check.yaml
index cff5019f8e6ee4b..47aeb0ad8fdef9b 100644
--- a/llvm/test/tools/llvm-tli-checker/ps4-tli-check.yaml
+++ b/llvm/test/tools/llvm-tli-checker/ps4-tli-check.yaml
@@ -34,7 +34,7 @@
 #
 # CHECK: << Total TLI yes SDK no:  18
 # CHECK: >> Total TLI no  SDK yes: 0
-# CHECK: == Total TLI yes SDK yes: 248
+# CHECK: == Total TLI yes SDK yes: 250
 #
 # WRONG_DETAIL: << TLI yes SDK no : '_ZdaPv' aka operator delete[](void*)
 # WRONG_DETAIL: >> TLI no  SDK yes: '_ZdaPvj' aka operator delete[](void*, unsigned int)
@@ -48,14 +48,14 @@
 # WRONG_DETAIL: << TLI yes SDK no : 'fminimum_numl'
 # WRONG_SUMMARY: << Total TLI yes SDK no:  19{{$}}
 # WRONG_SUMMARY: >> Total TLI no  SDK yes: 1{{$}}
-# WRONG_SUMMARY: == Total TLI yes SDK yes: 247
+# WRONG_SUMMARY: == Total TLI yes SDK yes: 249
 #
 ## The -COUNT suffix doesn't care if there are too many matches, so check
 ## the exact count first; the two directives should add up to that.
 ## Yes, this means additions to TLI will fail this test, but the argument
 ## to -COUNT can't be an expression.
-# AVAIL: TLI knows 499 symbols, 266 available
-# AVAIL-COUNT-266: {{^}} available
+# AVAIL: TLI knows 501 symbols, 268 available
+# AVAIL-COUNT-268: {{^}} available
 # AVAIL-NOT:       {{^}} available
 # UNAVAIL-COUNT-233: not available
 # UNAVAIL-NOT:       not available
@@ -174,6 +174,10 @@ DynamicSymbols:
     Type:            STT_FUNC
     Section:         .text
     Binding:         STB_GLOBAL
+  - Name:            __cxa_throw
+    Type:            STT_FUNC
+    Section:         .text
+    Binding:         STB_GLOBAL
   - Name:            __cxa_guard_release
     Type:            STT_FUNC
     Section:         .text
@@ -286,6 +290,10 @@ DynamicSymbols:
     Type:            STT_FUNC
     Section:         .text
     Binding:         STB_GLOBAL
+  - Name:            _ZSt9terminatev
+    Type:            STT_FUNC
+    Section:         .text
+    Binding:         STB_GLOBAL
   - Name:            atof
     Type:            STT_FUNC
     Section:         .text
diff --git a/llvm/unittests/Analysis/TargetLibraryInfoTest.cpp b/llvm/unittests/Analysis/TargetLibraryInfoTest.cpp
index ff7dec5bee31dfb..abaedf9c210b8c4 100644
--- a/llvm/unittests/Analysis/TargetLibraryInfoTest.cpp
+++ b/llvm/unittests/Analysis/TargetLibraryInfoTest.cpp
@@ -535,12 +535,14 @@ TEST_F(TargetLibraryInfoTest, ValidProto) {
       "declare void @__cxa_guard_abort(%struct*)\n"
       "declare i32 @__cxa_guard_acquire(%struct*)\n"
       "declare void @__cxa_guard_release(%struct*)\n"
+      "declare void @__cxa_throw(ptr, ptr, ptr)\n"
 
       "declare i32 @atexit(void ()*)\n"
 
       "declare void @abort()\n"
       "declare void @exit(i32)\n"
       "declare void @_Exit(i32)\n"
+      "declare void @_ZSt9terminatev()\n"
 
       "declare i32 @__nvvm_reflect(i8*)\n"
 

From 1a6bf94407af7962345ffaf6ac9e9a9766fd8913 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Sun, 18 Aug 2024 15:47:38 -0700
Subject: [PATCH 315/441] [MC] Remove ELFRelocationEntry::OriginalAddend

For MIPS's o32 ABI (REL), https://reviews.llvm.org/D19718 introduced
`OriginalAddend` to find the matching R_MIPS_LO16 relocation for
R_MIPS_GOT16 when STT_SECTION conversion is applicable.

    lw $2, %lo(local1)
    lui $2, %got(local1)

However, we could just store the original `Addend` in
`ELFRelocationEntry` and remove `OriginalAddend`.

Note: The relocation ordering algorithm in
https://reviews.llvm.org/D19718 is inefficient (#104562), which will be
addressed by another patch.
---
 llvm/include/llvm/MC/MCELFObjectWriter.h        |  9 +++------
 llvm/lib/MC/ELFObjectWriter.cpp                 | 17 ++++++-----------
 .../Mips/MCTargetDesc/MipsELFObjectWriter.cpp   |  9 ++++-----
 3 files changed, 13 insertions(+), 22 deletions(-)

diff --git a/llvm/include/llvm/MC/MCELFObjectWriter.h b/llvm/include/llvm/MC/MCELFObjectWriter.h
index 3b418083eecceba..d17fc931d1561c1 100644
--- a/llvm/include/llvm/MC/MCELFObjectWriter.h
+++ b/llvm/include/llvm/MC/MCELFObjectWriter.h
@@ -38,18 +38,15 @@ struct ELFRelocationEntry {
   unsigned Type;   // The type of the relocation.
   uint64_t Addend; // The addend to use.
   const MCSymbolELF *OriginalSymbol; // The original value of Symbol if we changed it.
-  uint64_t OriginalAddend; // The original value of addend.
 
   ELFRelocationEntry(uint64_t Offset, const MCSymbolELF *Symbol, unsigned Type,
-                     uint64_t Addend, const MCSymbolELF *OriginalSymbol,
-                     uint64_t OriginalAddend)
+                     uint64_t Addend, const MCSymbolELF *OriginalSymbol)
       : Offset(Offset), Symbol(Symbol), Type(Type), Addend(Addend),
-        OriginalSymbol(OriginalSymbol), OriginalAddend(OriginalAddend) {}
+        OriginalSymbol(OriginalSymbol) {}
 
   void print(raw_ostream &Out) const {
     Out << "Off=" << Offset << ", Sym=" << Symbol << ", Type=" << Type
-        << ", Addend=" << Addend << ", OriginalSymbol=" << OriginalSymbol
-        << ", OriginalAddend=" << OriginalAddend;
+        << ", Addend=" << Addend << ", OriginalSymbol=" << OriginalSymbol;
   }
 
   LLVM_DUMP_METHOD void dump() const { print(errs()); }
diff --git a/llvm/lib/MC/ELFObjectWriter.cpp b/llvm/lib/MC/ELFObjectWriter.cpp
index 7a8a6c8daf95938..35d0a2eb52dfc76 100644
--- a/llvm/lib/MC/ELFObjectWriter.cpp
+++ b/llvm/lib/MC/ELFObjectWriter.cpp
@@ -1443,22 +1443,17 @@ void ELFObjectWriter::recordRelocation(MCAssembler &Asm,
   bool RelocateWithSymbol =
       shouldRelocateWithSymbol(Asm, Target, SymA, C, Type) ||
       (Parent->getType() == ELF::SHT_LLVM_CALL_GRAPH_PROFILE);
-  uint64_t Addend = 0;
-
-  FixedValue = !RelocateWithSymbol && SymA && !SymA->isUndefined()
-                   ? C + Asm.getSymbolOffset(*SymA)
-                   : C;
-  if (usesRela(TO, FixupSection)) {
-    Addend = FixedValue;
-    FixedValue = 0;
-  }
+  uint64_t Addend = !RelocateWithSymbol && SymA && !SymA->isUndefined()
+                        ? C + Asm.getSymbolOffset(*SymA)
+                        : C;
+  FixedValue = usesRela(TO, FixupSection) ? 0 : Addend;
 
   if (!RelocateWithSymbol) {
     const auto *SectionSymbol =
         SecA ? cast<MCSymbolELF>(SecA->getBeginSymbol()) : nullptr;
     if (SectionSymbol)
       SectionSymbol->setUsedInReloc();
-    ELFRelocationEntry Rec(FixupOffset, SectionSymbol, Type, Addend, SymA, C);
+    ELFRelocationEntry Rec(FixupOffset, SectionSymbol, Type, Addend, SymA);
     Relocations[&FixupSection].push_back(Rec);
     return;
   }
@@ -1473,7 +1468,7 @@ void ELFObjectWriter::recordRelocation(MCAssembler &Asm,
     else
       RenamedSymA->setUsedInReloc();
   }
-  ELFRelocationEntry Rec(FixupOffset, RenamedSymA, Type, Addend, SymA, C);
+  ELFRelocationEntry Rec(FixupOffset, RenamedSymA, Type, Addend, SymA);
   Relocations[&FixupSection].push_back(Rec);
 }
 
diff --git a/llvm/lib/Target/Mips/MCTargetDesc/MipsELFObjectWriter.cpp b/llvm/lib/Target/Mips/MCTargetDesc/MipsELFObjectWriter.cpp
index 1f047020d96c80e..19b6bf0da22a60a 100644
--- a/llvm/lib/Target/Mips/MCTargetDesc/MipsELFObjectWriter.cpp
+++ b/llvm/lib/Target/Mips/MCTargetDesc/MipsELFObjectWriter.cpp
@@ -166,10 +166,9 @@ static FindBestPredicateResult isMatchingReloc(const MipsRelocationEntry &X,
                                                const ELFRelocationEntry &R,
                                                unsigned MatchingType) {
   if (X.R.Type == MatchingType && X.R.OriginalSymbol == R.OriginalSymbol) {
-    if (!X.Matched &&
-        X.R.OriginalAddend == R.OriginalAddend)
+    if (!X.Matched && X.R.Addend == R.Addend)
       return FindBest_PerfectMatch;
-    else if (X.R.OriginalAddend >= R.OriginalAddend)
+    else if (X.R.Addend >= R.Addend)
       return FindBest_Match;
   }
   return FindBest_NoMatch;
@@ -183,8 +182,8 @@ static FindBestPredicateResult isMatchingReloc(const MipsRelocationEntry &X,
 /// - It is not already involved in a match.
 static bool compareMatchingRelocs(const MipsRelocationEntry &Candidate,
                                   const MipsRelocationEntry &PreviousBest) {
-  if (Candidate.R.OriginalAddend != PreviousBest.R.OriginalAddend)
-    return Candidate.R.OriginalAddend < PreviousBest.R.OriginalAddend;
+  if (Candidate.R.Addend != PreviousBest.R.Addend)
+    return Candidate.R.Addend < PreviousBest.R.Addend;
   return PreviousBest.Matched && !Candidate.Matched;
 }
 

From f16125a13ce725b1e936468e08257c0fbb80c0fa Mon Sep 17 00:00:00 2001
From: Noah Goldstein <goldstein.w.n@gmail.com>
Date: Tue, 25 Jun 2024 19:50:51 +0800
Subject: [PATCH 316/441] [CodeGenPrepare][X86] Add tests for folding `urem`
 with loop invariant value; NFC

---
 llvm/test/CodeGen/X86/fold-loop-of-urem.ll    | 1159 +++++++++++++++++
 .../CodeGenPrepare/X86/fold-loop-of-urem.ll   |  858 ++++++++++++
 2 files changed, 2017 insertions(+)
 create mode 100644 llvm/test/CodeGen/X86/fold-loop-of-urem.ll
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/X86/fold-loop-of-urem.ll

diff --git a/llvm/test/CodeGen/X86/fold-loop-of-urem.ll b/llvm/test/CodeGen/X86/fold-loop-of-urem.ll
new file mode 100644
index 000000000000000..aad2e0dd7bd248a
--- /dev/null
+++ b/llvm/test/CodeGen/X86/fold-loop-of-urem.ll
@@ -0,0 +1,1159 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s
+
+declare void @use.i32(i32)
+declare void @use.2xi64(<2 x i64>)
+declare void @do_stuff0()
+declare void @do_stuff1()
+declare i1 @get.i1()
+declare i32 @get.i32()
+
+define void @simple_urem_to_sel(i32 %N, i32 %rem_amt) nounwind {
+; CHECK-LABEL: simple_urem_to_sel:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    testl %edi, %edi
+; CHECK-NEXT:    je .LBB0_4
+; CHECK-NEXT:  # %bb.1: # %for.body.preheader
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    movl %esi, %ebx
+; CHECK-NEXT:    movl %edi, %ebp
+; CHECK-NEXT:    xorl %r14d, %r14d
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB0_2: # %for.body
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movl %r14d, %eax
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divl %ebx
+; CHECK-NEXT:    movl %edx, %edi
+; CHECK-NEXT:    callq use.i32@PLT
+; CHECK-NEXT:    incl %r14d
+; CHECK-NEXT:    cmpl %r14d, %ebp
+; CHECK-NEXT:    jne .LBB0_2
+; CHECK-NEXT:  # %bb.3:
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:  .LBB0_4: # %for.cond.cleanup
+; CHECK-NEXT:    retq
+entry:
+  %cmp3.not = icmp eq i32 %N, 0
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %rem = urem i32 %i.04, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_to_sel_nested2(i32 %N, i32 %rem_amt) nounwind {
+; CHECK-LABEL: simple_urem_to_sel_nested2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    testl %edi, %edi
+; CHECK-NEXT:    je .LBB1_8
+; CHECK-NEXT:  # %bb.1: # %for.body.preheader
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    movl %esi, %ebx
+; CHECK-NEXT:    movl %edi, %ebp
+; CHECK-NEXT:    xorl %r14d, %r14d
+; CHECK-NEXT:    jmp .LBB1_2
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB1_5: # %for.body1
+; CHECK-NEXT:    # in Loop: Header=BB1_2 Depth=1
+; CHECK-NEXT:    movl %r14d, %eax
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divl %ebx
+; CHECK-NEXT:    movl %edx, %edi
+; CHECK-NEXT:    callq use.i32@PLT
+; CHECK-NEXT:  .LBB1_6: # %for.body.tail
+; CHECK-NEXT:    # in Loop: Header=BB1_2 Depth=1
+; CHECK-NEXT:    incl %r14d
+; CHECK-NEXT:    cmpl %r14d, %ebp
+; CHECK-NEXT:    je .LBB1_7
+; CHECK-NEXT:  .LBB1_2: # %for.body
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    callq get.i1@PLT
+; CHECK-NEXT:    testb $1, %al
+; CHECK-NEXT:    je .LBB1_6
+; CHECK-NEXT:  # %bb.3: # %for.body0
+; CHECK-NEXT:    # in Loop: Header=BB1_2 Depth=1
+; CHECK-NEXT:    callq get.i1@PLT
+; CHECK-NEXT:    testb $1, %al
+; CHECK-NEXT:    jne .LBB1_5
+; CHECK-NEXT:  # %bb.4: # %for.body2
+; CHECK-NEXT:    # in Loop: Header=BB1_2 Depth=1
+; CHECK-NEXT:    callq get.i1@PLT
+; CHECK-NEXT:    testb $1, %al
+; CHECK-NEXT:    jne .LBB1_5
+; CHECK-NEXT:    jmp .LBB1_6
+; CHECK-NEXT:  .LBB1_7:
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:  .LBB1_8: # %for.cond.cleanup
+; CHECK-NEXT:    retq
+entry:
+  %cmp3.not = icmp eq i32 %N, 0
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body.tail ], [ 0, %entry ]
+  %cond0 = call i1 @get.i1()
+  br i1 %cond0, label %for.body0, label %for.body.tail
+for.body0:
+  %cond1 = call i1 @get.i1()
+  br i1 %cond1, label %for.body1, label %for.body2
+for.body2:
+  %cond2 = call i1 @get.i1()
+  br i1 %cond2, label %for.body1, label %for.body.tail
+for.body1:
+  %rem = urem i32 %i.04, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  br label %for.body.tail
+for.body.tail:
+  %inc = add nuw i32 %i.04, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_fail_bad_incr3(i32 %N, i32 %rem_amt) nounwind {
+; CHECK-LABEL: simple_urem_fail_bad_incr3:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    testl %edi, %edi
+; CHECK-NEXT:    je .LBB2_9
+; CHECK-NEXT:  # %bb.1:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    movl %esi, %ebx
+; CHECK-NEXT:    jmp .LBB2_2
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB2_6: # %for.body1
+; CHECK-NEXT:    # in Loop: Header=BB2_2 Depth=1
+; CHECK-NEXT:    movl %ebp, %eax
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divl %ebx
+; CHECK-NEXT:    movl %edx, %edi
+; CHECK-NEXT:    callq use.i32@PLT
+; CHECK-NEXT:  .LBB2_7: # %for.body.tail
+; CHECK-NEXT:    # in Loop: Header=BB2_2 Depth=1
+; CHECK-NEXT:    callq get.i1@PLT
+; CHECK-NEXT:    testb $1, %al
+; CHECK-NEXT:    jne .LBB2_8
+; CHECK-NEXT:  .LBB2_2: # %for.body
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    callq get.i1@PLT
+; CHECK-NEXT:    testb $1, %al
+; CHECK-NEXT:    je .LBB2_5
+; CHECK-NEXT:  # %bb.3: # %for.body0
+; CHECK-NEXT:    # in Loop: Header=BB2_2 Depth=1
+; CHECK-NEXT:    callq get.i1@PLT
+; CHECK-NEXT:    movl %eax, %r14d
+; CHECK-NEXT:    callq get.i32@PLT
+; CHECK-NEXT:    testb $1, %r14b
+; CHECK-NEXT:    je .LBB2_7
+; CHECK-NEXT:  # %bb.4: # in Loop: Header=BB2_2 Depth=1
+; CHECK-NEXT:    movl %eax, %ebp
+; CHECK-NEXT:    incl %ebp
+; CHECK-NEXT:    jmp .LBB2_6
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB2_5: # %for.body2
+; CHECK-NEXT:    # in Loop: Header=BB2_2 Depth=1
+; CHECK-NEXT:    xorl %ebp, %ebp
+; CHECK-NEXT:    callq get.i1@PLT
+; CHECK-NEXT:    testb $1, %al
+; CHECK-NEXT:    jne .LBB2_6
+; CHECK-NEXT:    jmp .LBB2_7
+; CHECK-NEXT:  .LBB2_8:
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:  .LBB2_9: # %for.cond.cleanup
+; CHECK-NEXT:    retq
+entry:
+  %cmp3.not = icmp eq i32 %N, 0
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %cond0 = call i1 @get.i1()
+  br i1 %cond0, label %for.body0, label %for.body2
+for.body0:
+  %cond1 = call i1 @get.i1()
+  %val = call i32 @get.i32()
+  %inc = add nuw i32 %val, 1
+  br i1 %cond1, label %for.body1, label %for.body.tail
+for.body2:
+  %cond2 = call i1 @get.i1()
+  br i1 %cond2, label %for.body1, label %for.body.tail
+for.body1:
+  %i.04 = phi i32 [ %inc, %for.body0], [ 0, %for.body2 ]
+  %rem = urem i32 %i.04, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  br label %for.body.tail
+for.body.tail:
+  %exitcond.not = call i1 @get.i1()
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_to_sel_vec(<2 x i64> %rem_amt) nounwind {
+; CHECK-LABEL: simple_urem_to_sel_vec:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    subq $24, %rsp
+; CHECK-NEXT:    pxor %xmm1, %xmm1
+; CHECK-NEXT:    movdqa %xmm1, (%rsp) # 16-byte Spill
+; CHECK-NEXT:    movq %xmm0, %rbx
+; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; CHECK-NEXT:    movq %xmm0, %r14
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB3_1: # %for.body
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movdqa (%rsp), %xmm1 # 16-byte Reload
+; CHECK-NEXT:    movdqa %xmm1, (%rsp) # 16-byte Spill
+; CHECK-NEXT:    movq %xmm1, %rax
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divq %rbx
+; CHECK-NEXT:    movq %rdx, %xmm0
+; CHECK-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; CHECK-NEXT:    movq %xmm1, %rax
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divq %r14
+; CHECK-NEXT:    movq %rdx, %xmm1
+; CHECK-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; CHECK-NEXT:    callq use.2xi64@PLT
+; CHECK-NEXT:    movdqa (%rsp), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    psubq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; CHECK-NEXT:    movdqa %xmm0, (%rsp) # 16-byte Spill
+; CHECK-NEXT:    callq get.i1@PLT
+; CHECK-NEXT:    testb $1, %al
+; CHECK-NEXT:    je .LBB3_1
+; CHECK-NEXT:  # %bb.2: # %for.cond.cleanup
+; CHECK-NEXT:    addq $24, %rsp
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    retq
+entry:
+  br label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi <2 x i64> [ %inc, %for.body ], [ zeroinitializer, %entry ]
+  %rem = urem <2 x i64> %i.04, %rem_amt
+  tail call void @use.2xi64(<2 x i64> %rem)
+  %inc = add nuw <2 x i64> %i.04, <i64 1, i64 1>
+  %exitcond.not = call i1 @get.i1()
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_fail_bad_incr(i32 %N, i32 %rem_amt) nounwind {
+; CHECK-LABEL: simple_urem_fail_bad_incr:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    testl %edi, %edi
+; CHECK-NEXT:    je .LBB4_6
+; CHECK-NEXT:  # %bb.1: # %for.body.preheader
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    movl %esi, %ebx
+; CHECK-NEXT:    movl %edi, %ebp
+; CHECK-NEXT:    xorl %r14d, %r14d
+; CHECK-NEXT:    jmp .LBB4_2
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB4_4: # %for.body.tail
+; CHECK-NEXT:    # in Loop: Header=BB4_2 Depth=1
+; CHECK-NEXT:    movl %r14d, %eax
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divl %ebx
+; CHECK-NEXT:    movl %edx, %edi
+; CHECK-NEXT:    callq use.i32@PLT
+; CHECK-NEXT:    incl %r14d
+; CHECK-NEXT:    cmpl %ebp, %r14d
+; CHECK-NEXT:    je .LBB4_5
+; CHECK-NEXT:  .LBB4_2: # %for.body
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    callq get.i1@PLT
+; CHECK-NEXT:    testb $1, %al
+; CHECK-NEXT:    je .LBB4_4
+; CHECK-NEXT:  # %bb.3: # %for.body0
+; CHECK-NEXT:    # in Loop: Header=BB4_2 Depth=1
+; CHECK-NEXT:    callq get.i32@PLT
+; CHECK-NEXT:    movl %eax, %r14d
+; CHECK-NEXT:    jmp .LBB4_4
+; CHECK-NEXT:  .LBB4_5:
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:  .LBB4_6: # %for.cond.cleanup
+; CHECK-NEXT:    retq
+entry:
+  %cmp3.not = icmp eq i32 %N, 0
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.03 = phi i32 [ %inc, %for.body.tail ], [ 0, %entry ]
+  %cond0 = call i1 @get.i1()
+  br i1 %cond0, label %for.body0, label %for.body.tail
+for.body0:
+  %some_val = call i32 @get.i32()
+  br label %for.body.tail
+
+for.body.tail:
+  %i.04 = phi i32 [ %i.03, %for.body ], [ %some_val, %for.body0 ]
+  %rem = urem i32 %i.04, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_to_sel_second_acc(i32 %N, i32 %rem_amt) nounwind {
+; CHECK-LABEL: simple_urem_to_sel_second_acc:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cmpl $2, %edi
+; CHECK-NEXT:    jb .LBB5_4
+; CHECK-NEXT:  # %bb.1: # %for.body.preheader
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    movl %esi, %ebx
+; CHECK-NEXT:    movl %edi, %ebp
+; CHECK-NEXT:    movl $1, %r15d
+; CHECK-NEXT:    xorl %r14d, %r14d
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB5_2: # %for.body
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movl %r14d, %eax
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divl %ebx
+; CHECK-NEXT:    movl %edx, %edi
+; CHECK-NEXT:    callq use.i32@PLT
+; CHECK-NEXT:    incl %r14d
+; CHECK-NEXT:    addl $2, %r15d
+; CHECK-NEXT:    cmpl %ebp, %r15d
+; CHECK-NEXT:    jbe .LBB5_2
+; CHECK-NEXT:  # %bb.3:
+; CHECK-NEXT:    addq $8, %rsp
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:  .LBB5_4: # %for.cond.cleanup
+; CHECK-NEXT:    retq
+entry:
+  %cmp3.not = icmp ult i32 %N, 2
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %i.05 = phi i32 [ %inc2, %for.body ], [ 1, %entry ]
+  %rem = urem i32 %i.04, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 1
+  %inc2 = add nuw i32 %i.05, 2
+  %exitcond.not = icmp ugt i32 %inc2, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_fail_srem(i32 %N, i32 %rem_amt) nounwind {
+; CHECK-LABEL: simple_urem_fail_srem:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    testl %edi, %edi
+; CHECK-NEXT:    je .LBB6_4
+; CHECK-NEXT:  # %bb.1: # %for.body.preheader
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    movl %esi, %ebx
+; CHECK-NEXT:    movl %edi, %ebp
+; CHECK-NEXT:    xorl %r14d, %r14d
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB6_2: # %for.body
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movl %r14d, %eax
+; CHECK-NEXT:    cltd
+; CHECK-NEXT:    idivl %ebx
+; CHECK-NEXT:    movl %edx, %edi
+; CHECK-NEXT:    callq use.i32@PLT
+; CHECK-NEXT:    incl %r14d
+; CHECK-NEXT:    cmpl %r14d, %ebp
+; CHECK-NEXT:    jne .LBB6_2
+; CHECK-NEXT:  # %bb.3:
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:  .LBB6_4: # %for.cond.cleanup
+; CHECK-NEXT:    retq
+entry:
+  %cmp3.not = icmp eq i32 %N, 0
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %rem = srem i32 %i.04, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_fail_missing_nuw(i32 %N, i32 %rem_amt) nounwind {
+; CHECK-LABEL: simple_urem_fail_missing_nuw:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    testl %edi, %edi
+; CHECK-NEXT:    je .LBB7_4
+; CHECK-NEXT:  # %bb.1: # %for.body.preheader
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    movl %esi, %ebx
+; CHECK-NEXT:    movl %edi, %ebp
+; CHECK-NEXT:    xorl %r14d, %r14d
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB7_2: # %for.body
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movl %r14d, %eax
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divl %ebx
+; CHECK-NEXT:    movl %edx, %edi
+; CHECK-NEXT:    callq use.i32@PLT
+; CHECK-NEXT:    incl %r14d
+; CHECK-NEXT:    cmpl %r14d, %ebp
+; CHECK-NEXT:    jne .LBB7_2
+; CHECK-NEXT:  # %bb.3:
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:  .LBB7_4: # %for.cond.cleanup
+; CHECK-NEXT:    retq
+entry:
+  %cmp3.not = icmp eq i32 %N, 0
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %rem = urem i32 %i.04, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nsw i32 %i.04, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_fail_bad_incr2(i32 %N, i32 %rem_amt) nounwind {
+; CHECK-LABEL: simple_urem_fail_bad_incr2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    testl %edi, %edi
+; CHECK-NEXT:    je .LBB8_4
+; CHECK-NEXT:  # %bb.1: # %for.body.preheader
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    movl %esi, %ebx
+; CHECK-NEXT:    movl %edi, %ebp
+; CHECK-NEXT:    xorl %r14d, %r14d
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB8_2: # %for.body
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movl %r14d, %eax
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divl %ebx
+; CHECK-NEXT:    movl %edx, %edi
+; CHECK-NEXT:    callq use.i32@PLT
+; CHECK-NEXT:    addl $2, %r14d
+; CHECK-NEXT:    cmpl %r14d, %ebp
+; CHECK-NEXT:    jne .LBB8_2
+; CHECK-NEXT:  # %bb.3:
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:  .LBB8_4: # %for.cond.cleanup
+; CHECK-NEXT:    retq
+entry:
+  %cmp3.not = icmp eq i32 %N, 0
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %rem = urem i32 %i.04, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 2
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_non_zero_entry4(i32 %N, i32 %rem_amt) nounwind {
+; CHECK-LABEL: simple_urem_non_zero_entry4:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    testl %edi, %edi
+; CHECK-NEXT:    je .LBB9_4
+; CHECK-NEXT:  # %bb.1: # %for.body.preheader
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    movl %esi, %ebx
+; CHECK-NEXT:    movl %edi, %ebp
+; CHECK-NEXT:    movl $4, %r14d
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB9_2: # %for.body
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movl %r14d, %eax
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divl %ebx
+; CHECK-NEXT:    movl %edx, %edi
+; CHECK-NEXT:    callq use.i32@PLT
+; CHECK-NEXT:    incl %r14d
+; CHECK-NEXT:    cmpl %r14d, %ebp
+; CHECK-NEXT:    jne .LBB9_2
+; CHECK-NEXT:  # %bb.3:
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:  .LBB9_4: # %for.cond.cleanup
+; CHECK-NEXT:    retq
+entry:
+  %cmp3.not = icmp eq i32 %N, 0
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ 4, %entry ]
+  %rem = urem i32 %i.04, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_skip_const_rem_amt(i32 %N) nounwind {
+; CHECK-LABEL: simple_urem_skip_const_rem_amt:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    testl %edi, %edi
+; CHECK-NEXT:    je .LBB10_4
+; CHECK-NEXT:  # %bb.1: # %for.body.preheader
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    movl %edi, %ebx
+; CHECK-NEXT:    addl $-4, %ebx
+; CHECK-NEXT:    movl $4, %ebp
+; CHECK-NEXT:    movl $2938661835, %r14d # imm = 0xAF286BCB
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB10_2: # %for.body
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movl %ebp, %eax
+; CHECK-NEXT:    imulq %r14, %rax
+; CHECK-NEXT:    shrq $32, %rax
+; CHECK-NEXT:    movl %ebp, %ecx
+; CHECK-NEXT:    subl %eax, %ecx
+; CHECK-NEXT:    shrl %ecx
+; CHECK-NEXT:    addl %eax, %ecx
+; CHECK-NEXT:    shrl $4, %ecx
+; CHECK-NEXT:    leal (%rcx,%rcx,8), %eax
+; CHECK-NEXT:    leal (%rcx,%rax,2), %eax
+; CHECK-NEXT:    movl %ebp, %edi
+; CHECK-NEXT:    subl %eax, %edi
+; CHECK-NEXT:    callq use.i32@PLT
+; CHECK-NEXT:    incl %ebp
+; CHECK-NEXT:    decl %ebx
+; CHECK-NEXT:    jne .LBB10_2
+; CHECK-NEXT:  # %bb.3:
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:  .LBB10_4: # %for.cond.cleanup
+; CHECK-NEXT:    retq
+entry:
+  %cmp3.not = icmp eq i32 %N, 0
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ 4, %entry ]
+  %rem = urem i32 %i.04, 19
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_fail_no_preheader_non_canonical(i32 %N, i32 %rem_amt) nounwind {
+; CHECK-LABEL: simple_urem_fail_no_preheader_non_canonical:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    movl %esi, %ebx
+; CHECK-NEXT:    movl %edi, %ebp
+; CHECK-NEXT:    testl %edi, %edi
+; CHECK-NEXT:    je .LBB11_1
+; CHECK-NEXT:  # %bb.2: # %for.body1
+; CHECK-NEXT:    movl $1, %r14d
+; CHECK-NEXT:    jmp .LBB11_3
+; CHECK-NEXT:  .LBB11_1:
+; CHECK-NEXT:    xorl %r14d, %r14d
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB11_3: # %for.body
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movl %r14d, %eax
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divl %ebx
+; CHECK-NEXT:    movl %edx, %edi
+; CHECK-NEXT:    callq use.i32@PLT
+; CHECK-NEXT:    incl %r14d
+; CHECK-NEXT:    cmpl %r14d, %ebp
+; CHECK-NEXT:    jne .LBB11_3
+; CHECK-NEXT:  # %bb.4: # %for.cond.cleanup
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    retq
+entry:
+  %cmp3.not = icmp eq i32 %N, 0
+  br i1 %cmp3.not, label %for.body0, label %for.body1
+
+for.cond.cleanup:
+  ret void
+
+for.body0:
+  br label %for.body
+
+for.body1:
+  br label %for.body
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ 0, %for.body0 ], [ 1, %for.body1 ]
+  %rem = urem i32 %i.04, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_multi_latch_non_canonical(i32 %N, i32 %rem_amt) nounwind {
+; CHECK-LABEL: simple_urem_multi_latch_non_canonical:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    testl %edi, %edi
+; CHECK-NEXT:    je .LBB12_6
+; CHECK-NEXT:  # %bb.1: # %for.body.preheader
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    movl %esi, %ebx
+; CHECK-NEXT:    movl %edi, %ebp
+; CHECK-NEXT:    decl %ebp
+; CHECK-NEXT:    xorl %r14d, %r14d
+; CHECK-NEXT:    jmp .LBB12_2
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB12_3: # %for.body.backedge
+; CHECK-NEXT:    # in Loop: Header=BB12_2 Depth=1
+; CHECK-NEXT:    incl %r14d
+; CHECK-NEXT:  .LBB12_2: # %for.body
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movl %r14d, %eax
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divl %ebx
+; CHECK-NEXT:    movl %edx, %edi
+; CHECK-NEXT:    callq use.i32@PLT
+; CHECK-NEXT:    callq get.i1@PLT
+; CHECK-NEXT:    movl %eax, %r15d
+; CHECK-NEXT:    callq do_stuff0@PLT
+; CHECK-NEXT:    testb $1, %r15b
+; CHECK-NEXT:    je .LBB12_3
+; CHECK-NEXT:  # %bb.4: # %for.body0
+; CHECK-NEXT:    # in Loop: Header=BB12_2 Depth=1
+; CHECK-NEXT:    callq do_stuff1@PLT
+; CHECK-NEXT:    cmpl %r14d, %ebp
+; CHECK-NEXT:    jne .LBB12_3
+; CHECK-NEXT:  # %bb.5:
+; CHECK-NEXT:    addq $8, %rsp
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:  .LBB12_6: # %for.cond.cleanup
+; CHECK-NEXT:    retq
+entry:
+  %cmp3.not = icmp eq i32 %N, 0
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ %inc, %for.body0 ], [ 0, %entry ]
+  %rem = urem i32 %i.04, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 1
+  %cond = call i1 @get.i1()
+  call void @do_stuff0()
+  br i1 %cond, label %for.body0, label %for.body
+for.body0:
+  call void @do_stuff1()
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_fail_bad_loop(i32 %N, i32 %rem_amt) nounwind {
+; CHECK-LABEL: simple_urem_fail_bad_loop:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    movl %esi, %ebx
+; CHECK-NEXT:    movl %edi, %ebp
+; CHECK-NEXT:    callq get.i32@PLT
+; CHECK-NEXT:    testl %eax, %eax
+; CHECK-NEXT:    # implicit-def: $r14d
+; CHECK-NEXT:    jne .LBB13_4
+; CHECK-NEXT:  # %bb.1:
+; CHECK-NEXT:    xorl %r14d, %r14d
+; CHECK-NEXT:  .LBB13_2: # %for.cond
+; CHECK-NEXT:    cmpl %ebp, %r14d
+; CHECK-NEXT:    jae .LBB13_5
+; CHECK-NEXT:  # %bb.3: # %for.body
+; CHECK-NEXT:    movl %r14d, %edi
+; CHECK-NEXT:    xorl $1, %edi
+; CHECK-NEXT:    callq use.i32@PLT
+; CHECK-NEXT:  .LBB13_4: # %halfway
+; CHECK-NEXT:    movl %r14d, %eax
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divl %ebx
+; CHECK-NEXT:    movl %edx, %edi
+; CHECK-NEXT:    callq use.i32@PLT
+; CHECK-NEXT:    incl %r14d
+; CHECK-NEXT:    jmp .LBB13_2
+; CHECK-NEXT:  .LBB13_5: # %for.end
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    retq
+entry:
+  %call = call i32 @get.i32()
+  %tobool.not = icmp eq i32 %call, 0
+  br i1 %tobool.not, label %for.cond, label %halfway
+
+for.cond:
+  %i.0 = phi i32 [ %inc, %halfway ], [ 0, %entry ]
+  %cmp = icmp ult i32 %i.0, %N
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:
+  %xor = xor i32 %i.0, 1
+  call void @use.i32(i32 %xor)
+  br label %halfway
+
+halfway:
+  %i.1 = phi i32 [ poison, %entry ], [ %i.0, %for.body ]
+  %rem = urem i32 %i.1, %rem_amt
+  call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.1, 1
+  br label %for.cond
+
+for.end:
+  ret void
+}
+
+define void @simple_urem_fail_intermediate_inc(i32 %N, i32 %rem_amt) nounwind {
+; CHECK-LABEL: simple_urem_fail_intermediate_inc:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    testl %edi, %edi
+; CHECK-NEXT:    je .LBB14_4
+; CHECK-NEXT:  # %bb.1: # %for.body.preheader
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    movl %esi, %ebx
+; CHECK-NEXT:    movl %edi, %r14d
+; CHECK-NEXT:    negl %r14d
+; CHECK-NEXT:    movl $1, %r15d
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB14_2: # %for.body
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movl %r15d, %eax
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divl %ebx
+; CHECK-NEXT:    movl %edx, %edi
+; CHECK-NEXT:    callq use.i32@PLT
+; CHECK-NEXT:    leal 1(%r14,%r15), %eax
+; CHECK-NEXT:    movl %r15d, %ecx
+; CHECK-NEXT:    incl %ecx
+; CHECK-NEXT:    cmpl $1, %eax
+; CHECK-NEXT:    movl %ecx, %r15d
+; CHECK-NEXT:    jne .LBB14_2
+; CHECK-NEXT:  # %bb.3:
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:  .LBB14_4: # %for.cond.cleanup
+; CHECK-NEXT:    retq
+entry:
+  %cmp3.not = icmp eq i32 %N, 0
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %inc2 = add nuw i32 %i.04, 1
+  %rem = urem i32 %inc2, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @weird_loop(i64 %sub.ptr.div.i56) personality ptr null {
+; CHECK-LABEL: weird_loop:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB15_1: # %for.body
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    jmp .LBB15_1
+entry:
+  br label %for.preheader
+
+for.preheader:
+  %i57.0540.us = phi i64 [ 0, %entry ], [ %add74.us, %for.body ]
+  %add74.us = add nuw i64 %i57.0540.us, 1
+  br label %for.body
+
+for.body:
+  %rem.us = urem i64 %i57.0540.us, %sub.ptr.div.i56
+  br i1 false, label %for.preheader, label %for.body
+}
+
+define void @simple_urem_to_sel_non_zero_start(i32 %N, i32 %rem_amt) nounwind {
+; CHECK-LABEL: simple_urem_to_sel_non_zero_start:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cmpl $3, %edi
+; CHECK-NEXT:    jb .LBB16_4
+; CHECK-NEXT:  # %bb.1: # %for.body.preheader
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    movl %esi, %ebx
+; CHECK-NEXT:    movl %edi, %ebp
+; CHECK-NEXT:    movl $2, %r14d
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB16_2: # %for.body
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movl %r14d, %eax
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divl %ebx
+; CHECK-NEXT:    movl %edx, %edi
+; CHECK-NEXT:    callq use.i32@PLT
+; CHECK-NEXT:    incl %r14d
+; CHECK-NEXT:    cmpl %r14d, %ebp
+; CHECK-NEXT:    jne .LBB16_2
+; CHECK-NEXT:  # %bb.3:
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:  .LBB16_4: # %for.cond.cleanup
+; CHECK-NEXT:    retq
+entry:
+  %cmp3.not = icmp ult i32 %N, 3
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ 2, %entry ]
+  %rem = urem i32 %i.04, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_to_sel_non_zero_start_through_add(i32 %N, i32 %rem_amt_in) nounwind {
+; CHECK-LABEL: simple_urem_to_sel_non_zero_start_through_add:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cmpl $3, %edi
+; CHECK-NEXT:    jb .LBB17_4
+; CHECK-NEXT:  # %bb.1: # %for.body.preheader
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    movl %esi, %ebx
+; CHECK-NEXT:    movl %edi, %r14d
+; CHECK-NEXT:    orl $16, %ebx
+; CHECK-NEXT:    negl %r14d
+; CHECK-NEXT:    movl $7, %r15d
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB17_2: # %for.body
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movl %r15d, %eax
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divl %ebx
+; CHECK-NEXT:    movl %edx, %edi
+; CHECK-NEXT:    callq use.i32@PLT
+; CHECK-NEXT:    leal 1(%r14,%r15), %eax
+; CHECK-NEXT:    movl %r15d, %ecx
+; CHECK-NEXT:    incl %ecx
+; CHECK-NEXT:    cmpl $5, %eax
+; CHECK-NEXT:    movl %ecx, %r15d
+; CHECK-NEXT:    jne .LBB17_2
+; CHECK-NEXT:  # %bb.3:
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:  .LBB17_4: # %for.cond.cleanup
+; CHECK-NEXT:    retq
+entry:
+  %rem_amt = or i32 %rem_amt_in, 16
+  %cmp3.not = icmp ult i32 %N, 3
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ 2, %entry ]
+  %i_with_off = add nuw i32 %i.04, 5
+  %rem = urem i32 %i_with_off, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_to_sel_non_zero_start_through_add_fail_missing_nuw(i32 %N, i32 %rem_amt_in) nounwind {
+; CHECK-LABEL: simple_urem_to_sel_non_zero_start_through_add_fail_missing_nuw:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cmpl $3, %edi
+; CHECK-NEXT:    jb .LBB18_4
+; CHECK-NEXT:  # %bb.1: # %for.body.preheader
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    movl %esi, %ebx
+; CHECK-NEXT:    movl %edi, %r14d
+; CHECK-NEXT:    orl $16, %ebx
+; CHECK-NEXT:    negl %r14d
+; CHECK-NEXT:    movl $7, %r15d
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB18_2: # %for.body
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movl %r15d, %eax
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divl %ebx
+; CHECK-NEXT:    movl %edx, %edi
+; CHECK-NEXT:    callq use.i32@PLT
+; CHECK-NEXT:    leal 1(%r14,%r15), %eax
+; CHECK-NEXT:    movl %r15d, %ecx
+; CHECK-NEXT:    incl %ecx
+; CHECK-NEXT:    cmpl $5, %eax
+; CHECK-NEXT:    movl %ecx, %r15d
+; CHECK-NEXT:    jne .LBB18_2
+; CHECK-NEXT:  # %bb.3:
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:  .LBB18_4: # %for.cond.cleanup
+; CHECK-NEXT:    retq
+entry:
+  %rem_amt = or i32 %rem_amt_in, 16
+  %cmp3.not = icmp ult i32 %N, 3
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ 2, %entry ]
+  %i_with_off = add i32 %i.04, 5
+  %rem = urem i32 %i_with_off, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_to_sel_non_zero_start_through_add_fail_no_simplify_rem(i32 %N, i32 %rem_amt) nounwind {
+; CHECK-LABEL: simple_urem_to_sel_non_zero_start_through_add_fail_no_simplify_rem:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cmpl $3, %edi
+; CHECK-NEXT:    jb .LBB19_4
+; CHECK-NEXT:  # %bb.1: # %for.body.preheader
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    movl %esi, %ebx
+; CHECK-NEXT:    movl %edi, %r14d
+; CHECK-NEXT:    negl %r14d
+; CHECK-NEXT:    movl $7, %r15d
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB19_2: # %for.body
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movl %r15d, %eax
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divl %ebx
+; CHECK-NEXT:    movl %edx, %edi
+; CHECK-NEXT:    callq use.i32@PLT
+; CHECK-NEXT:    leal 1(%r14,%r15), %eax
+; CHECK-NEXT:    movl %r15d, %ecx
+; CHECK-NEXT:    incl %ecx
+; CHECK-NEXT:    cmpl $5, %eax
+; CHECK-NEXT:    movl %ecx, %r15d
+; CHECK-NEXT:    jne .LBB19_2
+; CHECK-NEXT:  # %bb.3:
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:  .LBB19_4: # %for.cond.cleanup
+; CHECK-NEXT:    retq
+entry:
+  %cmp3.not = icmp ult i32 %N, 3
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ 2, %entry ]
+  %i_with_off = add nuw i32 %i.04, 5
+  %rem = urem i32 %i_with_off, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_to_sel_non_zero_start_through_sub(i32 %N, i32 %rem_amt, i32 %start) nounwind {
+; CHECK-LABEL: simple_urem_to_sel_non_zero_start_through_sub:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    movl %edi, %ebp
+; CHECK-NEXT:    subl %edx, %ebp
+; CHECK-NEXT:    jbe .LBB20_3
+; CHECK-NEXT:  # %bb.1: # %for.body.preheader
+; CHECK-NEXT:    movl %esi, %ebx
+; CHECK-NEXT:    xorl %r14d, %r14d
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB20_2: # %for.body
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movl %r14d, %eax
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divl %ebx
+; CHECK-NEXT:    movl %edx, %edi
+; CHECK-NEXT:    callq use.i32@PLT
+; CHECK-NEXT:    incl %r14d
+; CHECK-NEXT:    cmpl %r14d, %ebp
+; CHECK-NEXT:    jne .LBB20_2
+; CHECK-NEXT:  .LBB20_3: # %for.cond.cleanup
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    retq
+entry:
+  %cmp3.not = icmp ule i32 %N, %start
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ %start, %entry ]
+  %i_with_off = sub nuw i32 %i.04, %start
+  %rem = urem i32 %i_with_off, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_to_sel_non_zero_start_through_sub_no_simplfy(i32 %N, i32 %rem_amt, i32 %start) nounwind {
+; CHECK-LABEL: simple_urem_to_sel_non_zero_start_through_sub_no_simplfy:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cmpl %edx, %edi
+; CHECK-NEXT:    jbe .LBB21_4
+; CHECK-NEXT:  # %bb.1: # %for.body.preheader
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    movl %edx, %r15d
+; CHECK-NEXT:    movl %esi, %ebx
+; CHECK-NEXT:    movl %edi, %r14d
+; CHECK-NEXT:    negl %r14d
+; CHECK-NEXT:    addl $-2, %r15d
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB21_2: # %for.body
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movl %r15d, %eax
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divl %ebx
+; CHECK-NEXT:    movl %edx, %edi
+; CHECK-NEXT:    callq use.i32@PLT
+; CHECK-NEXT:    leal 1(%r14,%r15), %eax
+; CHECK-NEXT:    movl %r15d, %ecx
+; CHECK-NEXT:    incl %ecx
+; CHECK-NEXT:    cmpl $-2, %eax
+; CHECK-NEXT:    movl %ecx, %r15d
+; CHECK-NEXT:    jne .LBB21_2
+; CHECK-NEXT:  # %bb.3:
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:  .LBB21_4: # %for.cond.cleanup
+; CHECK-NEXT:    retq
+entry:
+  %cmp3.not = icmp ule i32 %N, %start
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ %start, %entry ]
+  %i_with_off = sub nuw i32 %i.04, 2
+  %rem = urem i32 %i_with_off, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/fold-loop-of-urem.ll b/llvm/test/Transforms/CodeGenPrepare/X86/fold-loop-of-urem.ll
new file mode 100644
index 000000000000000..a019679e65905de
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/fold-loop-of-urem.ll
@@ -0,0 +1,858 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -mtriple=x86_64-unknown-unknown --loop-simplify -codegenprepare -S | FileCheck %s
+
+declare void @use.i32(i32)
+declare void @use.2xi64(<2 x i64>)
+declare void @do_stuff0()
+declare void @do_stuff1()
+declare i1 @get.i1()
+declare i32 @get.i32()
+
+define void @simple_urem_to_sel(i32 %N, i32 %rem_amt) nounwind {
+; CHECK-LABEL: define void @simple_urem_to_sel(
+; CHECK-SAME: i32 [[N:%.*]], i32 [[REM_AMT:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[CMP3_NOT:%.*]] = icmp eq i32 [[N]], 0
+; CHECK-NEXT:    br i1 [[CMP3_NOT]], label %[[FOR_COND_CLEANUP:.*]], label %[[FOR_BODY_PREHEADER:.*]]
+; CHECK:       [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT:    ret void
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[I_04:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[REM:%.*]] = urem i32 [[I_04]], [[REM_AMT]]
+; CHECK-NEXT:    tail call void @use.i32(i32 [[REM]])
+; CHECK-NEXT:    [[INC]] = add nuw i32 [[I_04]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+  %cmp3.not = icmp eq i32 %N, 0
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %rem = urem i32 %i.04, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_to_sel_nested2(i32 %N, i32 %rem_amt) nounwind {
+; CHECK-LABEL: define void @simple_urem_to_sel_nested2(
+; CHECK-SAME: i32 [[N:%.*]], i32 [[REM_AMT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[CMP3_NOT:%.*]] = icmp eq i32 [[N]], 0
+; CHECK-NEXT:    br i1 [[CMP3_NOT]], label %[[FOR_COND_CLEANUP:.*]], label %[[FOR_BODY_PREHEADER:.*]]
+; CHECK:       [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT:    ret void
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[I_04:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY_TAIL:.*]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[COND0:%.*]] = call i1 @get.i1()
+; CHECK-NEXT:    br i1 [[COND0]], label %[[FOR_BODY0:.*]], label %[[FOR_BODY_TAIL]]
+; CHECK:       [[FOR_BODY0]]:
+; CHECK-NEXT:    [[COND1:%.*]] = call i1 @get.i1()
+; CHECK-NEXT:    br i1 [[COND1]], label %[[FOR_BODY1:.*]], label %[[FOR_BODY2:.*]]
+; CHECK:       [[FOR_BODY2]]:
+; CHECK-NEXT:    [[COND2:%.*]] = call i1 @get.i1()
+; CHECK-NEXT:    br i1 [[COND2]], label %[[FOR_BODY1]], label %[[FOR_BODY_TAIL]]
+; CHECK:       [[FOR_BODY1]]:
+; CHECK-NEXT:    [[REM:%.*]] = urem i32 [[I_04]], [[REM_AMT]]
+; CHECK-NEXT:    tail call void @use.i32(i32 [[REM]])
+; CHECK-NEXT:    br label %[[FOR_BODY_TAIL]]
+; CHECK:       [[FOR_BODY_TAIL]]:
+; CHECK-NEXT:    [[INC]] = add nuw i32 [[I_04]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+  %cmp3.not = icmp eq i32 %N, 0
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body.tail ], [ 0, %entry ]
+  %cond0 = call i1 @get.i1()
+  br i1 %cond0, label %for.body0, label %for.body.tail
+for.body0:
+  %cond1 = call i1 @get.i1()
+  br i1 %cond1, label %for.body1, label %for.body2
+for.body2:
+  %cond2 = call i1 @get.i1()
+  br i1 %cond2, label %for.body1, label %for.body.tail
+for.body1:
+  %rem = urem i32 %i.04, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  br label %for.body.tail
+for.body.tail:
+  %inc = add nuw i32 %i.04, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_fail_bad_incr3(i32 %N, i32 %rem_amt) nounwind {
+; CHECK-LABEL: define void @simple_urem_fail_bad_incr3(
+; CHECK-SAME: i32 [[N:%.*]], i32 [[REM_AMT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[CMP3_NOT:%.*]] = icmp eq i32 [[N]], 0
+; CHECK-NEXT:    br i1 [[CMP3_NOT]], label %[[FOR_COND_CLEANUP:.*]], label %[[FOR_BODY_PREHEADER:.*]]
+; CHECK:       [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT:    ret void
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[COND0:%.*]] = call i1 @get.i1()
+; CHECK-NEXT:    br i1 [[COND0]], label %[[FOR_BODY0:.*]], label %[[FOR_BODY2:.*]]
+; CHECK:       [[FOR_BODY0]]:
+; CHECK-NEXT:    [[COND1:%.*]] = call i1 @get.i1()
+; CHECK-NEXT:    [[VAL:%.*]] = call i32 @get.i32()
+; CHECK-NEXT:    [[INC:%.*]] = add nuw i32 [[VAL]], 1
+; CHECK-NEXT:    br i1 [[COND1]], label %[[FOR_BODY1:.*]], label %[[FOR_BODY_TAIL:.*]]
+; CHECK:       [[FOR_BODY2]]:
+; CHECK-NEXT:    [[COND2:%.*]] = call i1 @get.i1()
+; CHECK-NEXT:    br i1 [[COND2]], label %[[FOR_BODY1]], label %[[FOR_BODY_TAIL]]
+; CHECK:       [[FOR_BODY1]]:
+; CHECK-NEXT:    [[I_04:%.*]] = phi i32 [ [[INC]], %[[FOR_BODY0]] ], [ 0, %[[FOR_BODY2]] ]
+; CHECK-NEXT:    [[REM:%.*]] = urem i32 [[I_04]], [[REM_AMT]]
+; CHECK-NEXT:    tail call void @use.i32(i32 [[REM]])
+; CHECK-NEXT:    br label %[[FOR_BODY_TAIL]]
+; CHECK:       [[FOR_BODY_TAIL]]:
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = call i1 @get.i1()
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+  %cmp3.not = icmp eq i32 %N, 0
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %cond0 = call i1 @get.i1()
+  br i1 %cond0, label %for.body0, label %for.body2
+for.body0:
+  %cond1 = call i1 @get.i1()
+  %val = call i32 @get.i32()
+  %inc = add nuw i32 %val, 1
+  br i1 %cond1, label %for.body1, label %for.body.tail
+for.body2:
+  %cond2 = call i1 @get.i1()
+  br i1 %cond2, label %for.body1, label %for.body.tail
+for.body1:
+  %i.04 = phi i32 [ %inc, %for.body0], [ 0, %for.body2 ]
+  %rem = urem i32 %i.04, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  br label %for.body.tail
+for.body.tail:
+  %exitcond.not = call i1 @get.i1()
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_to_sel_vec(<2 x i64> %rem_amt) nounwind {
+; CHECK-LABEL: define void @simple_urem_to_sel_vec(
+; CHECK-SAME: <2 x i64> [[REM_AMT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_COND_CLEANUP:.*]]:
+; CHECK-NEXT:    ret void
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[I_04:%.*]] = phi <2 x i64> [ [[INC:%.*]], %[[FOR_BODY]] ], [ zeroinitializer, %[[ENTRY]] ]
+; CHECK-NEXT:    [[REM:%.*]] = urem <2 x i64> [[I_04]], [[REM_AMT]]
+; CHECK-NEXT:    tail call void @use.2xi64(<2 x i64> [[REM]])
+; CHECK-NEXT:    [[INC]] = add nuw <2 x i64> [[I_04]], <i64 1, i64 1>
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = call i1 @get.i1()
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+  br label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi <2 x i64> [ %inc, %for.body ], [ zeroinitializer, %entry ]
+  %rem = urem <2 x i64> %i.04, %rem_amt
+  tail call void @use.2xi64(<2 x i64> %rem)
+  %inc = add nuw <2 x i64> %i.04, <i64 1, i64 1>
+  %exitcond.not = call i1 @get.i1()
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_fail_bad_incr(i32 %N, i32 %rem_amt) nounwind {
+; CHECK-LABEL: define void @simple_urem_fail_bad_incr(
+; CHECK-SAME: i32 [[N:%.*]], i32 [[REM_AMT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[CMP3_NOT:%.*]] = icmp eq i32 [[N]], 0
+; CHECK-NEXT:    br i1 [[CMP3_NOT]], label %[[FOR_COND_CLEANUP:.*]], label %[[FOR_BODY_PREHEADER:.*]]
+; CHECK:       [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT:    ret void
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[I_03:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY_TAIL:.*]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[COND0:%.*]] = call i1 @get.i1()
+; CHECK-NEXT:    br i1 [[COND0]], label %[[FOR_BODY0:.*]], label %[[FOR_BODY_TAIL]]
+; CHECK:       [[FOR_BODY0]]:
+; CHECK-NEXT:    [[SOME_VAL:%.*]] = call i32 @get.i32()
+; CHECK-NEXT:    br label %[[FOR_BODY_TAIL]]
+; CHECK:       [[FOR_BODY_TAIL]]:
+; CHECK-NEXT:    [[I_04:%.*]] = phi i32 [ [[I_03]], %[[FOR_BODY]] ], [ [[SOME_VAL]], %[[FOR_BODY0]] ]
+; CHECK-NEXT:    [[REM:%.*]] = urem i32 [[I_04]], [[REM_AMT]]
+; CHECK-NEXT:    tail call void @use.i32(i32 [[REM]])
+; CHECK-NEXT:    [[INC]] = add nuw i32 [[I_04]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+  %cmp3.not = icmp eq i32 %N, 0
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.03 = phi i32 [ %inc, %for.body.tail ], [ 0, %entry ]
+  %cond0 = call i1 @get.i1()
+  br i1 %cond0, label %for.body0, label %for.body.tail
+for.body0:
+  %some_val = call i32 @get.i32()
+  br label %for.body.tail
+
+for.body.tail:
+  %i.04 = phi i32 [ %i.03, %for.body ], [ %some_val, %for.body0 ]
+  %rem = urem i32 %i.04, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_to_sel_second_acc(i32 %N, i32 %rem_amt) nounwind {
+; CHECK-LABEL: define void @simple_urem_to_sel_second_acc(
+; CHECK-SAME: i32 [[N:%.*]], i32 [[REM_AMT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[CMP3_NOT:%.*]] = icmp ult i32 [[N]], 2
+; CHECK-NEXT:    br i1 [[CMP3_NOT]], label %[[FOR_COND_CLEANUP:.*]], label %[[FOR_BODY_PREHEADER:.*]]
+; CHECK:       [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT:    ret void
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[I_04:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[I_05:%.*]] = phi i32 [ [[INC2:%.*]], %[[FOR_BODY]] ], [ 1, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[REM:%.*]] = urem i32 [[I_04]], [[REM_AMT]]
+; CHECK-NEXT:    tail call void @use.i32(i32 [[REM]])
+; CHECK-NEXT:    [[INC]] = add nuw i32 [[I_04]], 1
+; CHECK-NEXT:    [[INC2]] = add nuw i32 [[I_05]], 2
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp ugt i32 [[INC2]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+  %cmp3.not = icmp ult i32 %N, 2
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %i.05 = phi i32 [ %inc2, %for.body ], [ 1, %entry ]
+  %rem = urem i32 %i.04, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 1
+  %inc2 = add nuw i32 %i.05, 2
+  %exitcond.not = icmp ugt i32 %inc2, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_fail_srem(i32 %N, i32 %rem_amt) nounwind {
+; CHECK-LABEL: define void @simple_urem_fail_srem(
+; CHECK-SAME: i32 [[N:%.*]], i32 [[REM_AMT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[CMP3_NOT:%.*]] = icmp eq i32 [[N]], 0
+; CHECK-NEXT:    br i1 [[CMP3_NOT]], label %[[FOR_COND_CLEANUP:.*]], label %[[FOR_BODY_PREHEADER:.*]]
+; CHECK:       [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT:    ret void
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[I_04:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[REM:%.*]] = srem i32 [[I_04]], [[REM_AMT]]
+; CHECK-NEXT:    tail call void @use.i32(i32 [[REM]])
+; CHECK-NEXT:    [[INC]] = add nuw i32 [[I_04]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+  %cmp3.not = icmp eq i32 %N, 0
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %rem = srem i32 %i.04, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_fail_missing_nuw(i32 %N, i32 %rem_amt) nounwind {
+; CHECK-LABEL: define void @simple_urem_fail_missing_nuw(
+; CHECK-SAME: i32 [[N:%.*]], i32 [[REM_AMT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[CMP3_NOT:%.*]] = icmp eq i32 [[N]], 0
+; CHECK-NEXT:    br i1 [[CMP3_NOT]], label %[[FOR_COND_CLEANUP:.*]], label %[[FOR_BODY_PREHEADER:.*]]
+; CHECK:       [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT:    ret void
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[I_04:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[REM:%.*]] = urem i32 [[I_04]], [[REM_AMT]]
+; CHECK-NEXT:    tail call void @use.i32(i32 [[REM]])
+; CHECK-NEXT:    [[INC]] = add nsw i32 [[I_04]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+  %cmp3.not = icmp eq i32 %N, 0
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %rem = urem i32 %i.04, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nsw i32 %i.04, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_fail_bad_incr2(i32 %N, i32 %rem_amt) nounwind {
+; CHECK-LABEL: define void @simple_urem_fail_bad_incr2(
+; CHECK-SAME: i32 [[N:%.*]], i32 [[REM_AMT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[CMP3_NOT:%.*]] = icmp eq i32 [[N]], 0
+; CHECK-NEXT:    br i1 [[CMP3_NOT]], label %[[FOR_COND_CLEANUP:.*]], label %[[FOR_BODY_PREHEADER:.*]]
+; CHECK:       [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT:    ret void
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[I_04:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[REM:%.*]] = urem i32 [[I_04]], [[REM_AMT]]
+; CHECK-NEXT:    tail call void @use.i32(i32 [[REM]])
+; CHECK-NEXT:    [[INC]] = add nuw i32 [[I_04]], 2
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+  %cmp3.not = icmp eq i32 %N, 0
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %rem = urem i32 %i.04, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 2
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_non_zero_entry4(i32 %N, i32 %rem_amt) nounwind {
+; CHECK-LABEL: define void @simple_urem_non_zero_entry4(
+; CHECK-SAME: i32 [[N:%.*]], i32 [[REM_AMT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[CMP3_NOT:%.*]] = icmp eq i32 [[N]], 0
+; CHECK-NEXT:    br i1 [[CMP3_NOT]], label %[[FOR_COND_CLEANUP:.*]], label %[[FOR_BODY_PREHEADER:.*]]
+; CHECK:       [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT:    ret void
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[I_04:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 4, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[REM:%.*]] = urem i32 [[I_04]], [[REM_AMT]]
+; CHECK-NEXT:    tail call void @use.i32(i32 [[REM]])
+; CHECK-NEXT:    [[INC]] = add nuw i32 [[I_04]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+  %cmp3.not = icmp eq i32 %N, 0
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ 4, %entry ]
+  %rem = urem i32 %i.04, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_skip_const_rem_amt(i32 %N) nounwind {
+; CHECK-LABEL: define void @simple_urem_skip_const_rem_amt(
+; CHECK-SAME: i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[CMP3_NOT:%.*]] = icmp eq i32 [[N]], 0
+; CHECK-NEXT:    br i1 [[CMP3_NOT]], label %[[FOR_COND_CLEANUP:.*]], label %[[FOR_BODY_PREHEADER:.*]]
+; CHECK:       [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT:    ret void
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[I_04:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 4, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[REM:%.*]] = urem i32 [[I_04]], 19
+; CHECK-NEXT:    tail call void @use.i32(i32 [[REM]])
+; CHECK-NEXT:    [[INC]] = add nuw i32 [[I_04]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+  %cmp3.not = icmp eq i32 %N, 0
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ 4, %entry ]
+  %rem = urem i32 %i.04, 19
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_fail_no_preheader_non_canonical(i32 %N, i32 %rem_amt) nounwind {
+; CHECK-LABEL: define void @simple_urem_fail_no_preheader_non_canonical(
+; CHECK-SAME: i32 [[N:%.*]], i32 [[REM_AMT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[CMP3_NOT:%.*]] = icmp eq i32 [[N]], 0
+; CHECK-NEXT:    br i1 [[CMP3_NOT]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_BODY1:.*]]
+; CHECK:       [[FOR_COND_CLEANUP:.*]]:
+; CHECK-NEXT:    ret void
+; CHECK:       [[FOR_BODY1]]:
+; CHECK-NEXT:    br label %[[FOR_BODY_PREHEADER]]
+; CHECK:       [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT:    [[I_04_PH:%.*]] = phi i32 [ 1, %[[FOR_BODY1]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[I_04:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[I_04_PH]], %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[REM:%.*]] = urem i32 [[I_04]], [[REM_AMT]]
+; CHECK-NEXT:    tail call void @use.i32(i32 [[REM]])
+; CHECK-NEXT:    [[INC]] = add nuw i32 [[I_04]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+  %cmp3.not = icmp eq i32 %N, 0
+  br i1 %cmp3.not, label %for.body0, label %for.body1
+
+for.cond.cleanup:
+  ret void
+
+for.body0:
+  br label %for.body
+
+for.body1:
+  br label %for.body
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ 0, %for.body0 ], [ 1, %for.body1 ]
+  %rem = urem i32 %i.04, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_multi_latch_non_canonical(i32 %N, i32 %rem_amt) nounwind {
+; CHECK-LABEL: define void @simple_urem_multi_latch_non_canonical(
+; CHECK-SAME: i32 [[N:%.*]], i32 [[REM_AMT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[CMP3_NOT:%.*]] = icmp eq i32 [[N]], 0
+; CHECK-NEXT:    br i1 [[CMP3_NOT]], label %[[FOR_COND_CLEANUP:.*]], label %[[FOR_BODY_PREHEADER:.*]]
+; CHECK:       [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT:    ret void
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[I_04:%.*]] = phi i32 [ 0, %[[FOR_BODY_PREHEADER]] ], [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[INC]], %[[FOR_BODY0:.*]] ]
+; CHECK-NEXT:    [[REM:%.*]] = urem i32 [[I_04]], [[REM_AMT]]
+; CHECK-NEXT:    tail call void @use.i32(i32 [[REM]])
+; CHECK-NEXT:    [[INC]] = add nuw i32 [[I_04]], 1
+; CHECK-NEXT:    [[COND:%.*]] = call i1 @get.i1()
+; CHECK-NEXT:    call void @do_stuff0()
+; CHECK-NEXT:    br i1 [[COND]], label %[[FOR_BODY0]], label %[[FOR_BODY]]
+; CHECK:       [[FOR_BODY0]]:
+; CHECK-NEXT:    call void @do_stuff1()
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+  %cmp3.not = icmp eq i32 %N, 0
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ %inc, %for.body0 ], [ 0, %entry ]
+  %rem = urem i32 %i.04, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 1
+  %cond = call i1 @get.i1()
+  call void @do_stuff0()
+  br i1 %cond, label %for.body0, label %for.body
+for.body0:
+  call void @do_stuff1()
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_fail_bad_loop(i32 %N, i32 %rem_amt) nounwind {
+; CHECK-LABEL: define void @simple_urem_fail_bad_loop(
+; CHECK-SAME: i32 [[N:%.*]], i32 [[REM_AMT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @get.i32()
+; CHECK-NEXT:    [[TOBOOL_NOT:%.*]] = icmp eq i32 [[CALL]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL_NOT]], label %[[FOR_COND:.*]], label %[[HALFWAY:.*]]
+; CHECK:       [[FOR_COND]]:
+; CHECK-NEXT:    [[I_0:%.*]] = phi i32 [ [[INC:%.*]], %[[HALFWAY]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[I_0]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END:.*]]
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[I_0]], 1
+; CHECK-NEXT:    call void @use.i32(i32 [[XOR]])
+; CHECK-NEXT:    br label %[[HALFWAY]]
+; CHECK:       [[HALFWAY]]:
+; CHECK-NEXT:    [[I_1:%.*]] = phi i32 [ poison, %[[ENTRY]] ], [ [[I_0]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[REM:%.*]] = urem i32 [[I_1]], [[REM_AMT]]
+; CHECK-NEXT:    call void @use.i32(i32 [[REM]])
+; CHECK-NEXT:    [[INC]] = add nuw i32 [[I_1]], 1
+; CHECK-NEXT:    br label %[[FOR_COND]]
+; CHECK:       [[FOR_END]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %call = call i32 @get.i32()
+  %tobool.not = icmp eq i32 %call, 0
+  br i1 %tobool.not, label %for.cond, label %halfway
+
+for.cond:
+  %i.0 = phi i32 [ %inc, %halfway ], [ 0, %entry ]
+  %cmp = icmp ult i32 %i.0, %N
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:
+  %xor = xor i32 %i.0, 1
+  call void @use.i32(i32 %xor)
+  br label %halfway
+
+halfway:
+  %i.1 = phi i32 [ poison, %entry ], [ %i.0, %for.body ]
+  %rem = urem i32 %i.1, %rem_amt
+  call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.1, 1
+  br label %for.cond
+
+for.end:
+  ret void
+}
+
+define void @simple_urem_fail_intermediate_inc(i32 %N, i32 %rem_amt) nounwind {
+; CHECK-LABEL: define void @simple_urem_fail_intermediate_inc(
+; CHECK-SAME: i32 [[N:%.*]], i32 [[REM_AMT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[CMP3_NOT:%.*]] = icmp eq i32 [[N]], 0
+; CHECK-NEXT:    br i1 [[CMP3_NOT]], label %[[FOR_COND_CLEANUP:.*]], label %[[FOR_BODY_PREHEADER:.*]]
+; CHECK:       [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT:    ret void
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[I_04:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[INC2:%.*]] = add nuw i32 [[I_04]], 1
+; CHECK-NEXT:    [[REM:%.*]] = urem i32 [[INC2]], [[REM_AMT]]
+; CHECK-NEXT:    tail call void @use.i32(i32 [[REM]])
+; CHECK-NEXT:    [[INC]] = add nuw i32 [[I_04]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+  %cmp3.not = icmp eq i32 %N, 0
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ 0, %entry ]
+  %inc2 = add nuw i32 %i.04, 1
+  %rem = urem i32 %inc2, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @weird_loop(i64 %sub.ptr.div.i56) personality ptr null {
+; CHECK-LABEL: define void @weird_loop(
+; CHECK-SAME: i64 [[SUB_PTR_DIV_I56:%.*]]) personality ptr null {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[ADD74_US:%.*]] = add nuw i64 0, 1
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[REM_US:%.*]] = urem i64 0, [[SUB_PTR_DIV_I56]]
+; CHECK-NEXT:    br label %[[FOR_BODY]]
+;
+entry:
+  br label %for.preheader
+
+for.preheader:
+  %i57.0540.us = phi i64 [ 0, %entry ], [ %add74.us, %for.body ]
+  %add74.us = add nuw i64 %i57.0540.us, 1
+  br label %for.body
+
+for.body:
+  %rem.us = urem i64 %i57.0540.us, %sub.ptr.div.i56
+  br i1 false, label %for.preheader, label %for.body
+}
+
+define void @simple_urem_to_sel_non_zero_start(i32 %N, i32 %rem_amt) nounwind {
+; CHECK-LABEL: define void @simple_urem_to_sel_non_zero_start(
+; CHECK-SAME: i32 [[N:%.*]], i32 [[REM_AMT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[CMP3_NOT:%.*]] = icmp ult i32 [[N]], 3
+; CHECK-NEXT:    br i1 [[CMP3_NOT]], label %[[FOR_COND_CLEANUP:.*]], label %[[FOR_BODY_PREHEADER:.*]]
+; CHECK:       [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT:    ret void
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[I_04:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 2, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[REM:%.*]] = urem i32 [[I_04]], [[REM_AMT]]
+; CHECK-NEXT:    tail call void @use.i32(i32 [[REM]])
+; CHECK-NEXT:    [[INC]] = add nuw i32 [[I_04]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+  %cmp3.not = icmp ult i32 %N, 3
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ 2, %entry ]
+  %rem = urem i32 %i.04, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_to_sel_non_zero_start_through_add(i32 %N, i32 %rem_amt_in) nounwind {
+; CHECK-LABEL: define void @simple_urem_to_sel_non_zero_start_through_add(
+; CHECK-SAME: i32 [[N:%.*]], i32 [[REM_AMT_IN:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[REM_AMT:%.*]] = or i32 [[REM_AMT_IN]], 16
+; CHECK-NEXT:    [[CMP3_NOT:%.*]] = icmp ult i32 [[N]], 3
+; CHECK-NEXT:    br i1 [[CMP3_NOT]], label %[[FOR_COND_CLEANUP:.*]], label %[[FOR_BODY_PREHEADER:.*]]
+; CHECK:       [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT:    ret void
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[I_04:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 2, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[I_WITH_OFF:%.*]] = add nuw i32 [[I_04]], 5
+; CHECK-NEXT:    [[REM:%.*]] = urem i32 [[I_WITH_OFF]], [[REM_AMT]]
+; CHECK-NEXT:    tail call void @use.i32(i32 [[REM]])
+; CHECK-NEXT:    [[INC]] = add nuw i32 [[I_04]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+  %rem_amt = or i32 %rem_amt_in, 16
+  %cmp3.not = icmp ult i32 %N, 3
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ 2, %entry ]
+  %i_with_off = add nuw i32 %i.04, 5
+  %rem = urem i32 %i_with_off, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_to_sel_non_zero_start_through_add_fail_missing_nuw(i32 %N, i32 %rem_amt_in) nounwind {
+; CHECK-LABEL: define void @simple_urem_to_sel_non_zero_start_through_add_fail_missing_nuw(
+; CHECK-SAME: i32 [[N:%.*]], i32 [[REM_AMT_IN:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[REM_AMT:%.*]] = or i32 [[REM_AMT_IN]], 16
+; CHECK-NEXT:    [[CMP3_NOT:%.*]] = icmp ult i32 [[N]], 3
+; CHECK-NEXT:    br i1 [[CMP3_NOT]], label %[[FOR_COND_CLEANUP:.*]], label %[[FOR_BODY_PREHEADER:.*]]
+; CHECK:       [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT:    ret void
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[I_04:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 2, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[I_WITH_OFF:%.*]] = add i32 [[I_04]], 5
+; CHECK-NEXT:    [[REM:%.*]] = urem i32 [[I_WITH_OFF]], [[REM_AMT]]
+; CHECK-NEXT:    tail call void @use.i32(i32 [[REM]])
+; CHECK-NEXT:    [[INC]] = add nuw i32 [[I_04]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+  %rem_amt = or i32 %rem_amt_in, 16
+  %cmp3.not = icmp ult i32 %N, 3
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ 2, %entry ]
+  %i_with_off = add i32 %i.04, 5
+  %rem = urem i32 %i_with_off, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_to_sel_non_zero_start_through_add_fail_no_simplify_rem(i32 %N, i32 %rem_amt) nounwind {
+; CHECK-LABEL: define void @simple_urem_to_sel_non_zero_start_through_add_fail_no_simplify_rem(
+; CHECK-SAME: i32 [[N:%.*]], i32 [[REM_AMT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[CMP3_NOT:%.*]] = icmp ult i32 [[N]], 3
+; CHECK-NEXT:    br i1 [[CMP3_NOT]], label %[[FOR_COND_CLEANUP:.*]], label %[[FOR_BODY_PREHEADER:.*]]
+; CHECK:       [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT:    ret void
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[I_04:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 2, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[I_WITH_OFF:%.*]] = add nuw i32 [[I_04]], 5
+; CHECK-NEXT:    [[REM:%.*]] = urem i32 [[I_WITH_OFF]], [[REM_AMT]]
+; CHECK-NEXT:    tail call void @use.i32(i32 [[REM]])
+; CHECK-NEXT:    [[INC]] = add nuw i32 [[I_04]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+  %cmp3.not = icmp ult i32 %N, 3
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ 2, %entry ]
+  %i_with_off = add nuw i32 %i.04, 5
+  %rem = urem i32 %i_with_off, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_to_sel_non_zero_start_through_sub(i32 %N, i32 %rem_amt, i32 %start) nounwind {
+; CHECK-LABEL: define void @simple_urem_to_sel_non_zero_start_through_sub(
+; CHECK-SAME: i32 [[N:%.*]], i32 [[REM_AMT:%.*]], i32 [[START:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[CMP3_NOT:%.*]] = icmp ule i32 [[N]], [[START]]
+; CHECK-NEXT:    br i1 [[CMP3_NOT]], label %[[FOR_COND_CLEANUP:.*]], label %[[FOR_BODY_PREHEADER:.*]]
+; CHECK:       [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT:    ret void
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[I_04:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[START]], %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[I_WITH_OFF:%.*]] = sub nuw i32 [[I_04]], [[START]]
+; CHECK-NEXT:    [[REM:%.*]] = urem i32 [[I_WITH_OFF]], [[REM_AMT]]
+; CHECK-NEXT:    tail call void @use.i32(i32 [[REM]])
+; CHECK-NEXT:    [[INC]] = add nuw i32 [[I_04]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+  %cmp3.not = icmp ule i32 %N, %start
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ %start, %entry ]
+  %i_with_off = sub nuw i32 %i.04, %start
+  %rem = urem i32 %i_with_off, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+define void @simple_urem_to_sel_non_zero_start_through_sub_no_simplfy(i32 %N, i32 %rem_amt, i32 %start) nounwind {
+; CHECK-LABEL: define void @simple_urem_to_sel_non_zero_start_through_sub_no_simplfy(
+; CHECK-SAME: i32 [[N:%.*]], i32 [[REM_AMT:%.*]], i32 [[START:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[CMP3_NOT:%.*]] = icmp ule i32 [[N]], [[START]]
+; CHECK-NEXT:    br i1 [[CMP3_NOT]], label %[[FOR_COND_CLEANUP:.*]], label %[[FOR_BODY_PREHEADER:.*]]
+; CHECK:       [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT:    ret void
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[I_04:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[START]], %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[I_WITH_OFF:%.*]] = sub nuw i32 [[I_04]], 2
+; CHECK-NEXT:    [[REM:%.*]] = urem i32 [[I_WITH_OFF]], [[REM_AMT]]
+; CHECK-NEXT:    tail call void @use.i32(i32 [[REM]])
+; CHECK-NEXT:    [[INC]] = add nuw i32 [[I_04]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
+;
+entry:
+  %cmp3.not = icmp ule i32 %N, %start
+  br i1 %cmp3.not, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.04 = phi i32 [ %inc, %for.body ], [ %start, %entry ]
+  %i_with_off = sub nuw i32 %i.04, 2
+  %rem = urem i32 %i_with_off, %rem_amt
+  tail call void @use.i32(i32 %rem)
+  %inc = add nuw i32 %i.04, 1
+  %exitcond.not = icmp eq i32 %inc, %N
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}

From c64ce8bf283120fd145a57d0e61f9697f719139d Mon Sep 17 00:00:00 2001
From: Noah Goldstein <goldstein.w.n@gmail.com>
Date: Tue, 25 Jun 2024 19:51:39 +0800
Subject: [PATCH 317/441] [CodeGenPrepare] Folding `urem` with loop invariant
 value

```
for(i = Start; i < End; ++i)
   Rem = (i nuw+ IncrLoopInvariant) u% RemAmtLoopInvariant;
```
 ->
```
Rem = (Start nuw+ IncrLoopInvariant) % RemAmtLoopInvariant;
for(i = Start; i < End; ++i, ++rem)
   Rem = rem == RemAmtLoopInvariant ? 0 : Rem;
```

In its current state, only if `IncrLoopInvariant` and `Start` both
being zero.

Alive2 seemed unable to prove this (see:
https://alive2.llvm.org/ce/z/ATGDp3 which is clearly wrong but still
checks out...) so wrote an exhaustive test here:
https://godbolt.org/z/WYa561388

Closes #96625
---
 llvm/lib/CodeGen/CodeGenPrepare.cpp           | 131 +++++++++++++++++
 llvm/test/CodeGen/X86/fold-loop-of-urem.ll    | 138 +++++++++++-------
 .../CodeGenPrepare/X86/fold-loop-of-urem.ll   |  37 ++++-
 3 files changed, 245 insertions(+), 61 deletions(-)

diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp
index 48253a613b41d2b..72db1f4adabf76f 100644
--- a/llvm/lib/CodeGen/CodeGenPrepare.cpp
+++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp
@@ -472,6 +472,7 @@ class CodeGenPrepare {
   bool replaceMathCmpWithIntrinsic(BinaryOperator *BO, Value *Arg0, Value *Arg1,
                                    CmpInst *Cmp, Intrinsic::ID IID);
   bool optimizeCmp(CmpInst *Cmp, ModifyDT &ModifiedDT);
+  bool optimizeURem(Instruction *Rem);
   bool combineToUSubWithOverflow(CmpInst *Cmp, ModifyDT &ModifiedDT);
   bool combineToUAddWithOverflow(CmpInst *Cmp, ModifyDT &ModifiedDT);
   void verifyBFIUpdates(Function &F);
@@ -1975,6 +1976,132 @@ static bool foldFCmpToFPClassTest(CmpInst *Cmp, const TargetLowering &TLI,
   return true;
 }
 
+static bool isRemOfLoopIncrementWithLoopInvariant(Instruction *Rem,
+                                                  const LoopInfo *LI,
+                                                  Value *&RemAmtOut,
+                                                  PHINode *&LoopIncrPNOut) {
+  Value *Incr, *RemAmt;
+  // NB: If RemAmt is a power of 2 it *should* have been transformed by now.
+  if (!match(Rem, m_URem(m_Value(Incr), m_Value(RemAmt))))
+    return false;
+
+  // Find out loop increment PHI.
+  auto *PN = dyn_cast<PHINode>(Incr);
+  if (!PN)
+    return false;
+
+  // This isn't strictly necessary, what we really need is one increment and any
+  // amount of initial values all being the same.
+  if (PN->getNumIncomingValues() != 2)
+    return false;
+
+  // Only trivially analyzable loops.
+  Loop *L = LI->getLoopFor(Rem->getParent());
+  if (!L || !L->getLoopPreheader() || !L->getLoopLatch())
+    return false;
+
+  // Only works if the remainder amount is a loop invaraint
+  if (!L->isLoopInvariant(RemAmt))
+    return false;
+
+  // Is the PHI a loop increment?
+  auto LoopIncrInfo = getIVIncrement(PN, LI);
+  if (!LoopIncrInfo)
+    return false;
+
+  // getIVIncrement finds the loop at PN->getParent(). This might be a different
+  // loop from the loop with Rem->getParent().
+  if (L->getHeader() != PN->getParent())
+    return false;
+
+  // We need remainder_amount % increment_amount to be zero. Increment of one
+  // satisfies that without any special logic and is overwhelmingly the common
+  // case.
+  if (!match(LoopIncrInfo->second, m_One()))
+    return false;
+
+  // Need the increment to not overflow.
+  if (!match(LoopIncrInfo->first, m_NUWAdd(m_Value(), m_Value())))
+    return false;
+
+  // Set output variables.
+  RemAmtOut = RemAmt;
+  LoopIncrPNOut = PN;
+
+  return true;
+}
+
+// Try to transform:
+//
+// for(i = Start; i < End; ++i)
+//    Rem = (i nuw+ IncrLoopInvariant) u% RemAmtLoopInvariant;
+//
+// ->
+//
+// Rem = (Start nuw+ IncrLoopInvariant) % RemAmtLoopInvariant;
+// for(i = Start; i < End; ++i, ++rem)
+//    Rem = rem == RemAmtLoopInvariant ? 0 : Rem;
+//
+// Currently only implemented for `IncrLoopInvariant` being zero.
+static bool foldURemOfLoopIncrement(Instruction *Rem, const DataLayout *DL,
+                                    const LoopInfo *LI,
+                                    SmallSet<BasicBlock *, 32> &FreshBBs,
+                                    bool IsHuge) {
+  Value *RemAmt;
+  PHINode *LoopIncrPN;
+  if (!isRemOfLoopIncrementWithLoopInvariant(Rem, LI, RemAmt, LoopIncrPN))
+    return false;
+
+  // Only non-constant remainder as the extra IV is probably not profitable
+  // in that case.
+  //
+  // Potential TODO(1): `urem` of a const ends up as `mul` + `shift` + `add`. If
+  // we can rule out register pressure and ensure this `urem` is executed each
+  // iteration, its probably profitable to handle the const case as well.
+  //
+  // Potential TODO(2): Should we have a check for how "nested" this remainder
+  // operation is? The new code runs every iteration so if the remainder is
+  // guarded behind unlikely conditions this might not be worth it.
+  if (match(RemAmt, m_ImmConstant()))
+    return false;
+  Loop *L = LI->getLoopFor(Rem->getParent());
+
+  Value *Start = LoopIncrPN->getIncomingValueForBlock(L->getLoopPreheader());
+
+  // Create new remainder with induction variable.
+  Type *Ty = Rem->getType();
+  IRBuilder<> Builder(Rem->getContext());
+
+  Builder.SetInsertPoint(LoopIncrPN);
+  PHINode *NewRem = Builder.CreatePHI(Ty, 2);
+
+  Builder.SetInsertPoint(cast<Instruction>(
+      LoopIncrPN->getIncomingValueForBlock(L->getLoopLatch())));
+  // `(add (urem x, y), 1)` is always nuw.
+  Value *RemAdd = Builder.CreateNUWAdd(NewRem, ConstantInt::get(Ty, 1));
+  Value *RemCmp = Builder.CreateICmp(ICmpInst::ICMP_EQ, RemAdd, RemAmt);
+  Value *RemSel =
+      Builder.CreateSelect(RemCmp, Constant::getNullValue(Ty), RemAdd);
+
+  NewRem->addIncoming(Start, L->getLoopPreheader());
+  NewRem->addIncoming(RemSel, L->getLoopLatch());
+
+  // Insert all touched BBs.
+  FreshBBs.insert(LoopIncrPN->getParent());
+  FreshBBs.insert(L->getLoopLatch());
+  FreshBBs.insert(Rem->getParent());
+
+  replaceAllUsesWith(Rem, NewRem, FreshBBs, IsHuge);
+  Rem->eraseFromParent();
+  return true;
+}
+
+bool CodeGenPrepare::optimizeURem(Instruction *Rem) {
+  if (foldURemOfLoopIncrement(Rem, DL, LI, FreshBBs, IsHugeFunc))
+    return true;
+  return false;
+}
+
 bool CodeGenPrepare::optimizeCmp(CmpInst *Cmp, ModifyDT &ModifiedDT) {
   if (sinkCmpExpression(Cmp, *TLI))
     return true;
@@ -8358,6 +8485,10 @@ bool CodeGenPrepare::optimizeInst(Instruction *I, ModifyDT &ModifiedDT) {
     if (optimizeCmp(Cmp, ModifiedDT))
       return true;
 
+  if (match(I, m_URem(m_Value(), m_Value())))
+    if (optimizeURem(I))
+      return true;
+
   if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
     LI->setMetadata(LLVMContext::MD_invariant_group, nullptr);
     bool Modified = optimizeLoadExt(LI);
diff --git a/llvm/test/CodeGen/X86/fold-loop-of-urem.ll b/llvm/test/CodeGen/X86/fold-loop-of-urem.ll
index aad2e0dd7bd248a..b9fe4d7c79c7fc2 100644
--- a/llvm/test/CodeGen/X86/fold-loop-of-urem.ll
+++ b/llvm/test/CodeGen/X86/fold-loop-of-urem.ll
@@ -15,25 +15,31 @@ define void @simple_urem_to_sel(i32 %N, i32 %rem_amt) nounwind {
 ; CHECK-NEXT:    je .LBB0_4
 ; CHECK-NEXT:  # %bb.1: # %for.body.preheader
 ; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    pushq %r15
 ; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %r12
 ; CHECK-NEXT:    pushq %rbx
 ; CHECK-NEXT:    movl %esi, %ebx
 ; CHECK-NEXT:    movl %edi, %ebp
+; CHECK-NEXT:    xorl %r15d, %r15d
 ; CHECK-NEXT:    xorl %r14d, %r14d
+; CHECK-NEXT:    xorl %r12d, %r12d
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  .LBB0_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    movl %r14d, %eax
-; CHECK-NEXT:    xorl %edx, %edx
-; CHECK-NEXT:    divl %ebx
-; CHECK-NEXT:    movl %edx, %edi
+; CHECK-NEXT:    movl %r14d, %edi
 ; CHECK-NEXT:    callq use.i32@PLT
 ; CHECK-NEXT:    incl %r14d
-; CHECK-NEXT:    cmpl %r14d, %ebp
+; CHECK-NEXT:    cmpl %ebx, %r14d
+; CHECK-NEXT:    cmovel %r15d, %r14d
+; CHECK-NEXT:    incl %r12d
+; CHECK-NEXT:    cmpl %r12d, %ebp
 ; CHECK-NEXT:    jne .LBB0_2
 ; CHECK-NEXT:  # %bb.3:
 ; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r12
 ; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %r15
 ; CHECK-NEXT:    popq %rbp
 ; CHECK-NEXT:  .LBB0_4: # %for.cond.cleanup
 ; CHECK-NEXT:    retq
@@ -60,24 +66,28 @@ define void @simple_urem_to_sel_nested2(i32 %N, i32 %rem_amt) nounwind {
 ; CHECK-NEXT:    je .LBB1_8
 ; CHECK-NEXT:  # %bb.1: # %for.body.preheader
 ; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    pushq %r15
 ; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %r12
 ; CHECK-NEXT:    pushq %rbx
 ; CHECK-NEXT:    movl %esi, %ebx
 ; CHECK-NEXT:    movl %edi, %ebp
+; CHECK-NEXT:    xorl %r15d, %r15d
 ; CHECK-NEXT:    xorl %r14d, %r14d
+; CHECK-NEXT:    xorl %r12d, %r12d
 ; CHECK-NEXT:    jmp .LBB1_2
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  .LBB1_5: # %for.body1
 ; CHECK-NEXT:    # in Loop: Header=BB1_2 Depth=1
-; CHECK-NEXT:    movl %r14d, %eax
-; CHECK-NEXT:    xorl %edx, %edx
-; CHECK-NEXT:    divl %ebx
-; CHECK-NEXT:    movl %edx, %edi
+; CHECK-NEXT:    movl %r14d, %edi
 ; CHECK-NEXT:    callq use.i32@PLT
 ; CHECK-NEXT:  .LBB1_6: # %for.body.tail
 ; CHECK-NEXT:    # in Loop: Header=BB1_2 Depth=1
 ; CHECK-NEXT:    incl %r14d
-; CHECK-NEXT:    cmpl %r14d, %ebp
+; CHECK-NEXT:    cmpl %ebx, %r14d
+; CHECK-NEXT:    cmovel %r15d, %r14d
+; CHECK-NEXT:    incl %r12d
+; CHECK-NEXT:    cmpl %r12d, %ebp
 ; CHECK-NEXT:    je .LBB1_7
 ; CHECK-NEXT:  .LBB1_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
@@ -97,7 +107,9 @@ define void @simple_urem_to_sel_nested2(i32 %N, i32 %rem_amt) nounwind {
 ; CHECK-NEXT:    jmp .LBB1_6
 ; CHECK-NEXT:  .LBB1_7:
 ; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r12
 ; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %r15
 ; CHECK-NEXT:    popq %rbp
 ; CHECK-NEXT:  .LBB1_8: # %for.cond.cleanup
 ; CHECK-NEXT:    retq
@@ -213,40 +225,36 @@ for.body.tail:
 define void @simple_urem_to_sel_vec(<2 x i64> %rem_amt) nounwind {
 ; CHECK-LABEL: simple_urem_to_sel_vec:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    pushq %r14
-; CHECK-NEXT:    pushq %rbx
-; CHECK-NEXT:    subq $24, %rsp
+; CHECK-NEXT:    subq $56, %rsp
+; CHECK-NEXT:    movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; CHECK-NEXT:    pxor %xmm1, %xmm1
-; CHECK-NEXT:    movdqa %xmm1, (%rsp) # 16-byte Spill
-; CHECK-NEXT:    movq %xmm0, %rbx
-; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
-; CHECK-NEXT:    movq %xmm0, %r14
+; CHECK-NEXT:    pxor %xmm0, %xmm0
+; CHECK-NEXT:    movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  .LBB3_1: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    movdqa (%rsp), %xmm1 # 16-byte Reload
 ; CHECK-NEXT:    movdqa %xmm1, (%rsp) # 16-byte Spill
-; CHECK-NEXT:    movq %xmm1, %rax
-; CHECK-NEXT:    xorl %edx, %edx
-; CHECK-NEXT:    divq %rbx
-; CHECK-NEXT:    movq %rdx, %xmm0
-; CHECK-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
-; CHECK-NEXT:    movq %xmm1, %rax
-; CHECK-NEXT:    xorl %edx, %edx
-; CHECK-NEXT:    divq %r14
-; CHECK-NEXT:    movq %rdx, %xmm1
-; CHECK-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; CHECK-NEXT:    movdqa %xmm1, %xmm0
 ; CHECK-NEXT:    callq use.2xi64@PLT
-; CHECK-NEXT:    movdqa (%rsp), %xmm0 # 16-byte Reload
-; CHECK-NEXT:    psubq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; CHECK-NEXT:    movdqa %xmm0, (%rsp) # 16-byte Spill
+; CHECK-NEXT:    pcmpeqd %xmm1, %xmm1
+; CHECK-NEXT:    movdqa (%rsp), %xmm2 # 16-byte Reload
+; CHECK-NEXT:    psubq %xmm1, %xmm2
+; CHECK-NEXT:    movdqa %xmm2, %xmm0
+; CHECK-NEXT:    movdqa %xmm2, %xmm3
+; CHECK-NEXT:    pcmpeqd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,0,3,2]
+; CHECK-NEXT:    pand %xmm0, %xmm2
+; CHECK-NEXT:    pandn %xmm3, %xmm2
+; CHECK-NEXT:    movdqa %xmm2, (%rsp) # 16-byte Spill
+; CHECK-NEXT:    movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    psubq %xmm1, %xmm0
+; CHECK-NEXT:    movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; CHECK-NEXT:    callq get.i1@PLT
 ; CHECK-NEXT:    testb $1, %al
+; CHECK-NEXT:    movdqa (%rsp), %xmm1 # 16-byte Reload
 ; CHECK-NEXT:    je .LBB3_1
 ; CHECK-NEXT:  # %bb.2: # %for.cond.cleanup
-; CHECK-NEXT:    addq $24, %rsp
-; CHECK-NEXT:    popq %rbx
-; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    addq $56, %rsp
 ; CHECK-NEXT:    retq
 entry:
   br label %for.body
@@ -336,27 +344,33 @@ define void @simple_urem_to_sel_second_acc(i32 %N, i32 %rem_amt) nounwind {
 ; CHECK-NEXT:    pushq %rbp
 ; CHECK-NEXT:    pushq %r15
 ; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    pushq %r12
 ; CHECK-NEXT:    pushq %rbx
 ; CHECK-NEXT:    pushq %rax
 ; CHECK-NEXT:    movl %esi, %ebx
 ; CHECK-NEXT:    movl %edi, %ebp
 ; CHECK-NEXT:    movl $1, %r15d
+; CHECK-NEXT:    xorl %r12d, %r12d
 ; CHECK-NEXT:    xorl %r14d, %r14d
+; CHECK-NEXT:    xorl %r13d, %r13d
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  .LBB5_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    movl %r14d, %eax
-; CHECK-NEXT:    xorl %edx, %edx
-; CHECK-NEXT:    divl %ebx
-; CHECK-NEXT:    movl %edx, %edi
+; CHECK-NEXT:    movl %r14d, %edi
 ; CHECK-NEXT:    callq use.i32@PLT
 ; CHECK-NEXT:    incl %r14d
+; CHECK-NEXT:    cmpl %ebx, %r14d
+; CHECK-NEXT:    cmovel %r12d, %r14d
+; CHECK-NEXT:    incl %r13d
 ; CHECK-NEXT:    addl $2, %r15d
 ; CHECK-NEXT:    cmpl %ebp, %r15d
 ; CHECK-NEXT:    jbe .LBB5_2
 ; CHECK-NEXT:  # %bb.3:
 ; CHECK-NEXT:    addq $8, %rsp
 ; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    popq %r13
 ; CHECK-NEXT:    popq %r14
 ; CHECK-NEXT:    popq %r15
 ; CHECK-NEXT:    popq %rbp
@@ -676,23 +690,27 @@ define void @simple_urem_multi_latch_non_canonical(i32 %N, i32 %rem_amt) nounwin
 ; CHECK-NEXT:    pushq %rbp
 ; CHECK-NEXT:    pushq %r15
 ; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    pushq %r12
 ; CHECK-NEXT:    pushq %rbx
 ; CHECK-NEXT:    pushq %rax
 ; CHECK-NEXT:    movl %esi, %ebx
 ; CHECK-NEXT:    movl %edi, %ebp
 ; CHECK-NEXT:    decl %ebp
+; CHECK-NEXT:    xorl %r12d, %r12d
 ; CHECK-NEXT:    xorl %r14d, %r14d
+; CHECK-NEXT:    xorl %r13d, %r13d
 ; CHECK-NEXT:    jmp .LBB12_2
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  .LBB12_3: # %for.body.backedge
 ; CHECK-NEXT:    # in Loop: Header=BB12_2 Depth=1
 ; CHECK-NEXT:    incl %r14d
+; CHECK-NEXT:    cmpl %ebx, %r14d
+; CHECK-NEXT:    cmovel %r12d, %r14d
+; CHECK-NEXT:    incl %r13d
 ; CHECK-NEXT:  .LBB12_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    movl %r14d, %eax
-; CHECK-NEXT:    xorl %edx, %edx
-; CHECK-NEXT:    divl %ebx
-; CHECK-NEXT:    movl %edx, %edi
+; CHECK-NEXT:    movl %r14d, %edi
 ; CHECK-NEXT:    callq use.i32@PLT
 ; CHECK-NEXT:    callq get.i1@PLT
 ; CHECK-NEXT:    movl %eax, %r15d
@@ -702,11 +720,13 @@ define void @simple_urem_multi_latch_non_canonical(i32 %N, i32 %rem_amt) nounwin
 ; CHECK-NEXT:  # %bb.4: # %for.body0
 ; CHECK-NEXT:    # in Loop: Header=BB12_2 Depth=1
 ; CHECK-NEXT:    callq do_stuff1@PLT
-; CHECK-NEXT:    cmpl %r14d, %ebp
+; CHECK-NEXT:    cmpl %r13d, %ebp
 ; CHECK-NEXT:    jne .LBB12_3
 ; CHECK-NEXT:  # %bb.5:
 ; CHECK-NEXT:    addq $8, %rsp
 ; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    popq %r13
 ; CHECK-NEXT:    popq %r14
 ; CHECK-NEXT:    popq %r15
 ; CHECK-NEXT:    popq %rbp
@@ -870,25 +890,31 @@ define void @simple_urem_to_sel_non_zero_start(i32 %N, i32 %rem_amt) nounwind {
 ; CHECK-NEXT:    jb .LBB16_4
 ; CHECK-NEXT:  # %bb.1: # %for.body.preheader
 ; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    pushq %r15
 ; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %r12
 ; CHECK-NEXT:    pushq %rbx
 ; CHECK-NEXT:    movl %esi, %ebx
 ; CHECK-NEXT:    movl %edi, %ebp
 ; CHECK-NEXT:    movl $2, %r14d
+; CHECK-NEXT:    xorl %r15d, %r15d
+; CHECK-NEXT:    movl $2, %r12d
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  .LBB16_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    movl %r14d, %eax
-; CHECK-NEXT:    xorl %edx, %edx
-; CHECK-NEXT:    divl %ebx
-; CHECK-NEXT:    movl %edx, %edi
+; CHECK-NEXT:    movl %r14d, %edi
 ; CHECK-NEXT:    callq use.i32@PLT
 ; CHECK-NEXT:    incl %r14d
-; CHECK-NEXT:    cmpl %r14d, %ebp
+; CHECK-NEXT:    cmpl %ebx, %r14d
+; CHECK-NEXT:    cmovel %r15d, %r14d
+; CHECK-NEXT:    incl %r12d
+; CHECK-NEXT:    cmpl %r12d, %ebp
 ; CHECK-NEXT:    jne .LBB16_2
 ; CHECK-NEXT:  # %bb.3:
 ; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r12
 ; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %r15
 ; CHECK-NEXT:    popq %rbp
 ; CHECK-NEXT:  .LBB16_4: # %for.cond.cleanup
 ; CHECK-NEXT:    retq
@@ -1066,28 +1092,34 @@ define void @simple_urem_to_sel_non_zero_start_through_sub(i32 %N, i32 %rem_amt,
 ; CHECK-LABEL: simple_urem_to_sel_non_zero_start_through_sub:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    pushq %r15
 ; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %r12
 ; CHECK-NEXT:    pushq %rbx
 ; CHECK-NEXT:    movl %edi, %ebp
 ; CHECK-NEXT:    subl %edx, %ebp
 ; CHECK-NEXT:    jbe .LBB20_3
 ; CHECK-NEXT:  # %bb.1: # %for.body.preheader
 ; CHECK-NEXT:    movl %esi, %ebx
+; CHECK-NEXT:    xorl %r15d, %r15d
 ; CHECK-NEXT:    xorl %r14d, %r14d
+; CHECK-NEXT:    xorl %r12d, %r12d
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  .LBB20_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    movl %r14d, %eax
-; CHECK-NEXT:    xorl %edx, %edx
-; CHECK-NEXT:    divl %ebx
-; CHECK-NEXT:    movl %edx, %edi
+; CHECK-NEXT:    movl %r14d, %edi
 ; CHECK-NEXT:    callq use.i32@PLT
 ; CHECK-NEXT:    incl %r14d
-; CHECK-NEXT:    cmpl %r14d, %ebp
+; CHECK-NEXT:    cmpl %ebx, %r14d
+; CHECK-NEXT:    cmovel %r15d, %r14d
+; CHECK-NEXT:    incl %r12d
+; CHECK-NEXT:    cmpl %r12d, %ebp
 ; CHECK-NEXT:    jne .LBB20_2
 ; CHECK-NEXT:  .LBB20_3: # %for.cond.cleanup
 ; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r12
 ; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %r15
 ; CHECK-NEXT:    popq %rbp
 ; CHECK-NEXT:    retq
 entry:
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/fold-loop-of-urem.ll b/llvm/test/Transforms/CodeGenPrepare/X86/fold-loop-of-urem.ll
index a019679e65905de..304ae337ed41973 100644
--- a/llvm/test/Transforms/CodeGenPrepare/X86/fold-loop-of-urem.ll
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/fold-loop-of-urem.ll
@@ -19,9 +19,12 @@ define void @simple_urem_to_sel(i32 %N, i32 %rem_amt) nounwind {
 ; CHECK:       [[FOR_COND_CLEANUP]]:
 ; CHECK-NEXT:    ret void
 ; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[REM:%.*]] = phi i32 [ 0, %[[FOR_BODY_PREHEADER]] ], [ [[TMP3:%.*]], %[[FOR_BODY]] ]
 ; CHECK-NEXT:    [[I_04:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT:    [[REM:%.*]] = urem i32 [[I_04]], [[REM_AMT]]
 ; CHECK-NEXT:    tail call void @use.i32(i32 [[REM]])
+; CHECK-NEXT:    [[TMP1:%.*]] = add nuw i32 [[REM]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], [[REM_AMT]]
+; CHECK-NEXT:    [[TMP3]] = select i1 [[TMP2]], i32 0, i32 [[TMP1]]
 ; CHECK-NEXT:    [[INC]] = add nuw i32 [[I_04]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
 ; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
@@ -53,7 +56,8 @@ define void @simple_urem_to_sel_nested2(i32 %N, i32 %rem_amt) nounwind {
 ; CHECK:       [[FOR_COND_CLEANUP]]:
 ; CHECK-NEXT:    ret void
 ; CHECK:       [[FOR_BODY]]:
-; CHECK-NEXT:    [[I_04:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY_TAIL:.*]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[REM:%.*]] = phi i32 [ 0, %[[FOR_BODY_PREHEADER]] ], [ [[TMP3:%.*]], %[[FOR_BODY_TAIL:.*]] ]
+; CHECK-NEXT:    [[I_04:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY_TAIL]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
 ; CHECK-NEXT:    [[COND0:%.*]] = call i1 @get.i1()
 ; CHECK-NEXT:    br i1 [[COND0]], label %[[FOR_BODY0:.*]], label %[[FOR_BODY_TAIL]]
 ; CHECK:       [[FOR_BODY0]]:
@@ -63,10 +67,12 @@ define void @simple_urem_to_sel_nested2(i32 %N, i32 %rem_amt) nounwind {
 ; CHECK-NEXT:    [[COND2:%.*]] = call i1 @get.i1()
 ; CHECK-NEXT:    br i1 [[COND2]], label %[[FOR_BODY1]], label %[[FOR_BODY_TAIL]]
 ; CHECK:       [[FOR_BODY1]]:
-; CHECK-NEXT:    [[REM:%.*]] = urem i32 [[I_04]], [[REM_AMT]]
 ; CHECK-NEXT:    tail call void @use.i32(i32 [[REM]])
 ; CHECK-NEXT:    br label %[[FOR_BODY_TAIL]]
 ; CHECK:       [[FOR_BODY_TAIL]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = add nuw i32 [[REM]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], [[REM_AMT]]
+; CHECK-NEXT:    [[TMP3]] = select i1 [[TMP2]], i32 0, i32 [[TMP1]]
 ; CHECK-NEXT:    [[INC]] = add nuw i32 [[I_04]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
 ; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
@@ -164,9 +170,12 @@ define void @simple_urem_to_sel_vec(<2 x i64> %rem_amt) nounwind {
 ; CHECK:       [[FOR_COND_CLEANUP:.*]]:
 ; CHECK-NEXT:    ret void
 ; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[REM:%.*]] = phi <2 x i64> [ zeroinitializer, %[[ENTRY]] ], [ [[TMP3:%.*]], %[[FOR_BODY]] ]
 ; CHECK-NEXT:    [[I_04:%.*]] = phi <2 x i64> [ [[INC:%.*]], %[[FOR_BODY]] ], [ zeroinitializer, %[[ENTRY]] ]
-; CHECK-NEXT:    [[REM:%.*]] = urem <2 x i64> [[I_04]], [[REM_AMT]]
 ; CHECK-NEXT:    tail call void @use.2xi64(<2 x i64> [[REM]])
+; CHECK-NEXT:    [[TMP1:%.*]] = add nuw <2 x i64> [[REM]], <i64 1, i64 1>
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq <2 x i64> [[TMP1]], [[REM_AMT]]
+; CHECK-NEXT:    [[TMP3]] = select <2 x i1> [[TMP2]], <2 x i64> zeroinitializer, <2 x i64> [[TMP1]]
 ; CHECK-NEXT:    [[INC]] = add nuw <2 x i64> [[I_04]], <i64 1, i64 1>
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = call i1 @get.i1()
 ; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
@@ -246,10 +255,13 @@ define void @simple_urem_to_sel_second_acc(i32 %N, i32 %rem_amt) nounwind {
 ; CHECK:       [[FOR_COND_CLEANUP]]:
 ; CHECK-NEXT:    ret void
 ; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[REM:%.*]] = phi i32 [ 0, %[[FOR_BODY_PREHEADER]] ], [ [[TMP3:%.*]], %[[FOR_BODY]] ]
 ; CHECK-NEXT:    [[I_04:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
 ; CHECK-NEXT:    [[I_05:%.*]] = phi i32 [ [[INC2:%.*]], %[[FOR_BODY]] ], [ 1, %[[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT:    [[REM:%.*]] = urem i32 [[I_04]], [[REM_AMT]]
 ; CHECK-NEXT:    tail call void @use.i32(i32 [[REM]])
+; CHECK-NEXT:    [[TMP1:%.*]] = add nuw i32 [[REM]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], [[REM_AMT]]
+; CHECK-NEXT:    [[TMP3]] = select i1 [[TMP2]], i32 0, i32 [[TMP1]]
 ; CHECK-NEXT:    [[INC]] = add nuw i32 [[I_04]], 1
 ; CHECK-NEXT:    [[INC2]] = add nuw i32 [[I_05]], 2
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp ugt i32 [[INC2]], [[N]]
@@ -386,9 +398,12 @@ define void @simple_urem_non_zero_entry4(i32 %N, i32 %rem_amt) nounwind {
 ; CHECK:       [[FOR_COND_CLEANUP]]:
 ; CHECK-NEXT:    ret void
 ; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[REM:%.*]] = phi i32 [ 4, %[[FOR_BODY_PREHEADER]] ], [ [[TMP3:%.*]], %[[FOR_BODY]] ]
 ; CHECK-NEXT:    [[I_04:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 4, %[[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT:    [[REM:%.*]] = urem i32 [[I_04]], [[REM_AMT]]
 ; CHECK-NEXT:    tail call void @use.i32(i32 [[REM]])
+; CHECK-NEXT:    [[TMP1:%.*]] = add nuw i32 [[REM]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], [[REM_AMT]]
+; CHECK-NEXT:    [[TMP3]] = select i1 [[TMP2]], i32 0, i32 [[TMP1]]
 ; CHECK-NEXT:    [[INC]] = add nuw i32 [[I_04]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
 ; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
@@ -457,9 +472,12 @@ define void @simple_urem_fail_no_preheader_non_canonical(i32 %N, i32 %rem_amt) n
 ; CHECK-NEXT:    [[I_04_PH:%.*]] = phi i32 [ 1, %[[FOR_BODY1]] ], [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
 ; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[REM:%.*]] = phi i32 [ [[I_04_PH]], %[[FOR_BODY_PREHEADER]] ], [ [[TMP3:%.*]], %[[FOR_BODY]] ]
 ; CHECK-NEXT:    [[I_04:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[I_04_PH]], %[[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT:    [[REM:%.*]] = urem i32 [[I_04]], [[REM_AMT]]
 ; CHECK-NEXT:    tail call void @use.i32(i32 [[REM]])
+; CHECK-NEXT:    [[TMP1:%.*]] = add nuw i32 [[REM]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], [[REM_AMT]]
+; CHECK-NEXT:    [[TMP3]] = select i1 [[TMP2]], i32 0, i32 [[TMP1]]
 ; CHECK-NEXT:    [[INC]] = add nuw i32 [[I_04]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
 ; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
@@ -650,9 +668,12 @@ define void @simple_urem_to_sel_non_zero_start(i32 %N, i32 %rem_amt) nounwind {
 ; CHECK:       [[FOR_COND_CLEANUP]]:
 ; CHECK-NEXT:    ret void
 ; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[REM:%.*]] = phi i32 [ 2, %[[FOR_BODY_PREHEADER]] ], [ [[TMP3:%.*]], %[[FOR_BODY]] ]
 ; CHECK-NEXT:    [[I_04:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 2, %[[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT:    [[REM:%.*]] = urem i32 [[I_04]], [[REM_AMT]]
 ; CHECK-NEXT:    tail call void @use.i32(i32 [[REM]])
+; CHECK-NEXT:    [[TMP1:%.*]] = add nuw i32 [[REM]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], [[REM_AMT]]
+; CHECK-NEXT:    [[TMP3]] = select i1 [[TMP2]], i32 0, i32 [[TMP1]]
 ; CHECK-NEXT:    [[INC]] = add nuw i32 [[I_04]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
 ; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]

From 7d5281a66d5d42c65cfb9d95eaf9aa01afb089fb Mon Sep 17 00:00:00 2001
From: Austin Kerbow <Austin.Kerbow@amd.com>
Date: Sun, 18 Aug 2024 17:04:27 -0700
Subject: [PATCH 318/441] [AMDGPU][NFC] Fix preload-kernarg.ll test after
 attributor move (#98840)

Update was to stale version of the test with missing functions and extra
runlines that had been removed.
---
 llvm/test/CodeGen/AMDGPU/preload-kernargs.ll | 3044 ++++++------------
 1 file changed, 953 insertions(+), 2091 deletions(-)

diff --git a/llvm/test/CodeGen/AMDGPU/preload-kernargs.ll b/llvm/test/CodeGen/AMDGPU/preload-kernargs.ll
index a547c258e3921d7..b7113a65607fc56 100644
--- a/llvm/test/CodeGen/AMDGPU/preload-kernargs.ll
+++ b/llvm/test/CodeGen/AMDGPU/preload-kernargs.ll
@@ -1,721 +1,157 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
-; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx940 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX940-NO-PRELOAD %s
-; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx940 -amdgpu-kernarg-preload-count=1 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX940-PRELOAD-1 %s
-; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx940 -amdgpu-kernarg-preload-count=2 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX940-PRELOAD-2 %s
-; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx940 -amdgpu-kernarg-preload-count=4 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX940-PRELOAD-4 %s
-; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx940 -amdgpu-kernarg-preload-count=8 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX940-PRELOAD-8 %s
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx940 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX940 %s
 
-; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90a-NO-PRELOAD %s
-; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx90a -amdgpu-kernarg-preload-count=1 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90a-PRELOAD-1 %s
-; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx90a -amdgpu-kernarg-preload-count=2 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90a-PRELOAD-2 %s
-; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx90a -amdgpu-kernarg-preload-count=4 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90a-PRELOAD-4 %s
-; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx90a -amdgpu-kernarg-preload-count=8 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90a-PRELOAD-8 %s
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90a %s
 
-define amdgpu_kernel void @ptr1_i8(ptr addrspace(1) %out, i8 %arg0) #0 {
-; GFX940-NO-PRELOAD-LABEL: ptr1_i8:
-; GFX940-NO-PRELOAD:       ; %bb.0:
-; GFX940-NO-PRELOAD-NEXT:    s_load_dword s4, s[0:1], 0x8
-; GFX940-NO-PRELOAD-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-NO-PRELOAD-NEXT:    s_and_b32 s0, s4, 0xff
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s0
-; GFX940-NO-PRELOAD-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
-; GFX940-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-1-LABEL: ptr1_i8:
-; GFX940-PRELOAD-1:       ; %bb.0:
-; GFX940-PRELOAD-1-NEXT:    s_load_dword s4, s[0:1], 0x8
-; GFX940-PRELOAD-1-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-1-NEXT:    s_and_b32 s0, s4, 0xff
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s0
-; GFX940-PRELOAD-1-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
-; GFX940-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-2-LABEL: ptr1_i8:
-; GFX940-PRELOAD-2:       ; %bb.0:
-; GFX940-PRELOAD-2-NEXT:    s_load_dword s4, s[0:1], 0x8
-; GFX940-PRELOAD-2-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-2-NEXT:    s_and_b32 s0, s4, 0xff
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s0
-; GFX940-PRELOAD-2-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
-; GFX940-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-4-LABEL: ptr1_i8:
-; GFX940-PRELOAD-4:       ; %bb.0:
-; GFX940-PRELOAD-4-NEXT:    s_load_dword s4, s[0:1], 0x8
-; GFX940-PRELOAD-4-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-4-NEXT:    s_and_b32 s0, s4, 0xff
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s0
-; GFX940-PRELOAD-4-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
-; GFX940-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-8-LABEL: ptr1_i8:
-; GFX940-PRELOAD-8:       ; %bb.0:
-; GFX940-PRELOAD-8-NEXT:    s_load_dword s4, s[0:1], 0x8
-; GFX940-PRELOAD-8-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-8-NEXT:    s_and_b32 s0, s4, 0xff
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s0
-; GFX940-PRELOAD-8-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
-; GFX940-PRELOAD-8-NEXT:    s_endpgm
-;
-; GFX90a-NO-PRELOAD-LABEL: ptr1_i8:
-; GFX90a-NO-PRELOAD:       ; %bb.0:
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-NO-PRELOAD-NEXT:    s_and_b32 s2, s2, 0xff
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-NO-PRELOAD-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX90a-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-1-LABEL: ptr1_i8:
-; GFX90a-PRELOAD-1:       ; %bb.0:
-; GFX90a-PRELOAD-1-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX90a-PRELOAD-1-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-1-NEXT:    s_and_b32 s2, s2, 0xff
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-PRELOAD-1-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX90a-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-2-LABEL: ptr1_i8:
-; GFX90a-PRELOAD-2:       ; %bb.0:
-; GFX90a-PRELOAD-2-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX90a-PRELOAD-2-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-2-NEXT:    s_and_b32 s2, s2, 0xff
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-PRELOAD-2-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX90a-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-4-LABEL: ptr1_i8:
-; GFX90a-PRELOAD-4:       ; %bb.0:
-; GFX90a-PRELOAD-4-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX90a-PRELOAD-4-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-4-NEXT:    s_and_b32 s2, s2, 0xff
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-PRELOAD-4-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX90a-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-8-LABEL: ptr1_i8:
-; GFX90a-PRELOAD-8:       ; %bb.0:
-; GFX90a-PRELOAD-8-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX90a-PRELOAD-8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-8-NEXT:    s_and_b32 s2, s2, 0xff
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-PRELOAD-8-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX90a-PRELOAD-8-NEXT:    s_endpgm
+define amdgpu_kernel void @ptr1_i8(ptr addrspace(1) inreg %out, i8 inreg %arg0) #0 {
+; GFX940-LABEL: ptr1_i8:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    s_and_b32 s0, s4, 0xff
+; GFX940-NEXT:    v_mov_b32_e32 v0, 0
+; GFX940-NEXT:    v_mov_b32_e32 v1, s0
+; GFX940-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: ptr1_i8:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    s_and_b32 s0, s8, 0xff
+; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
+; GFX90a-NEXT:    global_store_dword v0, v1, s[6:7]
+; GFX90a-NEXT:    s_endpgm
   %ext = zext i8 %arg0 to i32
   store i32 %ext, ptr addrspace(1) %out
   ret void
 }
 
-define amdgpu_kernel void @ptr1_i8_zext_arg(ptr addrspace(1) %out, i8 zeroext %arg0) #0 {
-; GFX940-NO-PRELOAD-LABEL: ptr1_i8_zext_arg:
-; GFX940-NO-PRELOAD:       ; %bb.0:
-; GFX940-NO-PRELOAD-NEXT:    s_load_dword s4, s[0:1], 0x8
-; GFX940-NO-PRELOAD-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-NO-PRELOAD-NEXT:    s_and_b32 s0, s4, 0xff
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s0
-; GFX940-NO-PRELOAD-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
-; GFX940-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-1-LABEL: ptr1_i8_zext_arg:
-; GFX940-PRELOAD-1:       ; %bb.0:
-; GFX940-PRELOAD-1-NEXT:    s_load_dword s4, s[0:1], 0x8
-; GFX940-PRELOAD-1-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-1-NEXT:    s_and_b32 s0, s4, 0xff
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s0
-; GFX940-PRELOAD-1-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
-; GFX940-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-2-LABEL: ptr1_i8_zext_arg:
-; GFX940-PRELOAD-2:       ; %bb.0:
-; GFX940-PRELOAD-2-NEXT:    s_load_dword s4, s[0:1], 0x8
-; GFX940-PRELOAD-2-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-2-NEXT:    s_and_b32 s0, s4, 0xff
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s0
-; GFX940-PRELOAD-2-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
-; GFX940-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-4-LABEL: ptr1_i8_zext_arg:
-; GFX940-PRELOAD-4:       ; %bb.0:
-; GFX940-PRELOAD-4-NEXT:    s_load_dword s4, s[0:1], 0x8
-; GFX940-PRELOAD-4-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-4-NEXT:    s_and_b32 s0, s4, 0xff
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s0
-; GFX940-PRELOAD-4-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
-; GFX940-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-8-LABEL: ptr1_i8_zext_arg:
-; GFX940-PRELOAD-8:       ; %bb.0:
-; GFX940-PRELOAD-8-NEXT:    s_load_dword s4, s[0:1], 0x8
-; GFX940-PRELOAD-8-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-8-NEXT:    s_and_b32 s0, s4, 0xff
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s0
-; GFX940-PRELOAD-8-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
-; GFX940-PRELOAD-8-NEXT:    s_endpgm
-;
-; GFX90a-NO-PRELOAD-LABEL: ptr1_i8_zext_arg:
-; GFX90a-NO-PRELOAD:       ; %bb.0:
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-NO-PRELOAD-NEXT:    s_and_b32 s2, s2, 0xff
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-NO-PRELOAD-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX90a-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-1-LABEL: ptr1_i8_zext_arg:
-; GFX90a-PRELOAD-1:       ; %bb.0:
-; GFX90a-PRELOAD-1-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX90a-PRELOAD-1-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-1-NEXT:    s_and_b32 s2, s2, 0xff
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-PRELOAD-1-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX90a-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-2-LABEL: ptr1_i8_zext_arg:
-; GFX90a-PRELOAD-2:       ; %bb.0:
-; GFX90a-PRELOAD-2-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX90a-PRELOAD-2-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-2-NEXT:    s_and_b32 s2, s2, 0xff
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-PRELOAD-2-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX90a-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-4-LABEL: ptr1_i8_zext_arg:
-; GFX90a-PRELOAD-4:       ; %bb.0:
-; GFX90a-PRELOAD-4-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX90a-PRELOAD-4-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-4-NEXT:    s_and_b32 s2, s2, 0xff
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-PRELOAD-4-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX90a-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-8-LABEL: ptr1_i8_zext_arg:
-; GFX90a-PRELOAD-8:       ; %bb.0:
-; GFX90a-PRELOAD-8-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX90a-PRELOAD-8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-8-NEXT:    s_and_b32 s2, s2, 0xff
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-PRELOAD-8-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX90a-PRELOAD-8-NEXT:    s_endpgm
+define amdgpu_kernel void @ptr1_i8_zext_arg(ptr addrspace(1) inreg %out, i8 zeroext inreg %arg0) #0 {
+; GFX940-LABEL: ptr1_i8_zext_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    s_mov_b32 s0, 0xffff
+; GFX940-NEXT:    v_mov_b32_e32 v1, s4
+; GFX940-NEXT:    v_mov_b32_e32 v0, 0
+; GFX940-NEXT:    v_and_b32_sdwa v1, s0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX940-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: ptr1_i8_zext_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    s_mov_b32 s0, 0xffff
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s8
+; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
+; GFX90a-NEXT:    v_and_b32_sdwa v1, s0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX90a-NEXT:    global_store_dword v0, v1, s[6:7]
+; GFX90a-NEXT:    s_endpgm
   %ext = zext i8 %arg0 to i32
   store i32 %ext, ptr addrspace(1) %out, align 4
   ret void
 }
 
-define amdgpu_kernel void @ptr1_i16_preload_arg(ptr addrspace(1) %out, i16 %arg0) #0 {
-; GFX940-NO-PRELOAD-LABEL: ptr1_i16_preload_arg:
-; GFX940-NO-PRELOAD:       ; %bb.0:
-; GFX940-NO-PRELOAD-NEXT:    s_load_dword s4, s[0:1], 0x8
-; GFX940-NO-PRELOAD-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-NO-PRELOAD-NEXT:    s_and_b32 s0, s4, 0xffff
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s0
-; GFX940-NO-PRELOAD-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
-; GFX940-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-1-LABEL: ptr1_i16_preload_arg:
-; GFX940-PRELOAD-1:       ; %bb.0:
-; GFX940-PRELOAD-1-NEXT:    s_load_dword s4, s[0:1], 0x8
-; GFX940-PRELOAD-1-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-1-NEXT:    s_and_b32 s0, s4, 0xffff
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s0
-; GFX940-PRELOAD-1-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
-; GFX940-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-2-LABEL: ptr1_i16_preload_arg:
-; GFX940-PRELOAD-2:       ; %bb.0:
-; GFX940-PRELOAD-2-NEXT:    s_load_dword s4, s[0:1], 0x8
-; GFX940-PRELOAD-2-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-2-NEXT:    s_and_b32 s0, s4, 0xffff
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s0
-; GFX940-PRELOAD-2-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
-; GFX940-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-4-LABEL: ptr1_i16_preload_arg:
-; GFX940-PRELOAD-4:       ; %bb.0:
-; GFX940-PRELOAD-4-NEXT:    s_load_dword s4, s[0:1], 0x8
-; GFX940-PRELOAD-4-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-4-NEXT:    s_and_b32 s0, s4, 0xffff
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s0
-; GFX940-PRELOAD-4-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
-; GFX940-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-8-LABEL: ptr1_i16_preload_arg:
-; GFX940-PRELOAD-8:       ; %bb.0:
-; GFX940-PRELOAD-8-NEXT:    s_load_dword s4, s[0:1], 0x8
-; GFX940-PRELOAD-8-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-8-NEXT:    s_and_b32 s0, s4, 0xffff
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s0
-; GFX940-PRELOAD-8-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
-; GFX940-PRELOAD-8-NEXT:    s_endpgm
-;
-; GFX90a-NO-PRELOAD-LABEL: ptr1_i16_preload_arg:
-; GFX90a-NO-PRELOAD:       ; %bb.0:
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-NO-PRELOAD-NEXT:    s_and_b32 s2, s2, 0xffff
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-NO-PRELOAD-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX90a-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-1-LABEL: ptr1_i16_preload_arg:
-; GFX90a-PRELOAD-1:       ; %bb.0:
-; GFX90a-PRELOAD-1-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX90a-PRELOAD-1-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-1-NEXT:    s_and_b32 s2, s2, 0xffff
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-PRELOAD-1-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX90a-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-2-LABEL: ptr1_i16_preload_arg:
-; GFX90a-PRELOAD-2:       ; %bb.0:
-; GFX90a-PRELOAD-2-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX90a-PRELOAD-2-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-2-NEXT:    s_and_b32 s2, s2, 0xffff
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-PRELOAD-2-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX90a-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-4-LABEL: ptr1_i16_preload_arg:
-; GFX90a-PRELOAD-4:       ; %bb.0:
-; GFX90a-PRELOAD-4-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX90a-PRELOAD-4-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-4-NEXT:    s_and_b32 s2, s2, 0xffff
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-PRELOAD-4-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX90a-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-8-LABEL: ptr1_i16_preload_arg:
-; GFX90a-PRELOAD-8:       ; %bb.0:
-; GFX90a-PRELOAD-8-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX90a-PRELOAD-8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-8-NEXT:    s_and_b32 s2, s2, 0xffff
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-PRELOAD-8-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX90a-PRELOAD-8-NEXT:    s_endpgm
+define amdgpu_kernel void @ptr1_i16_preload_arg(ptr addrspace(1) inreg %out, i16 inreg %arg0) #0 {
+; GFX940-LABEL: ptr1_i16_preload_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    s_and_b32 s0, s4, 0xffff
+; GFX940-NEXT:    v_mov_b32_e32 v0, 0
+; GFX940-NEXT:    v_mov_b32_e32 v1, s0
+; GFX940-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: ptr1_i16_preload_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    s_and_b32 s0, s8, 0xffff
+; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
+; GFX90a-NEXT:    global_store_dword v0, v1, s[6:7]
+; GFX90a-NEXT:    s_endpgm
   %ext = zext i16 %arg0 to i32
   store i32 %ext, ptr addrspace(1) %out, align 4
   ret void
 }
 
-define amdgpu_kernel void @ptr1_i32_preload_arg(ptr addrspace(1) %out, i32 %arg0) #0 {
-; GFX940-NO-PRELOAD-LABEL: ptr1_i32_preload_arg:
-; GFX940-NO-PRELOAD:       ; %bb.0:
-; GFX940-NO-PRELOAD-NEXT:    s_load_dword s4, s[0:1], 0x8
-; GFX940-NO-PRELOAD-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s4
-; GFX940-NO-PRELOAD-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
-; GFX940-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-1-LABEL: ptr1_i32_preload_arg:
-; GFX940-PRELOAD-1:       ; %bb.0:
-; GFX940-PRELOAD-1-NEXT:    s_load_dword s4, s[0:1], 0x8
-; GFX940-PRELOAD-1-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s4
-; GFX940-PRELOAD-1-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
-; GFX940-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-2-LABEL: ptr1_i32_preload_arg:
-; GFX940-PRELOAD-2:       ; %bb.0:
-; GFX940-PRELOAD-2-NEXT:    s_load_dword s4, s[0:1], 0x8
-; GFX940-PRELOAD-2-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s4
-; GFX940-PRELOAD-2-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
-; GFX940-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-4-LABEL: ptr1_i32_preload_arg:
-; GFX940-PRELOAD-4:       ; %bb.0:
-; GFX940-PRELOAD-4-NEXT:    s_load_dword s4, s[0:1], 0x8
-; GFX940-PRELOAD-4-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s4
-; GFX940-PRELOAD-4-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
-; GFX940-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-8-LABEL: ptr1_i32_preload_arg:
-; GFX940-PRELOAD-8:       ; %bb.0:
-; GFX940-PRELOAD-8-NEXT:    s_load_dword s4, s[0:1], 0x8
-; GFX940-PRELOAD-8-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s4
-; GFX940-PRELOAD-8-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
-; GFX940-PRELOAD-8-NEXT:    s_endpgm
-;
-; GFX90a-NO-PRELOAD-LABEL: ptr1_i32_preload_arg:
-; GFX90a-NO-PRELOAD:       ; %bb.0:
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-NO-PRELOAD-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX90a-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-1-LABEL: ptr1_i32_preload_arg:
-; GFX90a-PRELOAD-1:       ; %bb.0:
-; GFX90a-PRELOAD-1-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX90a-PRELOAD-1-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-PRELOAD-1-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX90a-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-2-LABEL: ptr1_i32_preload_arg:
-; GFX90a-PRELOAD-2:       ; %bb.0:
-; GFX90a-PRELOAD-2-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX90a-PRELOAD-2-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-PRELOAD-2-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX90a-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-4-LABEL: ptr1_i32_preload_arg:
-; GFX90a-PRELOAD-4:       ; %bb.0:
-; GFX90a-PRELOAD-4-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX90a-PRELOAD-4-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-PRELOAD-4-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX90a-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-8-LABEL: ptr1_i32_preload_arg:
-; GFX90a-PRELOAD-8:       ; %bb.0:
-; GFX90a-PRELOAD-8-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX90a-PRELOAD-8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-PRELOAD-8-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX90a-PRELOAD-8-NEXT:    s_endpgm
+define amdgpu_kernel void @ptr1_i32_preload_arg(ptr addrspace(1) inreg %out, i32 inreg %arg0) #0 {
+; GFX940-LABEL: ptr1_i32_preload_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    v_mov_b32_e32 v0, 0
+; GFX940-NEXT:    v_mov_b32_e32 v1, s4
+; GFX940-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: ptr1_i32_preload_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s8
+; GFX90a-NEXT:    global_store_dword v0, v1, s[6:7]
+; GFX90a-NEXT:    s_endpgm
   store i32 %arg0, ptr addrspace(1) %out
   ret void
 }
 
 
-define amdgpu_kernel void @i32_ptr1_i32_preload_arg(i32 %arg0, ptr addrspace(1) %out, i32 %arg1) #0 {
-; GFX940-NO-PRELOAD-LABEL: i32_ptr1_i32_preload_arg:
-; GFX940-NO-PRELOAD:       ; %bb.0:
-; GFX940-NO-PRELOAD-NEXT:    s_load_dword s4, s[0:1], 0x10
-; GFX940-NO-PRELOAD-NEXT:    s_load_dword s5, s[0:1], 0x0
-; GFX940-NO-PRELOAD-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-NO-PRELOAD-NEXT:    s_add_i32 s0, s5, s4
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s0
-; GFX940-NO-PRELOAD-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
-; GFX940-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-1-LABEL: i32_ptr1_i32_preload_arg:
-; GFX940-PRELOAD-1:       ; %bb.0:
-; GFX940-PRELOAD-1-NEXT:    s_load_dword s4, s[0:1], 0x10
-; GFX940-PRELOAD-1-NEXT:    s_load_dword s5, s[0:1], 0x0
-; GFX940-PRELOAD-1-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-1-NEXT:    s_add_i32 s0, s5, s4
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s0
-; GFX940-PRELOAD-1-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
-; GFX940-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-2-LABEL: i32_ptr1_i32_preload_arg:
-; GFX940-PRELOAD-2:       ; %bb.0:
-; GFX940-PRELOAD-2-NEXT:    s_load_dword s4, s[0:1], 0x10
-; GFX940-PRELOAD-2-NEXT:    s_load_dword s5, s[0:1], 0x0
-; GFX940-PRELOAD-2-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-2-NEXT:    s_add_i32 s0, s5, s4
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s0
-; GFX940-PRELOAD-2-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
-; GFX940-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-4-LABEL: i32_ptr1_i32_preload_arg:
-; GFX940-PRELOAD-4:       ; %bb.0:
-; GFX940-PRELOAD-4-NEXT:    s_load_dword s4, s[0:1], 0x10
-; GFX940-PRELOAD-4-NEXT:    s_load_dword s5, s[0:1], 0x0
-; GFX940-PRELOAD-4-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-4-NEXT:    s_add_i32 s0, s5, s4
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s0
-; GFX940-PRELOAD-4-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
-; GFX940-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-8-LABEL: i32_ptr1_i32_preload_arg:
-; GFX940-PRELOAD-8:       ; %bb.0:
-; GFX940-PRELOAD-8-NEXT:    s_load_dword s4, s[0:1], 0x10
-; GFX940-PRELOAD-8-NEXT:    s_load_dword s5, s[0:1], 0x0
-; GFX940-PRELOAD-8-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x8
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-8-NEXT:    s_add_i32 s0, s5, s4
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s0
-; GFX940-PRELOAD-8-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
-; GFX940-PRELOAD-8-NEXT:    s_endpgm
-;
-; GFX90a-NO-PRELOAD-LABEL: i32_ptr1_i32_preload_arg:
-; GFX90a-NO-PRELOAD:       ; %bb.0:
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dword s2, s[4:5], 0x10
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dword s3, s[4:5], 0x0
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-NO-PRELOAD-NEXT:    s_add_i32 s2, s3, s2
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-NO-PRELOAD-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX90a-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-1-LABEL: i32_ptr1_i32_preload_arg:
-; GFX90a-PRELOAD-1:       ; %bb.0:
-; GFX90a-PRELOAD-1-NEXT:    s_load_dword s2, s[4:5], 0x10
-; GFX90a-PRELOAD-1-NEXT:    s_load_dword s3, s[4:5], 0x0
-; GFX90a-PRELOAD-1-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-1-NEXT:    s_add_i32 s2, s3, s2
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-PRELOAD-1-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX90a-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-2-LABEL: i32_ptr1_i32_preload_arg:
-; GFX90a-PRELOAD-2:       ; %bb.0:
-; GFX90a-PRELOAD-2-NEXT:    s_load_dword s2, s[4:5], 0x10
-; GFX90a-PRELOAD-2-NEXT:    s_load_dword s3, s[4:5], 0x0
-; GFX90a-PRELOAD-2-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-2-NEXT:    s_add_i32 s2, s3, s2
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-PRELOAD-2-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX90a-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-4-LABEL: i32_ptr1_i32_preload_arg:
-; GFX90a-PRELOAD-4:       ; %bb.0:
-; GFX90a-PRELOAD-4-NEXT:    s_load_dword s2, s[4:5], 0x10
-; GFX90a-PRELOAD-4-NEXT:    s_load_dword s3, s[4:5], 0x0
-; GFX90a-PRELOAD-4-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-4-NEXT:    s_add_i32 s2, s3, s2
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-PRELOAD-4-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX90a-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-8-LABEL: i32_ptr1_i32_preload_arg:
-; GFX90a-PRELOAD-8:       ; %bb.0:
-; GFX90a-PRELOAD-8-NEXT:    s_load_dword s2, s[4:5], 0x10
-; GFX90a-PRELOAD-8-NEXT:    s_load_dword s3, s[4:5], 0x0
-; GFX90a-PRELOAD-8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-8-NEXT:    s_add_i32 s2, s3, s2
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-PRELOAD-8-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX90a-PRELOAD-8-NEXT:    s_endpgm
+define amdgpu_kernel void @i32_ptr1_i32_preload_arg(i32 inreg %arg0, ptr addrspace(1) inreg %out, i32 inreg %arg1) #0 {
+; GFX940-LABEL: i32_ptr1_i32_preload_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    s_add_i32 s0, s2, s6
+; GFX940-NEXT:    v_mov_b32_e32 v0, 0
+; GFX940-NEXT:    v_mov_b32_e32 v1, s0
+; GFX940-NEXT:    global_store_dword v0, v1, s[4:5] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: i32_ptr1_i32_preload_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    s_add_i32 s0, s6, s10
+; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
+; GFX90a-NEXT:    global_store_dword v0, v1, s[8:9]
+; GFX90a-NEXT:    s_endpgm
   %add = add i32 %arg0, %arg1
   store i32 %add, ptr addrspace(1) %out
   ret void
 }
 
-define amdgpu_kernel void @ptr1_i16_i16_preload_arg(ptr addrspace(1) %out, i16 %arg0, i16 %arg1) #0 {
-; GFX940-NO-PRELOAD-LABEL: ptr1_i16_i16_preload_arg:
-; GFX940-NO-PRELOAD:       ; %bb.0:
-; GFX940-NO-PRELOAD-NEXT:    s_load_dword s4, s[0:1], 0x8
-; GFX940-NO-PRELOAD-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-NO-PRELOAD-NEXT:    s_lshr_b32 s0, s4, 16
-; GFX940-NO-PRELOAD-NEXT:    s_and_b32 s1, s4, 0xffff
-; GFX940-NO-PRELOAD-NEXT:    s_add_i32 s0, s1, s0
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s0
-; GFX940-NO-PRELOAD-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
-; GFX940-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-1-LABEL: ptr1_i16_i16_preload_arg:
-; GFX940-PRELOAD-1:       ; %bb.0:
-; GFX940-PRELOAD-1-NEXT:    s_load_dword s4, s[0:1], 0x8
-; GFX940-PRELOAD-1-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-1-NEXT:    s_lshr_b32 s0, s4, 16
-; GFX940-PRELOAD-1-NEXT:    s_and_b32 s1, s4, 0xffff
-; GFX940-PRELOAD-1-NEXT:    s_add_i32 s0, s1, s0
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s0
-; GFX940-PRELOAD-1-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
-; GFX940-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-2-LABEL: ptr1_i16_i16_preload_arg:
-; GFX940-PRELOAD-2:       ; %bb.0:
-; GFX940-PRELOAD-2-NEXT:    s_load_dword s4, s[0:1], 0x8
-; GFX940-PRELOAD-2-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-2-NEXT:    s_lshr_b32 s0, s4, 16
-; GFX940-PRELOAD-2-NEXT:    s_and_b32 s1, s4, 0xffff
-; GFX940-PRELOAD-2-NEXT:    s_add_i32 s0, s1, s0
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s0
-; GFX940-PRELOAD-2-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
-; GFX940-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-4-LABEL: ptr1_i16_i16_preload_arg:
-; GFX940-PRELOAD-4:       ; %bb.0:
-; GFX940-PRELOAD-4-NEXT:    s_load_dword s4, s[0:1], 0x8
-; GFX940-PRELOAD-4-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-4-NEXT:    s_lshr_b32 s0, s4, 16
-; GFX940-PRELOAD-4-NEXT:    s_and_b32 s1, s4, 0xffff
-; GFX940-PRELOAD-4-NEXT:    s_add_i32 s0, s1, s0
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s0
-; GFX940-PRELOAD-4-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
-; GFX940-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-8-LABEL: ptr1_i16_i16_preload_arg:
-; GFX940-PRELOAD-8:       ; %bb.0:
-; GFX940-PRELOAD-8-NEXT:    s_load_dword s4, s[0:1], 0x8
-; GFX940-PRELOAD-8-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-8-NEXT:    s_lshr_b32 s0, s4, 16
-; GFX940-PRELOAD-8-NEXT:    s_and_b32 s1, s4, 0xffff
-; GFX940-PRELOAD-8-NEXT:    s_add_i32 s0, s1, s0
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s0
-; GFX940-PRELOAD-8-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
-; GFX940-PRELOAD-8-NEXT:    s_endpgm
-;
-; GFX90a-NO-PRELOAD-LABEL: ptr1_i16_i16_preload_arg:
-; GFX90a-NO-PRELOAD:       ; %bb.0:
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-NO-PRELOAD-NEXT:    s_lshr_b32 s3, s2, 16
-; GFX90a-NO-PRELOAD-NEXT:    s_and_b32 s2, s2, 0xffff
-; GFX90a-NO-PRELOAD-NEXT:    s_add_i32 s2, s2, s3
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-NO-PRELOAD-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX90a-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-1-LABEL: ptr1_i16_i16_preload_arg:
-; GFX90a-PRELOAD-1:       ; %bb.0:
-; GFX90a-PRELOAD-1-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX90a-PRELOAD-1-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-1-NEXT:    s_lshr_b32 s3, s2, 16
-; GFX90a-PRELOAD-1-NEXT:    s_and_b32 s2, s2, 0xffff
-; GFX90a-PRELOAD-1-NEXT:    s_add_i32 s2, s2, s3
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-PRELOAD-1-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX90a-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-2-LABEL: ptr1_i16_i16_preload_arg:
-; GFX90a-PRELOAD-2:       ; %bb.0:
-; GFX90a-PRELOAD-2-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX90a-PRELOAD-2-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-2-NEXT:    s_lshr_b32 s3, s2, 16
-; GFX90a-PRELOAD-2-NEXT:    s_and_b32 s2, s2, 0xffff
-; GFX90a-PRELOAD-2-NEXT:    s_add_i32 s2, s2, s3
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-PRELOAD-2-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX90a-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-4-LABEL: ptr1_i16_i16_preload_arg:
-; GFX90a-PRELOAD-4:       ; %bb.0:
-; GFX90a-PRELOAD-4-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX90a-PRELOAD-4-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-4-NEXT:    s_lshr_b32 s3, s2, 16
-; GFX90a-PRELOAD-4-NEXT:    s_and_b32 s2, s2, 0xffff
-; GFX90a-PRELOAD-4-NEXT:    s_add_i32 s2, s2, s3
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-PRELOAD-4-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX90a-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-8-LABEL: ptr1_i16_i16_preload_arg:
-; GFX90a-PRELOAD-8:       ; %bb.0:
-; GFX90a-PRELOAD-8-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX90a-PRELOAD-8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-8-NEXT:    s_lshr_b32 s3, s2, 16
-; GFX90a-PRELOAD-8-NEXT:    s_and_b32 s2, s2, 0xffff
-; GFX90a-PRELOAD-8-NEXT:    s_add_i32 s2, s2, s3
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-PRELOAD-8-NEXT:    global_store_dword v0, v1, s[0:1]
-; GFX90a-PRELOAD-8-NEXT:    s_endpgm
+define amdgpu_kernel void @ptr1_i16_i16_preload_arg(ptr addrspace(1) inreg %out, i16 inreg %arg0, i16 inreg %arg1) #0 {
+; GFX940-LABEL: ptr1_i16_i16_preload_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    s_lshr_b32 s0, s4, 16
+; GFX940-NEXT:    s_and_b32 s1, s4, 0xffff
+; GFX940-NEXT:    s_add_i32 s0, s1, s0
+; GFX940-NEXT:    v_mov_b32_e32 v0, 0
+; GFX940-NEXT:    v_mov_b32_e32 v1, s0
+; GFX940-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: ptr1_i16_i16_preload_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    s_lshr_b32 s0, s8, 16
+; GFX90a-NEXT:    s_and_b32 s1, s8, 0xffff
+; GFX90a-NEXT:    s_add_i32 s0, s1, s0
+; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
+; GFX90a-NEXT:    global_store_dword v0, v1, s[6:7]
+; GFX90a-NEXT:    s_endpgm
   %ext = zext i16 %arg0 to i32
   %ext1 = zext i16 %arg1 to i32
   %add = add i32 %ext, %ext1
@@ -723,1452 +159,878 @@ define amdgpu_kernel void @ptr1_i16_i16_preload_arg(ptr addrspace(1) %out, i16 %
   ret void
 }
 
-define amdgpu_kernel void @ptr1_v2i8_preload_arg(ptr addrspace(1) %out, <2 x i8> %in) #0 {
-; GFX940-NO-PRELOAD-LABEL: ptr1_v2i8_preload_arg:
-; GFX940-NO-PRELOAD:       ; %bb.0:
-; GFX940-NO-PRELOAD-NEXT:    s_load_dword s4, s[0:1], 0x8
-; GFX940-NO-PRELOAD-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s4
-; GFX940-NO-PRELOAD-NEXT:    global_store_short v0, v1, s[2:3] sc0 sc1
-; GFX940-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-1-LABEL: ptr1_v2i8_preload_arg:
-; GFX940-PRELOAD-1:       ; %bb.0:
-; GFX940-PRELOAD-1-NEXT:    s_load_dword s4, s[0:1], 0x8
-; GFX940-PRELOAD-1-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s4
-; GFX940-PRELOAD-1-NEXT:    global_store_short v0, v1, s[2:3] sc0 sc1
-; GFX940-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-2-LABEL: ptr1_v2i8_preload_arg:
-; GFX940-PRELOAD-2:       ; %bb.0:
-; GFX940-PRELOAD-2-NEXT:    s_load_dword s4, s[0:1], 0x8
-; GFX940-PRELOAD-2-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s4
-; GFX940-PRELOAD-2-NEXT:    global_store_short v0, v1, s[2:3] sc0 sc1
-; GFX940-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-4-LABEL: ptr1_v2i8_preload_arg:
-; GFX940-PRELOAD-4:       ; %bb.0:
-; GFX940-PRELOAD-4-NEXT:    s_load_dword s4, s[0:1], 0x8
-; GFX940-PRELOAD-4-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s4
-; GFX940-PRELOAD-4-NEXT:    global_store_short v0, v1, s[2:3] sc0 sc1
-; GFX940-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-8-LABEL: ptr1_v2i8_preload_arg:
-; GFX940-PRELOAD-8:       ; %bb.0:
-; GFX940-PRELOAD-8-NEXT:    s_load_dword s4, s[0:1], 0x8
-; GFX940-PRELOAD-8-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s4
-; GFX940-PRELOAD-8-NEXT:    global_store_short v0, v1, s[2:3] sc0 sc1
-; GFX940-PRELOAD-8-NEXT:    s_endpgm
-;
-; GFX90a-NO-PRELOAD-LABEL: ptr1_v2i8_preload_arg:
-; GFX90a-NO-PRELOAD:       ; %bb.0:
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-NO-PRELOAD-NEXT:    global_store_short v0, v1, s[0:1]
-; GFX90a-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-1-LABEL: ptr1_v2i8_preload_arg:
-; GFX90a-PRELOAD-1:       ; %bb.0:
-; GFX90a-PRELOAD-1-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX90a-PRELOAD-1-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-PRELOAD-1-NEXT:    global_store_short v0, v1, s[0:1]
-; GFX90a-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-2-LABEL: ptr1_v2i8_preload_arg:
-; GFX90a-PRELOAD-2:       ; %bb.0:
-; GFX90a-PRELOAD-2-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX90a-PRELOAD-2-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-PRELOAD-2-NEXT:    global_store_short v0, v1, s[0:1]
-; GFX90a-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-4-LABEL: ptr1_v2i8_preload_arg:
-; GFX90a-PRELOAD-4:       ; %bb.0:
-; GFX90a-PRELOAD-4-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX90a-PRELOAD-4-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-PRELOAD-4-NEXT:    global_store_short v0, v1, s[0:1]
-; GFX90a-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-8-LABEL: ptr1_v2i8_preload_arg:
-; GFX90a-PRELOAD-8:       ; %bb.0:
-; GFX90a-PRELOAD-8-NEXT:    s_load_dword s2, s[4:5], 0x8
-; GFX90a-PRELOAD-8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s2
-; GFX90a-PRELOAD-8-NEXT:    global_store_short v0, v1, s[0:1]
-; GFX90a-PRELOAD-8-NEXT:    s_endpgm
+define amdgpu_kernel void @ptr1_v2i8_preload_arg(ptr addrspace(1) inreg %out, <2 x i8> inreg %in) #0 {
+; GFX940-LABEL: ptr1_v2i8_preload_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    s_lshr_b32 s0, s4, 8
+; GFX940-NEXT:    v_lshlrev_b16_e64 v0, 8, s0
+; GFX940-NEXT:    v_or_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX940-NEXT:    v_mov_b32_e32 v1, 0
+; GFX940-NEXT:    global_store_short v1, v0, s[2:3] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: ptr1_v2i8_preload_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    s_lshr_b32 s0, s8, 8
+; GFX90a-NEXT:    v_lshlrev_b16_e64 v0, 8, s0
+; GFX90a-NEXT:    v_or_b32_sdwa v0, s8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX90a-NEXT:    v_mov_b32_e32 v1, 0
+; GFX90a-NEXT:    global_store_short v1, v0, s[6:7]
+; GFX90a-NEXT:    s_endpgm
   store <2 x i8> %in, ptr addrspace(1) %out
   ret void
 }
 
 
-define amdgpu_kernel void @byref_preload_arg(ptr addrspace(1) %out, ptr addrspace(4) byref(i32) align(256) %in.byref, i32 %after.offset) #0 {
-; GFX940-NO-PRELOAD-LABEL: byref_preload_arg:
-; GFX940-NO-PRELOAD:       ; %bb.0:
-; GFX940-NO-PRELOAD-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x100
-; GFX940-NO-PRELOAD-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s2
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v2, s3
-; GFX940-NO-PRELOAD-NEXT:    global_store_dword v0, v1, s[4:5] sc0 sc1
-; GFX940-NO-PRELOAD-NEXT:    s_waitcnt vmcnt(0)
-; GFX940-NO-PRELOAD-NEXT:    global_store_dword v0, v2, s[4:5] sc0 sc1
-; GFX940-NO-PRELOAD-NEXT:    s_waitcnt vmcnt(0)
-; GFX940-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-1-LABEL: byref_preload_arg:
-; GFX940-PRELOAD-1:       ; %bb.0:
-; GFX940-PRELOAD-1-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x100
-; GFX940-PRELOAD-1-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s2
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v2, s3
-; GFX940-PRELOAD-1-NEXT:    global_store_dword v0, v1, s[4:5] sc0 sc1
-; GFX940-PRELOAD-1-NEXT:    s_waitcnt vmcnt(0)
-; GFX940-PRELOAD-1-NEXT:    global_store_dword v0, v2, s[4:5] sc0 sc1
-; GFX940-PRELOAD-1-NEXT:    s_waitcnt vmcnt(0)
-; GFX940-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-2-LABEL: byref_preload_arg:
-; GFX940-PRELOAD-2:       ; %bb.0:
-; GFX940-PRELOAD-2-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x100
-; GFX940-PRELOAD-2-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s2
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v2, s3
-; GFX940-PRELOAD-2-NEXT:    global_store_dword v0, v1, s[4:5] sc0 sc1
-; GFX940-PRELOAD-2-NEXT:    s_waitcnt vmcnt(0)
-; GFX940-PRELOAD-2-NEXT:    global_store_dword v0, v2, s[4:5] sc0 sc1
-; GFX940-PRELOAD-2-NEXT:    s_waitcnt vmcnt(0)
-; GFX940-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-4-LABEL: byref_preload_arg:
-; GFX940-PRELOAD-4:       ; %bb.0:
-; GFX940-PRELOAD-4-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x100
-; GFX940-PRELOAD-4-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s2
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v2, s3
-; GFX940-PRELOAD-4-NEXT:    global_store_dword v0, v1, s[4:5] sc0 sc1
-; GFX940-PRELOAD-4-NEXT:    s_waitcnt vmcnt(0)
-; GFX940-PRELOAD-4-NEXT:    global_store_dword v0, v2, s[4:5] sc0 sc1
-; GFX940-PRELOAD-4-NEXT:    s_waitcnt vmcnt(0)
-; GFX940-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-8-LABEL: byref_preload_arg:
-; GFX940-PRELOAD-8:       ; %bb.0:
-; GFX940-PRELOAD-8-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x100
-; GFX940-PRELOAD-8-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s2
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v2, s3
-; GFX940-PRELOAD-8-NEXT:    global_store_dword v0, v1, s[4:5] sc0 sc1
-; GFX940-PRELOAD-8-NEXT:    s_waitcnt vmcnt(0)
-; GFX940-PRELOAD-8-NEXT:    global_store_dword v0, v2, s[4:5] sc0 sc1
-; GFX940-PRELOAD-8-NEXT:    s_waitcnt vmcnt(0)
-; GFX940-PRELOAD-8-NEXT:    s_endpgm
-;
-; GFX90a-NO-PRELOAD-LABEL: byref_preload_arg:
-; GFX90a-NO-PRELOAD:       ; %bb.0:
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x100
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s0
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v2, s1
-; GFX90a-NO-PRELOAD-NEXT:    global_store_dword v0, v1, s[2:3]
-; GFX90a-NO-PRELOAD-NEXT:    s_waitcnt vmcnt(0)
-; GFX90a-NO-PRELOAD-NEXT:    global_store_dword v0, v2, s[2:3]
-; GFX90a-NO-PRELOAD-NEXT:    s_waitcnt vmcnt(0)
-; GFX90a-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-1-LABEL: byref_preload_arg:
-; GFX90a-PRELOAD-1:       ; %bb.0:
-; GFX90a-PRELOAD-1-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x100
-; GFX90a-PRELOAD-1-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s0
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v2, s1
-; GFX90a-PRELOAD-1-NEXT:    global_store_dword v0, v1, s[2:3]
-; GFX90a-PRELOAD-1-NEXT:    s_waitcnt vmcnt(0)
-; GFX90a-PRELOAD-1-NEXT:    global_store_dword v0, v2, s[2:3]
-; GFX90a-PRELOAD-1-NEXT:    s_waitcnt vmcnt(0)
-; GFX90a-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-2-LABEL: byref_preload_arg:
-; GFX90a-PRELOAD-2:       ; %bb.0:
-; GFX90a-PRELOAD-2-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x100
-; GFX90a-PRELOAD-2-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s0
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v2, s1
-; GFX90a-PRELOAD-2-NEXT:    global_store_dword v0, v1, s[2:3]
-; GFX90a-PRELOAD-2-NEXT:    s_waitcnt vmcnt(0)
-; GFX90a-PRELOAD-2-NEXT:    global_store_dword v0, v2, s[2:3]
-; GFX90a-PRELOAD-2-NEXT:    s_waitcnt vmcnt(0)
-; GFX90a-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-4-LABEL: byref_preload_arg:
-; GFX90a-PRELOAD-4:       ; %bb.0:
-; GFX90a-PRELOAD-4-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x100
-; GFX90a-PRELOAD-4-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s0
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v2, s1
-; GFX90a-PRELOAD-4-NEXT:    global_store_dword v0, v1, s[2:3]
-; GFX90a-PRELOAD-4-NEXT:    s_waitcnt vmcnt(0)
-; GFX90a-PRELOAD-4-NEXT:    global_store_dword v0, v2, s[2:3]
-; GFX90a-PRELOAD-4-NEXT:    s_waitcnt vmcnt(0)
-; GFX90a-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-8-LABEL: byref_preload_arg:
-; GFX90a-PRELOAD-8:       ; %bb.0:
-; GFX90a-PRELOAD-8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x100
-; GFX90a-PRELOAD-8-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s0
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v2, s1
-; GFX90a-PRELOAD-8-NEXT:    global_store_dword v0, v1, s[2:3]
-; GFX90a-PRELOAD-8-NEXT:    s_waitcnt vmcnt(0)
-; GFX90a-PRELOAD-8-NEXT:    global_store_dword v0, v2, s[2:3]
-; GFX90a-PRELOAD-8-NEXT:    s_waitcnt vmcnt(0)
-; GFX90a-PRELOAD-8-NEXT:    s_endpgm
+define amdgpu_kernel void @byref_preload_arg(ptr addrspace(1) inreg %out, ptr addrspace(4) byref(i32) align(256) %in.byref, i32 %after.offset) #0 {
+; GFX940-LABEL: byref_preload_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x100
+; GFX940-NEXT:    v_mov_b32_e32 v0, 0
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b32_e32 v1, s4
+; GFX940-NEXT:    v_mov_b32_e32 v2, s5
+; GFX940-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    global_store_dword v0, v2, s[2:3] sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: byref_preload_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x100
+; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
+; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
+; GFX90a-NEXT:    v_mov_b32_e32 v2, s1
+; GFX90a-NEXT:    global_store_dword v0, v1, s[6:7]
+; GFX90a-NEXT:    s_waitcnt vmcnt(0)
+; GFX90a-NEXT:    global_store_dword v0, v2, s[6:7]
+; GFX90a-NEXT:    s_waitcnt vmcnt(0)
+; GFX90a-NEXT:    s_endpgm
   %in = load i32, ptr addrspace(4) %in.byref
   store volatile i32 %in, ptr addrspace(1) %out, align 4
   store volatile i32 %after.offset, ptr addrspace(1) %out, align 4
   ret void
 }
 
+; The second argument is not expected to be preloaded with the current behavior.
 
-define amdgpu_kernel void @v8i32_arg(ptr addrspace(1) nocapture %out, <8 x i32> %in) #0 {
-; GFX940-NO-PRELOAD-LABEL: v8i32_arg:
-; GFX940-NO-PRELOAD:       ; %bb.0:
-; GFX940-NO-PRELOAD-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0x20
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-NO-PRELOAD-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
-; GFX940-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, s8
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s9
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v2, s10
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v3, s11
-; GFX940-NO-PRELOAD-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] offset:16 sc0 sc1
-; GFX940-NO-PRELOAD-NEXT:    s_nop 1
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, s4
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s5
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v2, s6
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v3, s7
-; GFX940-NO-PRELOAD-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
-; GFX940-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-1-LABEL: v8i32_arg:
-; GFX940-PRELOAD-1:       ; %bb.0:
-; GFX940-PRELOAD-1-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0x20
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-PRELOAD-1-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
-; GFX940-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, s8
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s9
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v2, s10
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v3, s11
-; GFX940-PRELOAD-1-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] offset:16 sc0 sc1
-; GFX940-PRELOAD-1-NEXT:    s_nop 1
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, s4
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s5
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v2, s6
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v3, s7
-; GFX940-PRELOAD-1-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
-; GFX940-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-2-LABEL: v8i32_arg:
-; GFX940-PRELOAD-2:       ; %bb.0:
-; GFX940-PRELOAD-2-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0x20
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-PRELOAD-2-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
-; GFX940-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, s8
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s9
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v2, s10
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v3, s11
-; GFX940-PRELOAD-2-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] offset:16 sc0 sc1
-; GFX940-PRELOAD-2-NEXT:    s_nop 1
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, s4
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s5
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v2, s6
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v3, s7
-; GFX940-PRELOAD-2-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
-; GFX940-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-4-LABEL: v8i32_arg:
-; GFX940-PRELOAD-4:       ; %bb.0:
-; GFX940-PRELOAD-4-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0x20
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-PRELOAD-4-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
-; GFX940-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, s8
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s9
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v2, s10
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v3, s11
-; GFX940-PRELOAD-4-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] offset:16 sc0 sc1
-; GFX940-PRELOAD-4-NEXT:    s_nop 1
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, s4
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s5
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v2, s6
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v3, s7
-; GFX940-PRELOAD-4-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
-; GFX940-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-8-LABEL: v8i32_arg:
-; GFX940-PRELOAD-8:       ; %bb.0:
-; GFX940-PRELOAD-8-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0x20
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-PRELOAD-8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
-; GFX940-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, s8
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s9
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v2, s10
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v3, s11
-; GFX940-PRELOAD-8-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] offset:16 sc0 sc1
-; GFX940-PRELOAD-8-NEXT:    s_nop 1
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, s4
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s5
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v2, s6
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v3, s7
-; GFX940-PRELOAD-8-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] sc0 sc1
-; GFX940-PRELOAD-8-NEXT:    s_endpgm
-;
-; GFX90a-NO-PRELOAD-LABEL: v8i32_arg:
-; GFX90a-NO-PRELOAD:       ; %bb.0:
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x20
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90a-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, s12
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s13
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v2, s14
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v3, s15
-; GFX90a-NO-PRELOAD-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] offset:16
-; GFX90a-NO-PRELOAD-NEXT:    s_nop 0
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, s8
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s9
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v2, s10
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v3, s11
-; GFX90a-NO-PRELOAD-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1]
-; GFX90a-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-1-LABEL: v8i32_arg:
-; GFX90a-PRELOAD-1:       ; %bb.0:
-; GFX90a-PRELOAD-1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x20
-; GFX90a-PRELOAD-1-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90a-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, s12
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s13
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v2, s14
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v3, s15
-; GFX90a-PRELOAD-1-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] offset:16
-; GFX90a-PRELOAD-1-NEXT:    s_nop 0
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, s8
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s9
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v2, s10
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v3, s11
-; GFX90a-PRELOAD-1-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1]
-; GFX90a-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-2-LABEL: v8i32_arg:
-; GFX90a-PRELOAD-2:       ; %bb.0:
-; GFX90a-PRELOAD-2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x20
-; GFX90a-PRELOAD-2-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90a-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, s12
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s13
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v2, s14
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v3, s15
-; GFX90a-PRELOAD-2-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] offset:16
-; GFX90a-PRELOAD-2-NEXT:    s_nop 0
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, s8
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s9
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v2, s10
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v3, s11
-; GFX90a-PRELOAD-2-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1]
-; GFX90a-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-4-LABEL: v8i32_arg:
-; GFX90a-PRELOAD-4:       ; %bb.0:
-; GFX90a-PRELOAD-4-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x20
-; GFX90a-PRELOAD-4-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90a-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, s12
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s13
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v2, s14
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v3, s15
-; GFX90a-PRELOAD-4-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] offset:16
-; GFX90a-PRELOAD-4-NEXT:    s_nop 0
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, s8
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s9
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v2, s10
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v3, s11
-; GFX90a-PRELOAD-4-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1]
-; GFX90a-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-8-LABEL: v8i32_arg:
-; GFX90a-PRELOAD-8:       ; %bb.0:
-; GFX90a-PRELOAD-8-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x20
-; GFX90a-PRELOAD-8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90a-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, s12
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s13
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v2, s14
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v3, s15
-; GFX90a-PRELOAD-8-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] offset:16
-; GFX90a-PRELOAD-8-NEXT:    s_nop 0
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, s8
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s9
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v2, s10
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v3, s11
-; GFX90a-PRELOAD-8-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1]
-; GFX90a-PRELOAD-8-NEXT:    s_endpgm
+define amdgpu_kernel void @byref_staggered_preload_arg(ptr addrspace(1) inreg %out, ptr addrspace(4) byref(i32) align(256) %in.byref, i32 inreg %after.offset) #0 {
+; GFX940-LABEL: byref_staggered_preload_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x100
+; GFX940-NEXT:    v_mov_b32_e32 v0, 0
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b32_e32 v1, s4
+; GFX940-NEXT:    v_mov_b32_e32 v2, s5
+; GFX940-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    global_store_dword v0, v2, s[2:3] sc0 sc1
+; GFX940-NEXT:    s_waitcnt vmcnt(0)
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: byref_staggered_preload_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x100
+; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
+; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
+; GFX90a-NEXT:    v_mov_b32_e32 v2, s1
+; GFX90a-NEXT:    global_store_dword v0, v1, s[6:7]
+; GFX90a-NEXT:    s_waitcnt vmcnt(0)
+; GFX90a-NEXT:    global_store_dword v0, v2, s[6:7]
+; GFX90a-NEXT:    s_waitcnt vmcnt(0)
+; GFX90a-NEXT:    s_endpgm
+  %in = load i32, ptr addrspace(4) %in.byref
+  store volatile i32 %in, ptr addrspace(1) %out, align 4
+  store volatile i32 %after.offset, ptr addrspace(1) %out, align 4
+  ret void
+}
+
+
+define amdgpu_kernel void @v8i32_arg(ptr addrspace(1) nocapture inreg %out, <8 x i32> inreg %in) #0 {
+; GFX940-LABEL: v8i32_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0x20
+; GFX940-NEXT:    v_mov_b32_e32 v4, 0
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b32_e32 v0, s8
+; GFX940-NEXT:    v_mov_b32_e32 v1, s9
+; GFX940-NEXT:    v_mov_b32_e32 v2, s10
+; GFX940-NEXT:    v_mov_b32_e32 v3, s11
+; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[2:3] offset:16 sc0 sc1
+; GFX940-NEXT:    s_nop 1
+; GFX940-NEXT:    v_mov_b32_e32 v0, s4
+; GFX940-NEXT:    v_mov_b32_e32 v1, s5
+; GFX940-NEXT:    v_mov_b32_e32 v2, s6
+; GFX940-NEXT:    v_mov_b32_e32 v3, s7
+; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[2:3] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: v8i32_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x20
+; GFX90a-NEXT:    v_mov_b32_e32 v4, 0
+; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX90a-NEXT:    v_mov_b32_e32 v0, s12
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s13
+; GFX90a-NEXT:    v_mov_b32_e32 v2, s14
+; GFX90a-NEXT:    v_mov_b32_e32 v3, s15
+; GFX90a-NEXT:    global_store_dwordx4 v4, v[0:3], s[6:7] offset:16
+; GFX90a-NEXT:    s_nop 0
+; GFX90a-NEXT:    v_mov_b32_e32 v0, s8
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s9
+; GFX90a-NEXT:    v_mov_b32_e32 v2, s10
+; GFX90a-NEXT:    v_mov_b32_e32 v3, s11
+; GFX90a-NEXT:    global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX90a-NEXT:    s_endpgm
   store <8 x i32> %in, ptr addrspace(1) %out, align 4
   ret void
 }
 
-define amdgpu_kernel void @v3i16_preload_arg(ptr addrspace(1) nocapture %out, <3 x i16> %in) #0 {
-; GFX940-NO-PRELOAD-LABEL: v3i16_preload_arg:
-; GFX940-NO-PRELOAD:       ; %bb.0:
-; GFX940-NO-PRELOAD-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s3
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v2, s2
-; GFX940-NO-PRELOAD-NEXT:    global_store_short v0, v1, s[0:1] offset:4 sc0 sc1
-; GFX940-NO-PRELOAD-NEXT:    global_store_dword v0, v2, s[0:1] sc0 sc1
-; GFX940-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-1-LABEL: v3i16_preload_arg:
-; GFX940-PRELOAD-1:       ; %bb.0:
-; GFX940-PRELOAD-1-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s3
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v2, s2
-; GFX940-PRELOAD-1-NEXT:    global_store_short v0, v1, s[0:1] offset:4 sc0 sc1
-; GFX940-PRELOAD-1-NEXT:    global_store_dword v0, v2, s[0:1] sc0 sc1
-; GFX940-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-2-LABEL: v3i16_preload_arg:
-; GFX940-PRELOAD-2:       ; %bb.0:
-; GFX940-PRELOAD-2-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s3
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v2, s2
-; GFX940-PRELOAD-2-NEXT:    global_store_short v0, v1, s[0:1] offset:4 sc0 sc1
-; GFX940-PRELOAD-2-NEXT:    global_store_dword v0, v2, s[0:1] sc0 sc1
-; GFX940-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-4-LABEL: v3i16_preload_arg:
-; GFX940-PRELOAD-4:       ; %bb.0:
-; GFX940-PRELOAD-4-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s3
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v2, s2
-; GFX940-PRELOAD-4-NEXT:    global_store_short v0, v1, s[0:1] offset:4 sc0 sc1
-; GFX940-PRELOAD-4-NEXT:    global_store_dword v0, v2, s[0:1] sc0 sc1
-; GFX940-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-8-LABEL: v3i16_preload_arg:
-; GFX940-PRELOAD-8:       ; %bb.0:
-; GFX940-PRELOAD-8-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s3
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v2, s2
-; GFX940-PRELOAD-8-NEXT:    global_store_short v0, v1, s[0:1] offset:4 sc0 sc1
-; GFX940-PRELOAD-8-NEXT:    global_store_dword v0, v2, s[0:1] sc0 sc1
-; GFX940-PRELOAD-8-NEXT:    s_endpgm
-;
-; GFX90a-NO-PRELOAD-LABEL: v3i16_preload_arg:
-; GFX90a-NO-PRELOAD:       ; %bb.0:
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s3
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v2, s2
-; GFX90a-NO-PRELOAD-NEXT:    global_store_short v0, v1, s[0:1] offset:4
-; GFX90a-NO-PRELOAD-NEXT:    global_store_dword v0, v2, s[0:1]
-; GFX90a-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-1-LABEL: v3i16_preload_arg:
-; GFX90a-PRELOAD-1:       ; %bb.0:
-; GFX90a-PRELOAD-1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s3
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v2, s2
-; GFX90a-PRELOAD-1-NEXT:    global_store_short v0, v1, s[0:1] offset:4
-; GFX90a-PRELOAD-1-NEXT:    global_store_dword v0, v2, s[0:1]
-; GFX90a-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-2-LABEL: v3i16_preload_arg:
-; GFX90a-PRELOAD-2:       ; %bb.0:
-; GFX90a-PRELOAD-2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s3
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v2, s2
-; GFX90a-PRELOAD-2-NEXT:    global_store_short v0, v1, s[0:1] offset:4
-; GFX90a-PRELOAD-2-NEXT:    global_store_dword v0, v2, s[0:1]
-; GFX90a-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-4-LABEL: v3i16_preload_arg:
-; GFX90a-PRELOAD-4:       ; %bb.0:
-; GFX90a-PRELOAD-4-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s3
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v2, s2
-; GFX90a-PRELOAD-4-NEXT:    global_store_short v0, v1, s[0:1] offset:4
-; GFX90a-PRELOAD-4-NEXT:    global_store_dword v0, v2, s[0:1]
-; GFX90a-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-8-LABEL: v3i16_preload_arg:
-; GFX90a-PRELOAD-8:       ; %bb.0:
-; GFX90a-PRELOAD-8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s3
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v2, s2
-; GFX90a-PRELOAD-8-NEXT:    global_store_short v0, v1, s[0:1] offset:4
-; GFX90a-PRELOAD-8-NEXT:    global_store_dword v0, v2, s[0:1]
-; GFX90a-PRELOAD-8-NEXT:    s_endpgm
+define amdgpu_kernel void @v3i16_preload_arg(ptr addrspace(1) nocapture inreg %out, <3 x i16> inreg %in) #0 {
+; GFX940-LABEL: v3i16_preload_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    v_mov_b32_e32 v0, 0
+; GFX940-NEXT:    v_mov_b32_e32 v1, s5
+; GFX940-NEXT:    global_store_short v0, v1, s[2:3] offset:4 sc0 sc1
+; GFX940-NEXT:    v_mov_b32_e32 v1, s4
+; GFX940-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: v3i16_preload_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s9
+; GFX90a-NEXT:    global_store_short v0, v1, s[6:7] offset:4
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s8
+; GFX90a-NEXT:    global_store_dword v0, v1, s[6:7]
+; GFX90a-NEXT:    s_endpgm
   store <3 x i16> %in, ptr addrspace(1) %out, align 4
   ret void
 }
 
-define amdgpu_kernel void @v3i32_preload_arg(ptr addrspace(1) nocapture %out, <3 x i32> %in) #0 {
-; GFX940-NO-PRELOAD-LABEL: v3i32_preload_arg:
-; GFX940-NO-PRELOAD:       ; %bb.0:
-; GFX940-NO-PRELOAD-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x10
-; GFX940-NO-PRELOAD-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v3, 0
-; GFX940-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, s4
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s5
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v2, s6
-; GFX940-NO-PRELOAD-NEXT:    global_store_dwordx3 v3, v[0:2], s[2:3] sc0 sc1
-; GFX940-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-1-LABEL: v3i32_preload_arg:
-; GFX940-PRELOAD-1:       ; %bb.0:
-; GFX940-PRELOAD-1-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x10
-; GFX940-PRELOAD-1-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v3, 0
-; GFX940-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, s4
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s5
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v2, s6
-; GFX940-PRELOAD-1-NEXT:    global_store_dwordx3 v3, v[0:2], s[2:3] sc0 sc1
-; GFX940-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-2-LABEL: v3i32_preload_arg:
-; GFX940-PRELOAD-2:       ; %bb.0:
-; GFX940-PRELOAD-2-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x10
-; GFX940-PRELOAD-2-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v3, 0
-; GFX940-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, s4
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s5
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v2, s6
-; GFX940-PRELOAD-2-NEXT:    global_store_dwordx3 v3, v[0:2], s[2:3] sc0 sc1
-; GFX940-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-4-LABEL: v3i32_preload_arg:
-; GFX940-PRELOAD-4:       ; %bb.0:
-; GFX940-PRELOAD-4-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x10
-; GFX940-PRELOAD-4-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v3, 0
-; GFX940-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, s4
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s5
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v2, s6
-; GFX940-PRELOAD-4-NEXT:    global_store_dwordx3 v3, v[0:2], s[2:3] sc0 sc1
-; GFX940-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-8-LABEL: v3i32_preload_arg:
-; GFX940-PRELOAD-8:       ; %bb.0:
-; GFX940-PRELOAD-8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x10
-; GFX940-PRELOAD-8-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v3, 0
-; GFX940-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, s4
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s5
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v2, s6
-; GFX940-PRELOAD-8-NEXT:    global_store_dwordx3 v3, v[0:2], s[2:3] sc0 sc1
-; GFX940-PRELOAD-8-NEXT:    s_endpgm
-;
-; GFX90a-NO-PRELOAD-LABEL: v3i32_preload_arg:
-; GFX90a-NO-PRELOAD:       ; %bb.0:
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x10
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v3, 0
-; GFX90a-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, s0
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s1
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v2, s2
-; GFX90a-NO-PRELOAD-NEXT:    global_store_dwordx3 v3, v[0:2], s[6:7]
-; GFX90a-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-1-LABEL: v3i32_preload_arg:
-; GFX90a-PRELOAD-1:       ; %bb.0:
-; GFX90a-PRELOAD-1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x10
-; GFX90a-PRELOAD-1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v3, 0
-; GFX90a-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, s0
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s1
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v2, s2
-; GFX90a-PRELOAD-1-NEXT:    global_store_dwordx3 v3, v[0:2], s[6:7]
-; GFX90a-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-2-LABEL: v3i32_preload_arg:
-; GFX90a-PRELOAD-2:       ; %bb.0:
-; GFX90a-PRELOAD-2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x10
-; GFX90a-PRELOAD-2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v3, 0
-; GFX90a-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, s0
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s1
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v2, s2
-; GFX90a-PRELOAD-2-NEXT:    global_store_dwordx3 v3, v[0:2], s[6:7]
-; GFX90a-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-4-LABEL: v3i32_preload_arg:
-; GFX90a-PRELOAD-4:       ; %bb.0:
-; GFX90a-PRELOAD-4-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x10
-; GFX90a-PRELOAD-4-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v3, 0
-; GFX90a-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, s0
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s1
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v2, s2
-; GFX90a-PRELOAD-4-NEXT:    global_store_dwordx3 v3, v[0:2], s[6:7]
-; GFX90a-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-8-LABEL: v3i32_preload_arg:
-; GFX90a-PRELOAD-8:       ; %bb.0:
-; GFX90a-PRELOAD-8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x10
-; GFX90a-PRELOAD-8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v3, 0
-; GFX90a-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, s0
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s1
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v2, s2
-; GFX90a-PRELOAD-8-NEXT:    global_store_dwordx3 v3, v[0:2], s[6:7]
-; GFX90a-PRELOAD-8-NEXT:    s_endpgm
+define amdgpu_kernel void @v3i32_preload_arg(ptr addrspace(1) nocapture inreg %out, <3 x i32> inreg %in) #0 {
+; GFX940-LABEL: v3i32_preload_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    v_mov_b32_e32 v0, s6
+; GFX940-NEXT:    v_mov_b32_e32 v1, s7
+; GFX940-NEXT:    v_mov_b32_e32 v2, s8
+; GFX940-NEXT:    v_mov_b32_e32 v3, 0
+; GFX940-NEXT:    global_store_dwordx3 v3, v[0:2], s[2:3] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: v3i32_preload_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    v_mov_b32_e32 v0, s10
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s11
+; GFX90a-NEXT:    v_mov_b32_e32 v2, s12
+; GFX90a-NEXT:    v_mov_b32_e32 v3, 0
+; GFX90a-NEXT:    global_store_dwordx3 v3, v[0:2], s[6:7]
+; GFX90a-NEXT:    s_endpgm
   store <3 x i32> %in, ptr addrspace(1) %out, align 4
   ret void
 }
 
-define amdgpu_kernel void @v3f32_preload_arg(ptr addrspace(1) nocapture %out, <3 x float> %in) #0 {
-; GFX940-NO-PRELOAD-LABEL: v3f32_preload_arg:
-; GFX940-NO-PRELOAD:       ; %bb.0:
-; GFX940-NO-PRELOAD-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x10
-; GFX940-NO-PRELOAD-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v3, 0
-; GFX940-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, s4
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s5
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v2, s6
-; GFX940-NO-PRELOAD-NEXT:    global_store_dwordx3 v3, v[0:2], s[2:3] sc0 sc1
-; GFX940-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-1-LABEL: v3f32_preload_arg:
-; GFX940-PRELOAD-1:       ; %bb.0:
-; GFX940-PRELOAD-1-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x10
-; GFX940-PRELOAD-1-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v3, 0
-; GFX940-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, s4
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s5
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v2, s6
-; GFX940-PRELOAD-1-NEXT:    global_store_dwordx3 v3, v[0:2], s[2:3] sc0 sc1
-; GFX940-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-2-LABEL: v3f32_preload_arg:
-; GFX940-PRELOAD-2:       ; %bb.0:
-; GFX940-PRELOAD-2-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x10
-; GFX940-PRELOAD-2-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v3, 0
-; GFX940-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, s4
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s5
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v2, s6
-; GFX940-PRELOAD-2-NEXT:    global_store_dwordx3 v3, v[0:2], s[2:3] sc0 sc1
-; GFX940-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-4-LABEL: v3f32_preload_arg:
-; GFX940-PRELOAD-4:       ; %bb.0:
-; GFX940-PRELOAD-4-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x10
-; GFX940-PRELOAD-4-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v3, 0
-; GFX940-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, s4
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s5
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v2, s6
-; GFX940-PRELOAD-4-NEXT:    global_store_dwordx3 v3, v[0:2], s[2:3] sc0 sc1
-; GFX940-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-8-LABEL: v3f32_preload_arg:
-; GFX940-PRELOAD-8:       ; %bb.0:
-; GFX940-PRELOAD-8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x10
-; GFX940-PRELOAD-8-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v3, 0
-; GFX940-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, s4
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s5
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v2, s6
-; GFX940-PRELOAD-8-NEXT:    global_store_dwordx3 v3, v[0:2], s[2:3] sc0 sc1
-; GFX940-PRELOAD-8-NEXT:    s_endpgm
-;
-; GFX90a-NO-PRELOAD-LABEL: v3f32_preload_arg:
-; GFX90a-NO-PRELOAD:       ; %bb.0:
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x10
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v3, 0
-; GFX90a-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, s0
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s1
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v2, s2
-; GFX90a-NO-PRELOAD-NEXT:    global_store_dwordx3 v3, v[0:2], s[6:7]
-; GFX90a-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-1-LABEL: v3f32_preload_arg:
-; GFX90a-PRELOAD-1:       ; %bb.0:
-; GFX90a-PRELOAD-1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x10
-; GFX90a-PRELOAD-1-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v3, 0
-; GFX90a-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, s0
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s1
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v2, s2
-; GFX90a-PRELOAD-1-NEXT:    global_store_dwordx3 v3, v[0:2], s[6:7]
-; GFX90a-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-2-LABEL: v3f32_preload_arg:
-; GFX90a-PRELOAD-2:       ; %bb.0:
-; GFX90a-PRELOAD-2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x10
-; GFX90a-PRELOAD-2-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v3, 0
-; GFX90a-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, s0
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s1
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v2, s2
-; GFX90a-PRELOAD-2-NEXT:    global_store_dwordx3 v3, v[0:2], s[6:7]
-; GFX90a-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-4-LABEL: v3f32_preload_arg:
-; GFX90a-PRELOAD-4:       ; %bb.0:
-; GFX90a-PRELOAD-4-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x10
-; GFX90a-PRELOAD-4-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v3, 0
-; GFX90a-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, s0
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s1
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v2, s2
-; GFX90a-PRELOAD-4-NEXT:    global_store_dwordx3 v3, v[0:2], s[6:7]
-; GFX90a-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-8-LABEL: v3f32_preload_arg:
-; GFX90a-PRELOAD-8:       ; %bb.0:
-; GFX90a-PRELOAD-8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x10
-; GFX90a-PRELOAD-8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v3, 0
-; GFX90a-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, s0
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s1
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v2, s2
-; GFX90a-PRELOAD-8-NEXT:    global_store_dwordx3 v3, v[0:2], s[6:7]
-; GFX90a-PRELOAD-8-NEXT:    s_endpgm
+define amdgpu_kernel void @v3f32_preload_arg(ptr addrspace(1) nocapture inreg %out, <3 x float> inreg %in) #0 {
+; GFX940-LABEL: v3f32_preload_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    v_mov_b32_e32 v3, 0
+; GFX940-NEXT:    v_mov_b32_e32 v0, s6
+; GFX940-NEXT:    v_mov_b32_e32 v1, s7
+; GFX940-NEXT:    v_mov_b32_e32 v2, s8
+; GFX940-NEXT:    global_store_dwordx3 v3, v[0:2], s[2:3] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: v3f32_preload_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    v_mov_b32_e32 v3, 0
+; GFX90a-NEXT:    v_mov_b32_e32 v0, s10
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s11
+; GFX90a-NEXT:    v_mov_b32_e32 v2, s12
+; GFX90a-NEXT:    global_store_dwordx3 v3, v[0:2], s[6:7]
+; GFX90a-NEXT:    s_endpgm
   store <3 x float> %in, ptr addrspace(1) %out, align 4
   ret void
 }
 
-define amdgpu_kernel void @v5i8_preload_arg(ptr addrspace(1) nocapture %out, <5 x i8> %in) #0 {
-; GFX940-NO-PRELOAD-LABEL: v5i8_preload_arg:
-; GFX940-NO-PRELOAD:       ; %bb.0:
-; GFX940-NO-PRELOAD-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s3
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v2, s2
-; GFX940-NO-PRELOAD-NEXT:    global_store_byte v0, v1, s[0:1] offset:4 sc0 sc1
-; GFX940-NO-PRELOAD-NEXT:    global_store_dword v0, v2, s[0:1] sc0 sc1
-; GFX940-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-1-LABEL: v5i8_preload_arg:
-; GFX940-PRELOAD-1:       ; %bb.0:
-; GFX940-PRELOAD-1-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s3
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v2, s2
-; GFX940-PRELOAD-1-NEXT:    global_store_byte v0, v1, s[0:1] offset:4 sc0 sc1
-; GFX940-PRELOAD-1-NEXT:    global_store_dword v0, v2, s[0:1] sc0 sc1
-; GFX940-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-2-LABEL: v5i8_preload_arg:
-; GFX940-PRELOAD-2:       ; %bb.0:
-; GFX940-PRELOAD-2-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s3
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v2, s2
-; GFX940-PRELOAD-2-NEXT:    global_store_byte v0, v1, s[0:1] offset:4 sc0 sc1
-; GFX940-PRELOAD-2-NEXT:    global_store_dword v0, v2, s[0:1] sc0 sc1
-; GFX940-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-4-LABEL: v5i8_preload_arg:
-; GFX940-PRELOAD-4:       ; %bb.0:
-; GFX940-PRELOAD-4-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s3
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v2, s2
-; GFX940-PRELOAD-4-NEXT:    global_store_byte v0, v1, s[0:1] offset:4 sc0 sc1
-; GFX940-PRELOAD-4-NEXT:    global_store_dword v0, v2, s[0:1] sc0 sc1
-; GFX940-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-8-LABEL: v5i8_preload_arg:
-; GFX940-PRELOAD-8:       ; %bb.0:
-; GFX940-PRELOAD-8-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, 0
-; GFX940-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s3
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v2, s2
-; GFX940-PRELOAD-8-NEXT:    global_store_byte v0, v1, s[0:1] offset:4 sc0 sc1
-; GFX940-PRELOAD-8-NEXT:    global_store_dword v0, v2, s[0:1] sc0 sc1
-; GFX940-PRELOAD-8-NEXT:    s_endpgm
-;
-; GFX90a-NO-PRELOAD-LABEL: v5i8_preload_arg:
-; GFX90a-NO-PRELOAD:       ; %bb.0:
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s3
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v2, s2
-; GFX90a-NO-PRELOAD-NEXT:    global_store_byte v0, v1, s[0:1] offset:4
-; GFX90a-NO-PRELOAD-NEXT:    global_store_dword v0, v2, s[0:1]
-; GFX90a-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-1-LABEL: v5i8_preload_arg:
-; GFX90a-PRELOAD-1:       ; %bb.0:
-; GFX90a-PRELOAD-1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s3
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v2, s2
-; GFX90a-PRELOAD-1-NEXT:    global_store_byte v0, v1, s[0:1] offset:4
-; GFX90a-PRELOAD-1-NEXT:    global_store_dword v0, v2, s[0:1]
-; GFX90a-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-2-LABEL: v5i8_preload_arg:
-; GFX90a-PRELOAD-2:       ; %bb.0:
-; GFX90a-PRELOAD-2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s3
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v2, s2
-; GFX90a-PRELOAD-2-NEXT:    global_store_byte v0, v1, s[0:1] offset:4
-; GFX90a-PRELOAD-2-NEXT:    global_store_dword v0, v2, s[0:1]
-; GFX90a-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-4-LABEL: v5i8_preload_arg:
-; GFX90a-PRELOAD-4:       ; %bb.0:
-; GFX90a-PRELOAD-4-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s3
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v2, s2
-; GFX90a-PRELOAD-4-NEXT:    global_store_byte v0, v1, s[0:1] offset:4
-; GFX90a-PRELOAD-4-NEXT:    global_store_dword v0, v2, s[0:1]
-; GFX90a-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-8-LABEL: v5i8_preload_arg:
-; GFX90a-PRELOAD-8:       ; %bb.0:
-; GFX90a-PRELOAD-8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, 0
-; GFX90a-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s3
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v2, s2
-; GFX90a-PRELOAD-8-NEXT:    global_store_byte v0, v1, s[0:1] offset:4
-; GFX90a-PRELOAD-8-NEXT:    global_store_dword v0, v2, s[0:1]
-; GFX90a-PRELOAD-8-NEXT:    s_endpgm
+define amdgpu_kernel void @v5i8_preload_arg(ptr addrspace(1) nocapture inreg %out, <5 x i8> inreg %in) #0 {
+; GFX940-LABEL: v5i8_preload_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    s_lshr_b32 s0, s4, 8
+; GFX940-NEXT:    v_lshlrev_b16_e64 v0, 8, s0
+; GFX940-NEXT:    s_lshr_b32 s0, s4, 24
+; GFX940-NEXT:    v_lshlrev_b16_e64 v1, 8, s0
+; GFX940-NEXT:    s_lshr_b32 s0, s4, 16
+; GFX940-NEXT:    v_or_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX940-NEXT:    v_or_b32_sdwa v1, s0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX940-NEXT:    v_mov_b32_e32 v2, s5
+; GFX940-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX940-NEXT:    v_mov_b32_e32 v1, 0
+; GFX940-NEXT:    global_store_byte v1, v2, s[2:3] offset:4 sc0 sc1
+; GFX940-NEXT:    global_store_dword v1, v0, s[2:3] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: v5i8_preload_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    s_lshr_b32 s0, s8, 8
+; GFX90a-NEXT:    v_lshlrev_b16_e64 v0, 8, s0
+; GFX90a-NEXT:    s_lshr_b32 s0, s8, 24
+; GFX90a-NEXT:    v_lshlrev_b16_e64 v1, 8, s0
+; GFX90a-NEXT:    s_lshr_b32 s0, s8, 16
+; GFX90a-NEXT:    v_or_b32_sdwa v0, s8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX90a-NEXT:    v_or_b32_sdwa v1, s0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX90a-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX90a-NEXT:    v_mov_b32_e32 v1, 0
+; GFX90a-NEXT:    v_mov_b32_e32 v2, s9
+; GFX90a-NEXT:    global_store_byte v1, v2, s[6:7] offset:4
+; GFX90a-NEXT:    global_store_dword v1, v0, s[6:7]
+; GFX90a-NEXT:    s_endpgm
   store <5 x i8> %in, ptr addrspace(1) %out, align 4
   ret void
 }
 
-define amdgpu_kernel void @v5f64_arg(ptr addrspace(1) nocapture %out, <5 x double> %in) #0 {
-; GFX940-NO-PRELOAD-LABEL: v5f64_arg:
-; GFX940-NO-PRELOAD:       ; %bb.0:
-; GFX940-NO-PRELOAD-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x60
-; GFX940-NO-PRELOAD-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0x40
-; GFX940-NO-PRELOAD-NEXT:    s_load_dwordx2 s[12:13], s[0:1], 0x0
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, s8
-; GFX940-NO-PRELOAD-NEXT:    global_store_dwordx2 v4, v[2:3], s[12:13] offset:32 sc0 sc1
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s9
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v2, s10
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v3, s11
-; GFX940-NO-PRELOAD-NEXT:    global_store_dwordx4 v4, v[0:3], s[12:13] offset:16 sc0 sc1
-; GFX940-NO-PRELOAD-NEXT:    s_nop 1
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, s4
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s5
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v2, s6
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v3, s7
-; GFX940-NO-PRELOAD-NEXT:    global_store_dwordx4 v4, v[0:3], s[12:13] sc0 sc1
-; GFX940-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-1-LABEL: v5f64_arg:
-; GFX940-PRELOAD-1:       ; %bb.0:
-; GFX940-PRELOAD-1-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x60
-; GFX940-PRELOAD-1-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0x40
-; GFX940-PRELOAD-1-NEXT:    s_load_dwordx2 s[12:13], s[0:1], 0x0
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-1-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, s8
-; GFX940-PRELOAD-1-NEXT:    global_store_dwordx2 v4, v[2:3], s[12:13] offset:32 sc0 sc1
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s9
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v2, s10
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v3, s11
-; GFX940-PRELOAD-1-NEXT:    global_store_dwordx4 v4, v[0:3], s[12:13] offset:16 sc0 sc1
-; GFX940-PRELOAD-1-NEXT:    s_nop 1
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, s4
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s5
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v2, s6
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v3, s7
-; GFX940-PRELOAD-1-NEXT:    global_store_dwordx4 v4, v[0:3], s[12:13] sc0 sc1
-; GFX940-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-2-LABEL: v5f64_arg:
-; GFX940-PRELOAD-2:       ; %bb.0:
-; GFX940-PRELOAD-2-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x60
-; GFX940-PRELOAD-2-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0x40
-; GFX940-PRELOAD-2-NEXT:    s_load_dwordx2 s[12:13], s[0:1], 0x0
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-2-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, s8
-; GFX940-PRELOAD-2-NEXT:    global_store_dwordx2 v4, v[2:3], s[12:13] offset:32 sc0 sc1
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s9
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v2, s10
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v3, s11
-; GFX940-PRELOAD-2-NEXT:    global_store_dwordx4 v4, v[0:3], s[12:13] offset:16 sc0 sc1
-; GFX940-PRELOAD-2-NEXT:    s_nop 1
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, s4
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s5
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v2, s6
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v3, s7
-; GFX940-PRELOAD-2-NEXT:    global_store_dwordx4 v4, v[0:3], s[12:13] sc0 sc1
-; GFX940-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-4-LABEL: v5f64_arg:
-; GFX940-PRELOAD-4:       ; %bb.0:
-; GFX940-PRELOAD-4-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x60
-; GFX940-PRELOAD-4-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0x40
-; GFX940-PRELOAD-4-NEXT:    s_load_dwordx2 s[12:13], s[0:1], 0x0
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-4-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, s8
-; GFX940-PRELOAD-4-NEXT:    global_store_dwordx2 v4, v[2:3], s[12:13] offset:32 sc0 sc1
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s9
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v2, s10
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v3, s11
-; GFX940-PRELOAD-4-NEXT:    global_store_dwordx4 v4, v[0:3], s[12:13] offset:16 sc0 sc1
-; GFX940-PRELOAD-4-NEXT:    s_nop 1
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, s4
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s5
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v2, s6
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v3, s7
-; GFX940-PRELOAD-4-NEXT:    global_store_dwordx4 v4, v[0:3], s[12:13] sc0 sc1
-; GFX940-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-8-LABEL: v5f64_arg:
-; GFX940-PRELOAD-8:       ; %bb.0:
-; GFX940-PRELOAD-8-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x60
-; GFX940-PRELOAD-8-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0x40
-; GFX940-PRELOAD-8-NEXT:    s_load_dwordx2 s[12:13], s[0:1], 0x0
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v4, 0
-; GFX940-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-8-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, s8
-; GFX940-PRELOAD-8-NEXT:    global_store_dwordx2 v4, v[2:3], s[12:13] offset:32 sc0 sc1
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s9
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v2, s10
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v3, s11
-; GFX940-PRELOAD-8-NEXT:    global_store_dwordx4 v4, v[0:3], s[12:13] offset:16 sc0 sc1
-; GFX940-PRELOAD-8-NEXT:    s_nop 1
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, s4
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s5
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v2, s6
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v3, s7
-; GFX940-PRELOAD-8-NEXT:    global_store_dwordx4 v4, v[0:3], s[12:13] sc0 sc1
-; GFX940-PRELOAD-8-NEXT:    s_endpgm
-;
-; GFX90a-NO-PRELOAD-LABEL: v5f64_arg:
-; GFX90a-NO-PRELOAD:       ; %bb.0:
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x60
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x40
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90a-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-NO-PRELOAD-NEXT:    v_pk_mov_b32 v[2:3], s[0:1], s[0:1] op_sel:[0,1]
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, s12
-; GFX90a-NO-PRELOAD-NEXT:    global_store_dwordx2 v4, v[2:3], s[2:3] offset:32
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s13
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v2, s14
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v3, s15
-; GFX90a-NO-PRELOAD-NEXT:    global_store_dwordx4 v4, v[0:3], s[2:3] offset:16
-; GFX90a-NO-PRELOAD-NEXT:    s_nop 0
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, s8
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s9
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v2, s10
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v3, s11
-; GFX90a-NO-PRELOAD-NEXT:    global_store_dwordx4 v4, v[0:3], s[2:3]
-; GFX90a-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-1-LABEL: v5f64_arg:
-; GFX90a-PRELOAD-1:       ; %bb.0:
-; GFX90a-PRELOAD-1-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x60
-; GFX90a-PRELOAD-1-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x40
-; GFX90a-PRELOAD-1-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90a-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-1-NEXT:    v_pk_mov_b32 v[2:3], s[0:1], s[0:1] op_sel:[0,1]
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, s12
-; GFX90a-PRELOAD-1-NEXT:    global_store_dwordx2 v4, v[2:3], s[2:3] offset:32
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s13
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v2, s14
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v3, s15
-; GFX90a-PRELOAD-1-NEXT:    global_store_dwordx4 v4, v[0:3], s[2:3] offset:16
-; GFX90a-PRELOAD-1-NEXT:    s_nop 0
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, s8
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s9
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v2, s10
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v3, s11
-; GFX90a-PRELOAD-1-NEXT:    global_store_dwordx4 v4, v[0:3], s[2:3]
-; GFX90a-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-2-LABEL: v5f64_arg:
-; GFX90a-PRELOAD-2:       ; %bb.0:
-; GFX90a-PRELOAD-2-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x60
-; GFX90a-PRELOAD-2-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x40
-; GFX90a-PRELOAD-2-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90a-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-2-NEXT:    v_pk_mov_b32 v[2:3], s[0:1], s[0:1] op_sel:[0,1]
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, s12
-; GFX90a-PRELOAD-2-NEXT:    global_store_dwordx2 v4, v[2:3], s[2:3] offset:32
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s13
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v2, s14
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v3, s15
-; GFX90a-PRELOAD-2-NEXT:    global_store_dwordx4 v4, v[0:3], s[2:3] offset:16
-; GFX90a-PRELOAD-2-NEXT:    s_nop 0
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, s8
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s9
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v2, s10
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v3, s11
-; GFX90a-PRELOAD-2-NEXT:    global_store_dwordx4 v4, v[0:3], s[2:3]
-; GFX90a-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-4-LABEL: v5f64_arg:
-; GFX90a-PRELOAD-4:       ; %bb.0:
-; GFX90a-PRELOAD-4-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x60
-; GFX90a-PRELOAD-4-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x40
-; GFX90a-PRELOAD-4-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90a-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-4-NEXT:    v_pk_mov_b32 v[2:3], s[0:1], s[0:1] op_sel:[0,1]
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, s12
-; GFX90a-PRELOAD-4-NEXT:    global_store_dwordx2 v4, v[2:3], s[2:3] offset:32
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s13
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v2, s14
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v3, s15
-; GFX90a-PRELOAD-4-NEXT:    global_store_dwordx4 v4, v[0:3], s[2:3] offset:16
-; GFX90a-PRELOAD-4-NEXT:    s_nop 0
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, s8
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s9
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v2, s10
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v3, s11
-; GFX90a-PRELOAD-4-NEXT:    global_store_dwordx4 v4, v[0:3], s[2:3]
-; GFX90a-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-8-LABEL: v5f64_arg:
-; GFX90a-PRELOAD-8:       ; %bb.0:
-; GFX90a-PRELOAD-8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x60
-; GFX90a-PRELOAD-8-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x40
-; GFX90a-PRELOAD-8-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v4, 0
-; GFX90a-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-8-NEXT:    v_pk_mov_b32 v[2:3], s[0:1], s[0:1] op_sel:[0,1]
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, s12
-; GFX90a-PRELOAD-8-NEXT:    global_store_dwordx2 v4, v[2:3], s[2:3] offset:32
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s13
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v2, s14
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v3, s15
-; GFX90a-PRELOAD-8-NEXT:    global_store_dwordx4 v4, v[0:3], s[2:3] offset:16
-; GFX90a-PRELOAD-8-NEXT:    s_nop 0
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, s8
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s9
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v2, s10
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v3, s11
-; GFX90a-PRELOAD-8-NEXT:    global_store_dwordx4 v4, v[0:3], s[2:3]
-; GFX90a-PRELOAD-8-NEXT:    s_endpgm
+define amdgpu_kernel void @v5f64_arg(ptr addrspace(1) nocapture inreg %out, <5 x double> inreg %in) #0 {
+; GFX940-LABEL: v5f64_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    s_load_dwordx2 s[12:13], s[0:1], 0x60
+; GFX940-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0x40
+; GFX940-NEXT:    v_mov_b32_e32 v4, 0
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b64_e32 v[2:3], s[12:13]
+; GFX940-NEXT:    v_mov_b32_e32 v0, s8
+; GFX940-NEXT:    global_store_dwordx2 v4, v[2:3], s[2:3] offset:32 sc0 sc1
+; GFX940-NEXT:    v_mov_b32_e32 v1, s9
+; GFX940-NEXT:    v_mov_b32_e32 v2, s10
+; GFX940-NEXT:    v_mov_b32_e32 v3, s11
+; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[2:3] offset:16 sc0 sc1
+; GFX940-NEXT:    s_nop 1
+; GFX940-NEXT:    v_mov_b32_e32 v0, s4
+; GFX940-NEXT:    v_mov_b32_e32 v1, s5
+; GFX940-NEXT:    v_mov_b32_e32 v2, s6
+; GFX940-NEXT:    v_mov_b32_e32 v3, s7
+; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[2:3] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: v5f64_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x60
+; GFX90a-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x40
+; GFX90a-NEXT:    v_mov_b32_e32 v4, 0
+; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX90a-NEXT:    v_pk_mov_b32 v[2:3], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90a-NEXT:    v_mov_b32_e32 v0, s12
+; GFX90a-NEXT:    global_store_dwordx2 v4, v[2:3], s[6:7] offset:32
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s13
+; GFX90a-NEXT:    v_mov_b32_e32 v2, s14
+; GFX90a-NEXT:    v_mov_b32_e32 v3, s15
+; GFX90a-NEXT:    global_store_dwordx4 v4, v[0:3], s[6:7] offset:16
+; GFX90a-NEXT:    s_nop 0
+; GFX90a-NEXT:    v_mov_b32_e32 v0, s8
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s9
+; GFX90a-NEXT:    v_mov_b32_e32 v2, s10
+; GFX90a-NEXT:    v_mov_b32_e32 v3, s11
+; GFX90a-NEXT:    global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX90a-NEXT:    s_endpgm
   store <5 x double> %in, ptr addrspace(1) %out, align 8
   ret void
 }
 
-define amdgpu_kernel void @v8i8_preload_arg(ptr addrspace(1) %out, <8 x i8> %in) #0 {
-; GFX940-NO-PRELOAD-LABEL: v8i8_preload_arg:
-; GFX940-NO-PRELOAD:       ; %bb.0:
-; GFX940-NO-PRELOAD-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v2, 0
-; GFX940-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
-; GFX940-NO-PRELOAD-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1] sc0 sc1
-; GFX940-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-1-LABEL: v8i8_preload_arg:
-; GFX940-PRELOAD-1:       ; %bb.0:
-; GFX940-PRELOAD-1-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v2, 0
-; GFX940-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-1-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
-; GFX940-PRELOAD-1-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1] sc0 sc1
-; GFX940-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-2-LABEL: v8i8_preload_arg:
-; GFX940-PRELOAD-2:       ; %bb.0:
-; GFX940-PRELOAD-2-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v2, 0
-; GFX940-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-2-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
-; GFX940-PRELOAD-2-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1] sc0 sc1
-; GFX940-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-4-LABEL: v8i8_preload_arg:
-; GFX940-PRELOAD-4:       ; %bb.0:
-; GFX940-PRELOAD-4-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v2, 0
-; GFX940-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-4-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
-; GFX940-PRELOAD-4-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1] sc0 sc1
-; GFX940-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-8-LABEL: v8i8_preload_arg:
-; GFX940-PRELOAD-8:       ; %bb.0:
-; GFX940-PRELOAD-8-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v2, 0
-; GFX940-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-8-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
-; GFX940-PRELOAD-8-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1] sc0 sc1
-; GFX940-PRELOAD-8-NEXT:    s_endpgm
-;
-; GFX90a-NO-PRELOAD-LABEL: v8i8_preload_arg:
-; GFX90a-NO-PRELOAD:       ; %bb.0:
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v2, 0
-; GFX90a-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-NO-PRELOAD-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
-; GFX90a-NO-PRELOAD-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX90a-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-1-LABEL: v8i8_preload_arg:
-; GFX90a-PRELOAD-1:       ; %bb.0:
-; GFX90a-PRELOAD-1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v2, 0
-; GFX90a-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-1-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
-; GFX90a-PRELOAD-1-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX90a-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-2-LABEL: v8i8_preload_arg:
-; GFX90a-PRELOAD-2:       ; %bb.0:
-; GFX90a-PRELOAD-2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v2, 0
-; GFX90a-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-2-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
-; GFX90a-PRELOAD-2-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX90a-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-4-LABEL: v8i8_preload_arg:
-; GFX90a-PRELOAD-4:       ; %bb.0:
-; GFX90a-PRELOAD-4-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v2, 0
-; GFX90a-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-4-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
-; GFX90a-PRELOAD-4-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX90a-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-8-LABEL: v8i8_preload_arg:
-; GFX90a-PRELOAD-8:       ; %bb.0:
-; GFX90a-PRELOAD-8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v2, 0
-; GFX90a-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-8-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
-; GFX90a-PRELOAD-8-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX90a-PRELOAD-8-NEXT:    s_endpgm
+define amdgpu_kernel void @v8i8_preload_arg(ptr addrspace(1) inreg %out, <8 x i8> inreg %in) #0 {
+; GFX940-LABEL: v8i8_preload_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    s_lshr_b32 s0, s5, 8
+; GFX940-NEXT:    v_lshlrev_b16_e64 v0, 8, s0
+; GFX940-NEXT:    s_lshr_b32 s0, s5, 24
+; GFX940-NEXT:    v_lshlrev_b16_e64 v1, 8, s0
+; GFX940-NEXT:    s_lshr_b32 s0, s5, 16
+; GFX940-NEXT:    v_or_b32_sdwa v0, s5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX940-NEXT:    v_or_b32_sdwa v1, s0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX940-NEXT:    s_lshr_b32 s0, s4, 8
+; GFX940-NEXT:    v_or_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX940-NEXT:    v_lshlrev_b16_e64 v0, 8, s0
+; GFX940-NEXT:    s_lshr_b32 s0, s4, 24
+; GFX940-NEXT:    v_lshlrev_b16_e64 v2, 8, s0
+; GFX940-NEXT:    s_lshr_b32 s0, s4, 16
+; GFX940-NEXT:    v_or_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX940-NEXT:    v_or_b32_sdwa v2, s0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX940-NEXT:    s_nop 0
+; GFX940-NEXT:    v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX940-NEXT:    v_mov_b32_e32 v2, 0
+; GFX940-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: v8i8_preload_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    s_lshr_b32 s0, s9, 8
+; GFX90a-NEXT:    v_lshlrev_b16_e64 v0, 8, s0
+; GFX90a-NEXT:    s_lshr_b32 s0, s9, 24
+; GFX90a-NEXT:    v_lshlrev_b16_e64 v1, 8, s0
+; GFX90a-NEXT:    s_lshr_b32 s0, s9, 16
+; GFX90a-NEXT:    v_or_b32_sdwa v0, s9, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX90a-NEXT:    v_or_b32_sdwa v1, s0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX90a-NEXT:    s_lshr_b32 s0, s8, 8
+; GFX90a-NEXT:    v_or_b32_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX90a-NEXT:    v_lshlrev_b16_e64 v0, 8, s0
+; GFX90a-NEXT:    s_lshr_b32 s0, s8, 24
+; GFX90a-NEXT:    v_lshlrev_b16_e64 v2, 8, s0
+; GFX90a-NEXT:    s_lshr_b32 s0, s8, 16
+; GFX90a-NEXT:    v_or_b32_sdwa v0, s8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX90a-NEXT:    v_or_b32_sdwa v2, s0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX90a-NEXT:    v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX90a-NEXT:    v_mov_b32_e32 v2, 0
+; GFX90a-NEXT:    global_store_dwordx2 v2, v[0:1], s[6:7]
+; GFX90a-NEXT:    s_endpgm
   store <8 x i8> %in, ptr addrspace(1) %out
   ret void
 }
 
-define amdgpu_kernel void @i64_kernel_preload_arg(ptr addrspace(1) %out, i64 %a) #0 {
-; GFX940-NO-PRELOAD-LABEL: i64_kernel_preload_arg:
-; GFX940-NO-PRELOAD:       ; %bb.0:
-; GFX940-NO-PRELOAD-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v2, 0
-; GFX940-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, s2
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s3
-; GFX940-NO-PRELOAD-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1] sc0 sc1
-; GFX940-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-1-LABEL: i64_kernel_preload_arg:
-; GFX940-PRELOAD-1:       ; %bb.0:
-; GFX940-PRELOAD-1-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v2, 0
-; GFX940-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, s2
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s3
-; GFX940-PRELOAD-1-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1] sc0 sc1
-; GFX940-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-2-LABEL: i64_kernel_preload_arg:
-; GFX940-PRELOAD-2:       ; %bb.0:
-; GFX940-PRELOAD-2-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v2, 0
-; GFX940-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, s2
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s3
-; GFX940-PRELOAD-2-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1] sc0 sc1
-; GFX940-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-4-LABEL: i64_kernel_preload_arg:
-; GFX940-PRELOAD-4:       ; %bb.0:
-; GFX940-PRELOAD-4-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v2, 0
-; GFX940-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, s2
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s3
-; GFX940-PRELOAD-4-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1] sc0 sc1
-; GFX940-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-8-LABEL: i64_kernel_preload_arg:
-; GFX940-PRELOAD-8:       ; %bb.0:
-; GFX940-PRELOAD-8-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v2, 0
-; GFX940-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, s2
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s3
-; GFX940-PRELOAD-8-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1] sc0 sc1
-; GFX940-PRELOAD-8-NEXT:    s_endpgm
-;
-; GFX90a-NO-PRELOAD-LABEL: i64_kernel_preload_arg:
-; GFX90a-NO-PRELOAD:       ; %bb.0:
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v2, 0
-; GFX90a-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, s2
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s3
-; GFX90a-NO-PRELOAD-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX90a-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-1-LABEL: i64_kernel_preload_arg:
-; GFX90a-PRELOAD-1:       ; %bb.0:
-; GFX90a-PRELOAD-1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v2, 0
-; GFX90a-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, s2
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s3
-; GFX90a-PRELOAD-1-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX90a-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-2-LABEL: i64_kernel_preload_arg:
-; GFX90a-PRELOAD-2:       ; %bb.0:
-; GFX90a-PRELOAD-2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v2, 0
-; GFX90a-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, s2
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s3
-; GFX90a-PRELOAD-2-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX90a-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-4-LABEL: i64_kernel_preload_arg:
-; GFX90a-PRELOAD-4:       ; %bb.0:
-; GFX90a-PRELOAD-4-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v2, 0
-; GFX90a-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, s2
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s3
-; GFX90a-PRELOAD-4-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX90a-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-8-LABEL: i64_kernel_preload_arg:
-; GFX90a-PRELOAD-8:       ; %bb.0:
-; GFX90a-PRELOAD-8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v2, 0
-; GFX90a-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, s2
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s3
-; GFX90a-PRELOAD-8-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX90a-PRELOAD-8-NEXT:    s_endpgm
+define amdgpu_kernel void @i64_kernel_preload_arg(ptr addrspace(1) inreg %out, i64 inreg %a) #0 {
+; GFX940-LABEL: i64_kernel_preload_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    v_mov_b32_e32 v2, 0
+; GFX940-NEXT:    v_mov_b64_e32 v[0:1], s[4:5]
+; GFX940-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: i64_kernel_preload_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    v_mov_b32_e32 v2, 0
+; GFX90a-NEXT:    v_pk_mov_b32 v[0:1], s[8:9], s[8:9] op_sel:[0,1]
+; GFX90a-NEXT:    global_store_dwordx2 v2, v[0:1], s[6:7]
+; GFX90a-NEXT:    s_endpgm
   store i64 %a, ptr addrspace(1) %out, align 8
   ret void
 }
 
-define amdgpu_kernel void @f64_kernel_preload_arg(ptr addrspace(1) %out, double %in) #0 {
-; GFX940-NO-PRELOAD-LABEL: f64_kernel_preload_arg:
-; GFX940-NO-PRELOAD:       ; %bb.0:
-; GFX940-NO-PRELOAD-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v2, 0
-; GFX940-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, s2
-; GFX940-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s3
-; GFX940-NO-PRELOAD-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1] sc0 sc1
-; GFX940-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-1-LABEL: f64_kernel_preload_arg:
-; GFX940-PRELOAD-1:       ; %bb.0:
-; GFX940-PRELOAD-1-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v2, 0
-; GFX940-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, s2
-; GFX940-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s3
-; GFX940-PRELOAD-1-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1] sc0 sc1
-; GFX940-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-2-LABEL: f64_kernel_preload_arg:
-; GFX940-PRELOAD-2:       ; %bb.0:
-; GFX940-PRELOAD-2-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v2, 0
-; GFX940-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, s2
-; GFX940-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s3
-; GFX940-PRELOAD-2-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1] sc0 sc1
-; GFX940-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-4-LABEL: f64_kernel_preload_arg:
-; GFX940-PRELOAD-4:       ; %bb.0:
-; GFX940-PRELOAD-4-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v2, 0
-; GFX940-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, s2
-; GFX940-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s3
-; GFX940-PRELOAD-4-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1] sc0 sc1
-; GFX940-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX940-PRELOAD-8-LABEL: f64_kernel_preload_arg:
-; GFX940-PRELOAD-8:       ; %bb.0:
-; GFX940-PRELOAD-8-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v2, 0
-; GFX940-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, s2
-; GFX940-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s3
-; GFX940-PRELOAD-8-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1] sc0 sc1
-; GFX940-PRELOAD-8-NEXT:    s_endpgm
-;
-; GFX90a-NO-PRELOAD-LABEL: f64_kernel_preload_arg:
-; GFX90a-NO-PRELOAD:       ; %bb.0:
-; GFX90a-NO-PRELOAD-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v2, 0
-; GFX90a-NO-PRELOAD-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v0, s2
-; GFX90a-NO-PRELOAD-NEXT:    v_mov_b32_e32 v1, s3
-; GFX90a-NO-PRELOAD-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX90a-NO-PRELOAD-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-1-LABEL: f64_kernel_preload_arg:
-; GFX90a-PRELOAD-1:       ; %bb.0:
-; GFX90a-PRELOAD-1-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v2, 0
-; GFX90a-PRELOAD-1-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v0, s2
-; GFX90a-PRELOAD-1-NEXT:    v_mov_b32_e32 v1, s3
-; GFX90a-PRELOAD-1-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX90a-PRELOAD-1-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-2-LABEL: f64_kernel_preload_arg:
-; GFX90a-PRELOAD-2:       ; %bb.0:
-; GFX90a-PRELOAD-2-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v2, 0
-; GFX90a-PRELOAD-2-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v0, s2
-; GFX90a-PRELOAD-2-NEXT:    v_mov_b32_e32 v1, s3
-; GFX90a-PRELOAD-2-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX90a-PRELOAD-2-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-4-LABEL: f64_kernel_preload_arg:
-; GFX90a-PRELOAD-4:       ; %bb.0:
-; GFX90a-PRELOAD-4-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v2, 0
-; GFX90a-PRELOAD-4-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v0, s2
-; GFX90a-PRELOAD-4-NEXT:    v_mov_b32_e32 v1, s3
-; GFX90a-PRELOAD-4-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX90a-PRELOAD-4-NEXT:    s_endpgm
-;
-; GFX90a-PRELOAD-8-LABEL: f64_kernel_preload_arg:
-; GFX90a-PRELOAD-8:       ; %bb.0:
-; GFX90a-PRELOAD-8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v2, 0
-; GFX90a-PRELOAD-8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v0, s2
-; GFX90a-PRELOAD-8-NEXT:    v_mov_b32_e32 v1, s3
-; GFX90a-PRELOAD-8-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX90a-PRELOAD-8-NEXT:    s_endpgm
+define amdgpu_kernel void @f64_kernel_preload_arg(ptr addrspace(1) inreg %out, double inreg %in) #0 {
+; GFX940-LABEL: f64_kernel_preload_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    v_mov_b32_e32 v2, 0
+; GFX940-NEXT:    v_mov_b64_e32 v[0:1], s[4:5]
+; GFX940-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: f64_kernel_preload_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    v_mov_b32_e32 v2, 0
+; GFX90a-NEXT:    v_pk_mov_b32 v[0:1], s[8:9], s[8:9] op_sel:[0,1]
+; GFX90a-NEXT:    global_store_dwordx2 v2, v[0:1], s[6:7]
+; GFX90a-NEXT:    s_endpgm
   store double %in, ptr addrspace(1) %out
   ret void
 }
 
-attributes #0 = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" }
+define amdgpu_kernel void @half_kernel_preload_arg(ptr addrspace(1) inreg %out, half inreg %in) #0 {
+; GFX940-LABEL: half_kernel_preload_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    v_mov_b32_e32 v0, 0
+; GFX940-NEXT:    v_mov_b32_e32 v1, s4
+; GFX940-NEXT:    global_store_short v0, v1, s[2:3] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: half_kernel_preload_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s8
+; GFX90a-NEXT:    global_store_short v0, v1, s[6:7]
+; GFX90a-NEXT:    s_endpgm
+  store half %in, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @bfloat_kernel_preload_arg(ptr addrspace(1) inreg %out, bfloat inreg %in) #0 {
+; GFX940-LABEL: bfloat_kernel_preload_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    v_mov_b32_e32 v0, 0
+; GFX940-NEXT:    v_mov_b32_e32 v1, s4
+; GFX940-NEXT:    global_store_short v0, v1, s[2:3] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: bfloat_kernel_preload_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s8
+; GFX90a-NEXT:    global_store_short v0, v1, s[6:7]
+; GFX90a-NEXT:    s_endpgm
+  store bfloat %in, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @v2bfloat_kernel_preload_arg(ptr addrspace(1) inreg %out, <2 x bfloat> inreg %in) #0 {
+; GFX940-LABEL: v2bfloat_kernel_preload_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    v_mov_b32_e32 v0, 0
+; GFX940-NEXT:    v_mov_b32_e32 v1, s4
+; GFX940-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: v2bfloat_kernel_preload_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s8
+; GFX90a-NEXT:    global_store_dword v0, v1, s[6:7]
+; GFX90a-NEXT:    s_endpgm
+  store <2 x bfloat> %in, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @v3bfloat_kernel_preload_arg(ptr addrspace(1) inreg %out, <3 x bfloat> inreg %in) #0 {
+; GFX940-LABEL: v3bfloat_kernel_preload_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    v_mov_b32_e32 v0, 0
+; GFX940-NEXT:    v_mov_b32_e32 v1, s5
+; GFX940-NEXT:    global_store_short v0, v1, s[2:3] offset:4 sc0 sc1
+; GFX940-NEXT:    v_mov_b32_e32 v1, s4
+; GFX940-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: v3bfloat_kernel_preload_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s9
+; GFX90a-NEXT:    global_store_short v0, v1, s[6:7] offset:4
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s8
+; GFX90a-NEXT:    global_store_dword v0, v1, s[6:7]
+; GFX90a-NEXT:    s_endpgm
+  store <3 x bfloat> %in, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @v6bfloat_kernel_preload_arg(ptr addrspace(1) inreg %out, <6 x bfloat> inreg %in) #0 {
+; GFX940-LABEL: v6bfloat_kernel_preload_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    v_mov_b32_e32 v0, s6
+; GFX940-NEXT:    v_mov_b32_e32 v1, s7
+; GFX940-NEXT:    v_mov_b32_e32 v2, s8
+; GFX940-NEXT:    v_mov_b32_e32 v3, 0
+; GFX940-NEXT:    global_store_dwordx3 v3, v[0:2], s[2:3] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: v6bfloat_kernel_preload_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    v_mov_b32_e32 v0, s10
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s11
+; GFX90a-NEXT:    v_mov_b32_e32 v2, s12
+; GFX90a-NEXT:    v_mov_b32_e32 v3, 0
+; GFX90a-NEXT:    global_store_dwordx3 v3, v[0:2], s[6:7]
+; GFX90a-NEXT:    s_endpgm
+  store <6 x bfloat> %in, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @half_v7bfloat_kernel_preload_arg(ptr addrspace(1) inreg %out, half inreg %in, <7 x bfloat> inreg %in2, ptr addrspace(1) inreg %out2) #0 {
+; GFX940-LABEL: half_v7bfloat_kernel_preload_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    v_mov_b32_e32 v3, 0
+; GFX940-NEXT:    v_mov_b32_e32 v0, s4
+; GFX940-NEXT:    global_store_short v3, v0, s[2:3] sc0 sc1
+; GFX940-NEXT:    v_mov_b32_e32 v0, s9
+; GFX940-NEXT:    global_store_short v3, v0, s[10:11] offset:12 sc0 sc1
+; GFX940-NEXT:    v_mov_b32_e32 v2, s8
+; GFX940-NEXT:    v_mov_b32_e32 v0, s6
+; GFX940-NEXT:    v_mov_b32_e32 v1, s7
+; GFX940-NEXT:    global_store_dwordx3 v3, v[0:2], s[10:11] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: half_v7bfloat_kernel_preload_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x20
+; GFX90a-NEXT:    v_mov_b32_e32 v3, 0
+; GFX90a-NEXT:    v_mov_b32_e32 v0, s8
+; GFX90a-NEXT:    global_store_short v3, v0, s[6:7]
+; GFX90a-NEXT:    v_mov_b32_e32 v0, s13
+; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX90a-NEXT:    global_store_short v3, v0, s[0:1] offset:12
+; GFX90a-NEXT:    v_mov_b32_e32 v2, s12
+; GFX90a-NEXT:    v_mov_b32_e32 v0, s10
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s11
+; GFX90a-NEXT:    global_store_dwordx3 v3, v[0:2], s[0:1]
+; GFX90a-NEXT:    s_endpgm
+  store half %in, ptr addrspace(1) %out
+  store <7 x bfloat> %in2, ptr addrspace(1) %out2
+  ret void
+}
+
+define amdgpu_kernel void @i1_kernel_preload_arg(ptr addrspace(1) inreg %out, i1 inreg %in) #0 {
+; GFX940-LABEL: i1_kernel_preload_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    s_and_b32 s0, s4, 1
+; GFX940-NEXT:    v_mov_b32_e32 v0, 0
+; GFX940-NEXT:    v_mov_b32_e32 v1, s0
+; GFX940-NEXT:    global_store_byte v0, v1, s[2:3] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: i1_kernel_preload_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    s_and_b32 s0, s8, 1
+; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
+; GFX90a-NEXT:    global_store_byte v0, v1, s[6:7]
+; GFX90a-NEXT:    s_endpgm
+  store i1 %in, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @fp128_kernel_preload_arg(ptr addrspace(1) inreg %out, fp128 inreg %in) #0 {
+; GFX940-LABEL: fp128_kernel_preload_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    v_mov_b32_e32 v4, 0
+; GFX940-NEXT:    v_mov_b32_e32 v0, s6
+; GFX940-NEXT:    v_mov_b32_e32 v1, s7
+; GFX940-NEXT:    v_mov_b32_e32 v2, s8
+; GFX940-NEXT:    v_mov_b32_e32 v3, s9
+; GFX940-NEXT:    global_store_dwordx4 v4, v[0:3], s[2:3] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: fp128_kernel_preload_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    v_mov_b32_e32 v4, 0
+; GFX90a-NEXT:    v_mov_b32_e32 v0, s10
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s11
+; GFX90a-NEXT:    v_mov_b32_e32 v2, s12
+; GFX90a-NEXT:    v_mov_b32_e32 v3, s13
+; GFX90a-NEXT:    global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX90a-NEXT:    s_endpgm
+  store fp128 %in, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @v7i8_kernel_preload_arg(ptr addrspace(1) inreg %out, <7 x i8> inreg %in) #0 {
+; GFX940-LABEL: v7i8_kernel_preload_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    s_lshr_b32 s0, s4, 8
+; GFX940-NEXT:    v_lshlrev_b16_e64 v0, 8, s0
+; GFX940-NEXT:    s_lshr_b32 s0, s4, 24
+; GFX940-NEXT:    v_lshlrev_b16_e64 v1, 8, s0
+; GFX940-NEXT:    s_lshr_b32 s0, s4, 16
+; GFX940-NEXT:    v_or_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX940-NEXT:    v_or_b32_sdwa v1, s0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX940-NEXT:    s_lshr_b32 s0, s5, 8
+; GFX940-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX940-NEXT:    v_lshlrev_b16_e64 v1, 8, s0
+; GFX940-NEXT:    v_mov_b32_e32 v2, 0
+; GFX940-NEXT:    v_mov_b32_e32 v3, s5
+; GFX940-NEXT:    v_or_b32_sdwa v1, s5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX940-NEXT:    global_store_byte_d16_hi v2, v3, s[2:3] offset:6 sc0 sc1
+; GFX940-NEXT:    global_store_short v2, v1, s[2:3] offset:4 sc0 sc1
+; GFX940-NEXT:    global_store_dword v2, v0, s[2:3] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: v7i8_kernel_preload_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    s_lshr_b32 s0, s8, 8
+; GFX90a-NEXT:    v_lshlrev_b16_e64 v0, 8, s0
+; GFX90a-NEXT:    s_lshr_b32 s0, s8, 24
+; GFX90a-NEXT:    v_lshlrev_b16_e64 v1, 8, s0
+; GFX90a-NEXT:    s_lshr_b32 s0, s8, 16
+; GFX90a-NEXT:    v_or_b32_sdwa v0, s8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX90a-NEXT:    v_or_b32_sdwa v1, s0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX90a-NEXT:    s_lshr_b32 s0, s9, 8
+; GFX90a-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX90a-NEXT:    v_lshlrev_b16_e64 v1, 8, s0
+; GFX90a-NEXT:    v_mov_b32_e32 v2, 0
+; GFX90a-NEXT:    v_mov_b32_e32 v3, s9
+; GFX90a-NEXT:    v_or_b32_sdwa v1, s9, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX90a-NEXT:    global_store_byte_d16_hi v2, v3, s[6:7] offset:6
+; GFX90a-NEXT:    global_store_short v2, v1, s[6:7] offset:4
+; GFX90a-NEXT:    global_store_dword v2, v0, s[6:7]
+; GFX90a-NEXT:    s_endpgm
+  store <7 x i8> %in, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @v7half_kernel_preload_arg(ptr addrspace(1) inreg %out, <7 x half> inreg %in) #0 {
+; GFX940-LABEL: v7half_kernel_preload_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    v_mov_b32_e32 v3, 0
+; GFX940-NEXT:    v_mov_b32_e32 v0, s9
+; GFX940-NEXT:    global_store_short v3, v0, s[2:3] offset:12 sc0 sc1
+; GFX940-NEXT:    v_mov_b32_e32 v2, s8
+; GFX940-NEXT:    v_mov_b32_e32 v0, s6
+; GFX940-NEXT:    v_mov_b32_e32 v1, s7
+; GFX940-NEXT:    global_store_dwordx3 v3, v[0:2], s[2:3] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: v7half_kernel_preload_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    v_mov_b32_e32 v3, 0
+; GFX90a-NEXT:    v_mov_b32_e32 v0, s13
+; GFX90a-NEXT:    global_store_short v3, v0, s[6:7] offset:12
+; GFX90a-NEXT:    v_mov_b32_e32 v2, s12
+; GFX90a-NEXT:    v_mov_b32_e32 v0, s10
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s11
+; GFX90a-NEXT:    global_store_dwordx3 v3, v[0:2], s[6:7]
+; GFX90a-NEXT:    s_endpgm
+  store <7 x half> %in, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @i16_i32_kernel_preload_arg(ptr addrspace(1) %out, i16 inreg %in, i32 inreg %in2, ptr addrspace(1) inreg %out2) #0 {
+; GFX940-LABEL: i16_i32_kernel_preload_arg:
+; GFX940:       ; %bb.0:
+; GFX940-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x0
+; GFX940-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x10
+; GFX940-NEXT:    v_mov_b32_e32 v0, 0
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    v_mov_b32_e32 v1, s6
+; GFX940-NEXT:    v_mov_b32_e32 v2, s7
+; GFX940-NEXT:    global_store_short v0, v1, s[4:5] sc0 sc1
+; GFX940-NEXT:    global_store_dword v0, v2, s[2:3] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: i16_i32_kernel_preload_arg:
+; GFX90a:       ; %bb.0:
+; GFX90a-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX90a-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x10
+; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
+; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s2
+; GFX90a-NEXT:    v_mov_b32_e32 v2, s3
+; GFX90a-NEXT:    global_store_short v0, v1, s[0:1]
+; GFX90a-NEXT:    global_store_dword v0, v2, s[6:7]
+; GFX90a-NEXT:    s_endpgm
+  store i16 %in, ptr addrspace(1) %out
+  store i32 %in2, ptr addrspace(1) %out2
+  ret void
+}
+
+define amdgpu_kernel void @i16_v3i32_kernel_preload_arg(ptr addrspace(1) inreg %out, i16 inreg %in, <3 x i32> inreg %in2, ptr addrspace(1) inreg %out2) #0 {
+; GFX940-LABEL: i16_v3i32_kernel_preload_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    v_mov_b32_e32 v3, 0
+; GFX940-NEXT:    v_mov_b32_e32 v4, s4
+; GFX940-NEXT:    v_mov_b32_e32 v0, s6
+; GFX940-NEXT:    v_mov_b32_e32 v1, s7
+; GFX940-NEXT:    v_mov_b32_e32 v2, s8
+; GFX940-NEXT:    global_store_short v3, v4, s[2:3] sc0 sc1
+; GFX940-NEXT:    global_store_dwordx3 v3, v[0:2], s[10:11] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: i16_v3i32_kernel_preload_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x20
+; GFX90a-NEXT:    v_mov_b32_e32 v3, 0
+; GFX90a-NEXT:    v_mov_b32_e32 v4, s8
+; GFX90a-NEXT:    v_mov_b32_e32 v0, s10
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s11
+; GFX90a-NEXT:    v_mov_b32_e32 v2, s12
+; GFX90a-NEXT:    global_store_short v3, v4, s[6:7]
+; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX90a-NEXT:    global_store_dwordx3 v3, v[0:2], s[0:1]
+; GFX90a-NEXT:    s_endpgm
+  store i16 %in, ptr addrspace(1) %out
+  store <3 x i32> %in2, ptr addrspace(1) %out2
+  ret void
+}
+
+define amdgpu_kernel void @i16_i16_kernel_preload_arg(ptr addrspace(1) inreg %out, i16 inreg %in, i16 inreg %in2, ptr addrspace(1) inreg %out2) #0 {
+; GFX940-LABEL: i16_i16_kernel_preload_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    v_mov_b32_e32 v0, 0
+; GFX940-NEXT:    v_mov_b32_e32 v1, s4
+; GFX940-NEXT:    global_store_short v0, v1, s[2:3] sc0 sc1
+; GFX940-NEXT:    global_store_short_d16_hi v0, v1, s[6:7] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: i16_i16_kernel_preload_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s8
+; GFX90a-NEXT:    global_store_short v0, v1, s[6:7]
+; GFX90a-NEXT:    global_store_short_d16_hi v0, v1, s[10:11]
+; GFX90a-NEXT:    s_endpgm
+  store i16 %in, ptr addrspace(1) %out
+  store i16 %in2, ptr addrspace(1) %out2
+  ret void
+}
+
+define amdgpu_kernel void @i16_v2i8_kernel_preload_arg(ptr addrspace(1) inreg %out, i16 inreg %in, <2 x i8> inreg %in2, ptr addrspace(1) inreg %out2) #0 {
+; GFX940-LABEL: i16_v2i8_kernel_preload_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    s_lshr_b32 s0, s4, 24
+; GFX940-NEXT:    v_lshlrev_b16_e64 v0, 8, s0
+; GFX940-NEXT:    s_lshr_b32 s0, s4, 16
+; GFX940-NEXT:    v_mov_b32_e32 v1, 0
+; GFX940-NEXT:    v_mov_b32_e32 v2, s4
+; GFX940-NEXT:    v_or_b32_sdwa v0, s0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX940-NEXT:    global_store_short v1, v2, s[2:3] sc0 sc1
+; GFX940-NEXT:    global_store_short v1, v0, s[6:7] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: i16_v2i8_kernel_preload_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    s_lshr_b32 s0, s8, 24
+; GFX90a-NEXT:    v_lshlrev_b16_e64 v0, 8, s0
+; GFX90a-NEXT:    s_lshr_b32 s0, s8, 16
+; GFX90a-NEXT:    v_mov_b32_e32 v1, 0
+; GFX90a-NEXT:    v_mov_b32_e32 v2, s8
+; GFX90a-NEXT:    v_or_b32_sdwa v0, s0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX90a-NEXT:    global_store_short v1, v2, s[6:7]
+; GFX90a-NEXT:    global_store_short v1, v0, s[10:11]
+; GFX90a-NEXT:    s_endpgm
+  store i16 %in, ptr addrspace(1) %out
+  store <2 x i8> %in2, ptr addrspace(1) %out2
+  ret void
+}
+
+; The second argument is not expected to be preloaded with the current behavior.
+
+define amdgpu_kernel void @i32_ptr1_i32_staggered_preload_arg(i32 inreg %arg0, ptr addrspace(1) %out, i32 inreg %arg1) #0 {
+; GFX940-LABEL: i32_ptr1_i32_staggered_preload_arg:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    s_load_dword s3, s[0:1], 0x10
+; GFX940-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x8
+; GFX940-NEXT:    v_mov_b32_e32 v0, 0
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    s_add_i32 s0, s2, s3
+; GFX940-NEXT:    v_mov_b32_e32 v1, s0
+; GFX940-NEXT:    global_store_dword v0, v1, s[4:5] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: i32_ptr1_i32_staggered_preload_arg:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    s_load_dword s2, s[4:5], 0x10
+; GFX90a-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
+; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX90a-NEXT:    s_add_i32 s2, s6, s2
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s2
+; GFX90a-NEXT:    global_store_dword v0, v1, s[0:1]
+; GFX90a-NEXT:    s_endpgm
+  %add = add i32 %arg0, %arg1
+  store i32 %add, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @ptr1_i8_trailing_unused(ptr addrspace(1) inreg %out, i8 inreg %arg0, i32 inreg %unused) #0 {
+; GFX940-LABEL: ptr1_i8_trailing_unused:
+; GFX940:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX940-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX940-NEXT:  ; %bb.0:
+; GFX940-NEXT:    s_and_b32 s0, s4, 0xff
+; GFX940-NEXT:    v_mov_b32_e32 v0, 0
+; GFX940-NEXT:    v_mov_b32_e32 v1, s0
+; GFX940-NEXT:    global_store_dword v0, v1, s[2:3] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+;
+; GFX90a-LABEL: ptr1_i8_trailing_unused:
+; GFX90a:         s_trap 2 ; Kernarg preload header. Trap with incompatible firmware that doesn't support preloading kernel arguments.
+; GFX90a-NEXT:    .fill 63, 4, 0xbf800000 ; s_nop 0
+; GFX90a-NEXT:  ; %bb.0:
+; GFX90a-NEXT:    s_and_b32 s0, s8, 0xff
+; GFX90a-NEXT:    v_mov_b32_e32 v0, 0
+; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
+; GFX90a-NEXT:    global_store_dword v0, v1, s[6:7]
+; GFX90a-NEXT:    s_endpgm
+  %ext = zext i8 %arg0 to i32
+  store i32 %ext, ptr addrspace(1) %out
+  ret void
+}
+
+attributes #0 = { "amdgpu-no-agpr" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "uniform-work-group-size"="false" }

From 4eefc8d4cee1808f44710622c8c3b66281feb8a3 Mon Sep 17 00:00:00 2001
From: Bimo <rui.xu@intel.com>
Date: Mon, 19 Aug 2024 09:06:48 +0800
Subject: [PATCH 319/441] [MLIR][Python] enhance python api for tensor.empty
 (#103087)

Since we have extended `EmptyOp`, maybe we should also provide a
corresponding `tensor.empty` method. In the downstream usage, I tend to
use APIs with all lowercase letters to create ops, so having a
`tensor.empty` to replace the extended `tensor.EmptyOp` would keep my
code style consistent.
---
 mlir/python/mlir/dialects/tensor.py                 | 13 +++++++++++++
 .../python/dialects/linalg/opdsl/emit_matmul.py     |  4 ++--
 mlir/test/python/dialects/linalg/ops.py             |  4 ++--
 3 files changed, 17 insertions(+), 4 deletions(-)

diff --git a/mlir/python/mlir/dialects/tensor.py b/mlir/python/mlir/dialects/tensor.py
index 79dd9476ad0ff91..0b30d1020990885 100644
--- a/mlir/python/mlir/dialects/tensor.py
+++ b/mlir/python/mlir/dialects/tensor.py
@@ -13,6 +13,7 @@
 
 from typing import Sequence, Union
 from ._ods_common import _cext as _ods_cext
+from ._ods_common import get_op_result_or_op_results as _get_op_result_or_op_results
 
 
 @_ods_cext.register_operation(_Dialect, replace=True)
@@ -43,6 +44,18 @@ def __init__(
         super().__init__(result_type, dynamic_sizes, loc=loc, ip=ip)
 
 
+def empty(
+    sizes: Sequence[Union[int, Value]],
+    element_type: Type,
+    *,
+    loc=None,
+    ip=None,
+) -> _ods_cext.ir.Value:
+    return _get_op_result_or_op_results(
+        EmptyOp(sizes=sizes, element_type=element_type, loc=loc, ip=ip)
+    )
+
+
 generate = region_op(
     lambda result, dynamic_extents: GenerateOp(result, dynamic_extents),
     terminator=lambda args: YieldOp(args[0]),
diff --git a/mlir/test/python/dialects/linalg/opdsl/emit_matmul.py b/mlir/test/python/dialects/linalg/opdsl/emit_matmul.py
index 18c237c68081a17..64df4e1276222f8 100644
--- a/mlir/test/python/dialects/linalg/opdsl/emit_matmul.py
+++ b/mlir/test/python/dialects/linalg/opdsl/emit_matmul.py
@@ -63,8 +63,8 @@ def matmul_poly(
             RankedTensorType.get((4, 16), f32), RankedTensorType.get((16, 8), f32)
         )
         def test_matmul_mono(lhs, rhs):
-            init_result = tensor.EmptyOp([4, 8], f32)
-            return matmul_mono(lhs, rhs, outs=[init_result.result])
+            init_result = tensor.empty([4, 8], f32)
+            return matmul_mono(lhs, rhs, outs=[init_result])
 
         # CHECK-LABEL: @test_i8i8i32_matmul
         # CHECK:      ^{{.*}}(%[[A_ARG:.+]]: i8, %[[B_ARG:.+]]: i8, %[[C_ARG:.+]]: i32)
diff --git a/mlir/test/python/dialects/linalg/ops.py b/mlir/test/python/dialects/linalg/ops.py
index b147551c2e73dbd..3bfbcf7d7f7c81d 100644
--- a/mlir/test/python/dialects/linalg/ops.py
+++ b/mlir/test/python/dialects/linalg/ops.py
@@ -97,7 +97,7 @@ def testNamedStructuredOpGenericForm():
                 RankedTensorType.get((4, 16), f32), RankedTensorType.get((16, 8), f32)
             )
             def named_form(lhs, rhs):
-                init_result = tensor.EmptyOp([4, 8], f32)
+                init_result = tensor.empty([4, 8], f32)
                 #      CHECK: "linalg.matmul"(%{{.*}})
                 # CHECK-SAME:    cast = #linalg.type_fn<cast_signed>
                 # CHECK-SAME:    operandSegmentSizes = array<i32: 2, 1>
@@ -106,7 +106,7 @@ def named_form(lhs, rhs):
                 # CHECK-NEXT:    arith.addf{{.*}} (f32, f32) -> f32
                 # CHECK-NEXT:    linalg.yield{{.*}} (f32) -> ()
                 # CHECK-NEXT: (tensor<4x16xf32>, tensor<16x8xf32>, tensor<4x8xf32>) -> tensor<4x8xf32>
-                return linalg.matmul(lhs, rhs, outs=[init_result.result])
+                return linalg.matmul(lhs, rhs, outs=[init_result])
 
     module.operation.print(print_generic_op_form=True)
 

From 7f87b5bf0e2f0e01735e5abe724d7fe5c69d202e Mon Sep 17 00:00:00 2001
From: Tianqing Wang <tianqing.wang@intel.com>
Date: Mon, 19 Aug 2024 11:04:48 +0800
Subject: [PATCH 320/441] [SelectionDAG][X86] Preserve unpredictable metadata
 for conditional branches in SelectionDAG, as well as JCCs generated by X86
 backend. (#102101)

This builds on 09515f2c2, which preserves unpredictable metadata in
CodeGen for `select`. This patch does it for conditional branches.
---
 llvm/include/llvm/CodeGen/SelectionDAG.h      |  5 ++++
 .../llvm/CodeGen/SwitchLoweringUtils.h        | 15 +++++++----
 llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp |  7 ++---
 .../lib/CodeGen/SelectionDAG/InstrEmitter.cpp | 12 ++++-----
 .../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 26 ++++++++++++++++---
 .../SelectionDAG/SelectionDAGBuilder.cpp      | 14 ++++++----
 llvm/lib/Target/X86/X86ISelLowering.cpp       | 22 ++++++++--------
 llvm/test/CodeGen/X86/unpredictable-brcond.ll | 10 +++----
 8 files changed, 72 insertions(+), 39 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h
index ade282bd7e2e299..1514d92b36b3c24 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAG.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAG.h
@@ -1163,8 +1163,13 @@ class SelectionDAG {
                   SDValue N2, SDValue N3, const SDNodeFlags Flags);
   SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1,
                   SDValue N2, SDValue N3, SDValue N4);
+  SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1,
+                  SDValue N2, SDValue N3, SDValue N4, const SDNodeFlags Flags);
   SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1,
                   SDValue N2, SDValue N3, SDValue N4, SDValue N5);
+  SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1,
+                  SDValue N2, SDValue N3, SDValue N4, SDValue N5,
+                  const SDNodeFlags Flags);
 
   // Specialize again based on number of operands for nodes with a VTList
   // rather than a single VT.
diff --git a/llvm/include/llvm/CodeGen/SwitchLoweringUtils.h b/llvm/include/llvm/CodeGen/SwitchLoweringUtils.h
index 99478e9f39e2260..9282c4a771afb2c 100644
--- a/llvm/include/llvm/CodeGen/SwitchLoweringUtils.h
+++ b/llvm/include/llvm/CodeGen/SwitchLoweringUtils.h
@@ -137,18 +137,21 @@ struct CaseBlock {
   SDLoc DL;
   DebugLoc DbgLoc;
 
-  // Branch weights.
+  // Branch weights and predictability.
   BranchProbability TrueProb, FalseProb;
+  bool IsUnpredictable;
 
   // Constructor for SelectionDAG.
   CaseBlock(ISD::CondCode cc, const Value *cmplhs, const Value *cmprhs,
             const Value *cmpmiddle, MachineBasicBlock *truebb,
             MachineBasicBlock *falsebb, MachineBasicBlock *me, SDLoc dl,
             BranchProbability trueprob = BranchProbability::getUnknown(),
-            BranchProbability falseprob = BranchProbability::getUnknown())
+            BranchProbability falseprob = BranchProbability::getUnknown(),
+            bool isunpredictable = false)
       : CC(cc), CmpLHS(cmplhs), CmpMHS(cmpmiddle), CmpRHS(cmprhs),
         TrueBB(truebb), FalseBB(falsebb), ThisBB(me), DL(dl),
-        TrueProb(trueprob), FalseProb(falseprob) {}
+        TrueProb(trueprob), FalseProb(falseprob),
+        IsUnpredictable(isunpredictable) {}
 
   // Constructor for GISel.
   CaseBlock(CmpInst::Predicate pred, bool nocmp, const Value *cmplhs,
@@ -156,10 +159,12 @@ struct CaseBlock {
             MachineBasicBlock *truebb, MachineBasicBlock *falsebb,
             MachineBasicBlock *me, DebugLoc dl,
             BranchProbability trueprob = BranchProbability::getUnknown(),
-            BranchProbability falseprob = BranchProbability::getUnknown())
+            BranchProbability falseprob = BranchProbability::getUnknown(),
+            bool isunpredictable = false)
       : PredInfo({pred, nocmp}), CmpLHS(cmplhs), CmpMHS(cmpmiddle),
         CmpRHS(cmprhs), TrueBB(truebb), FalseBB(falsebb), ThisBB(me),
-        DbgLoc(dl), TrueProb(trueprob), FalseProb(falseprob) {}
+        DbgLoc(dl), TrueProb(trueprob), FalseProb(falseprob),
+        IsUnpredictable(isunpredictable) {}
 };
 
 struct JumpTable {
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 8064cc7963b7ac4..80fdedcf9c62599 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -18299,7 +18299,7 @@ SDValue DAGCombiner::visitBRCOND(SDNode *N) {
   // nondeterministic jumps).
   if (N1->getOpcode() == ISD::FREEZE && N1.hasOneUse()) {
     return DAG.getNode(ISD::BRCOND, SDLoc(N), MVT::Other, Chain,
-                       N1->getOperand(0), N2);
+                       N1->getOperand(0), N2, N->getFlags());
   }
 
   // Variant of the previous fold where there is a SETCC in between:
@@ -18348,7 +18348,8 @@ SDValue DAGCombiner::visitBRCOND(SDNode *N) {
     if (Updated)
       return DAG.getNode(
           ISD::BRCOND, SDLoc(N), MVT::Other, Chain,
-          DAG.getSetCC(SDLoc(N1), N1->getValueType(0), S0, S1, Cond), N2);
+          DAG.getSetCC(SDLoc(N1), N1->getValueType(0), S0, S1, Cond), N2,
+          N->getFlags());
   }
 
   // If N is a constant we could fold this into a fallthrough or unconditional
@@ -18373,7 +18374,7 @@ SDValue DAGCombiner::visitBRCOND(SDNode *N) {
     HandleSDNode ChainHandle(Chain);
     if (SDValue NewN1 = rebuildSetCC(N1))
       return DAG.getNode(ISD::BRCOND, SDLoc(N), MVT::Other,
-                         ChainHandle.getValue(), NewN1, N2);
+                         ChainHandle.getValue(), NewN1, N2, N->getFlags());
   }
 
   return SDValue();
diff --git a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
index 4ce92e156cf85ea..db33d5242601e07 100644
--- a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
@@ -1065,14 +1065,17 @@ EmitMachineNode(SDNode *Node, bool IsClone, bool IsCloned,
   // Create the new machine instruction.
   MachineInstrBuilder MIB = BuildMI(*MF, Node->getDebugLoc(), II);
 
+  // Transfer IR flags from the SDNode to the MachineInstr
+  MachineInstr *MI = MIB.getInstr();
+  const SDNodeFlags Flags = Node->getFlags();
+  if (Flags.hasUnpredictable())
+    MI->setFlag(MachineInstr::MIFlag::Unpredictable);
+
   // Add result register values for things that are defined by this
   // instruction.
   if (NumResults) {
     CreateVirtualRegisters(Node, MIB, II, IsClone, IsCloned, VRBaseMap);
 
-    // Transfer any IR flags from the SDNode to the MachineInstr
-    MachineInstr *MI = MIB.getInstr();
-    const SDNodeFlags Flags = Node->getFlags();
     if (Flags.hasNoSignedZeros())
       MI->setFlag(MachineInstr::MIFlag::FmNsz);
 
@@ -1105,9 +1108,6 @@ EmitMachineNode(SDNode *Node, bool IsClone, bool IsCloned,
 
     if (Flags.hasNoFPExcept())
       MI->setFlag(MachineInstr::MIFlag::NoFPExcept);
-
-    if (Flags.hasUnpredictable())
-      MI->setFlag(MachineInstr::MIFlag::Unpredictable);
   }
 
   // Emit all of the actual operands of this instruction, adding them to the
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 2c6f7c09e5ca636..8d6f5334c9600be 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -7626,16 +7626,34 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
 }
 
 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
-                              SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
+                              SDValue N1, SDValue N2, SDValue N3, SDValue N4,
+                              const SDNodeFlags Flags) {
   SDValue Ops[] = { N1, N2, N3, N4 };
-  return getNode(Opcode, DL, VT, Ops);
+  return getNode(Opcode, DL, VT, Ops, Flags);
+}
+
+SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
+                              SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
+  SDNodeFlags Flags;
+  if (Inserter)
+    Flags = Inserter->getFlags();
+  return getNode(Opcode, DL, VT, N1, N2, N3, N4, Flags);
 }
 
 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
                               SDValue N1, SDValue N2, SDValue N3, SDValue N4,
-                              SDValue N5) {
+                              SDValue N5, const SDNodeFlags Flags) {
   SDValue Ops[] = { N1, N2, N3, N4, N5 };
-  return getNode(Opcode, DL, VT, Ops);
+  return getNode(Opcode, DL, VT, Ops, Flags);
+}
+
+SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
+                              SDValue N1, SDValue N2, SDValue N3, SDValue N4,
+                              SDValue N5) {
+  SDNodeFlags Flags;
+  if (Inserter)
+    Flags = Inserter->getFlags();
+  return getNode(Opcode, DL, VT, N1, N2, N3, N4, N5, Flags);
 }
 
 /// getStackArgumentTokenFactor - Compute a TokenFactor to force all
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index b0f26b54fe42536..60dcb1185427856 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -2818,9 +2818,10 @@ void SelectionDAGBuilder::visitBr(const BranchInst &I) {
   //     je foo
   //     cmp D, E
   //     jle foo
+  bool IsUnpredictable = I.hasMetadata(LLVMContext::MD_unpredictable);
   const Instruction *BOp = dyn_cast<Instruction>(CondVal);
   if (!DAG.getTargetLoweringInfo().isJumpExpensive() && BOp &&
-      BOp->hasOneUse() && !I.hasMetadata(LLVMContext::MD_unpredictable)) {
+      BOp->hasOneUse() && !IsUnpredictable) {
     Value *Vec;
     const Value *BOp0, *BOp1;
     Instruction::BinaryOps Opcode = (Instruction::BinaryOps)0;
@@ -2869,7 +2870,9 @@ void SelectionDAGBuilder::visitBr(const BranchInst &I) {
 
   // Create a CaseBlock record representing this branch.
   CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()),
-               nullptr, Succ0MBB, Succ1MBB, BrMBB, getCurSDLoc());
+               nullptr, Succ0MBB, Succ1MBB, BrMBB, getCurSDLoc(),
+               BranchProbability::getUnknown(), BranchProbability::getUnknown(),
+               IsUnpredictable);
 
   // Use visitSwitchCase to actually insert the fast branch sequence for this
   // cond branch.
@@ -2957,9 +2960,10 @@ void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB,
     Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);
   }
 
-  SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
-                               MVT::Other, getControlRoot(), Cond,
-                               DAG.getBasicBlock(CB.TrueBB));
+  SDNodeFlags Flags;
+  Flags.setUnpredictable(CB.IsUnpredictable);
+  SDValue BrCond = DAG.getNode(ISD::BRCOND, dl, MVT::Other, getControlRoot(),
+                               Cond, DAG.getBasicBlock(CB.TrueBB), Flags);
 
   setValue(CurInst, BrCond);
 
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 2fa416f2e331c43..ca4e0a31e932bf0 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -24833,14 +24833,14 @@ SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
 
       SDValue CCVal = DAG.getTargetConstant(X86Cond, dl, MVT::i8);
       return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
-                         Overflow);
+                         Overflow, Op->getFlags());
     }
 
     if (LHS.getSimpleValueType().isInteger()) {
       SDValue CCVal;
       SDValue EFLAGS = emitFlagsForSetcc(LHS, RHS, CC, SDLoc(Cond), DAG, CCVal);
       return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
-                         EFLAGS);
+                         EFLAGS, Op->getFlags());
     }
 
     if (CC == ISD::SETOEQ) {
@@ -24866,10 +24866,10 @@ SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
               DAG.getNode(X86ISD::FCMP, SDLoc(Cond), MVT::i32, LHS, RHS);
           SDValue CCVal = DAG.getTargetConstant(X86::COND_NE, dl, MVT::i8);
           Chain = DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest,
-                              CCVal, Cmp);
+                              CCVal, Cmp, Op->getFlags());
           CCVal = DAG.getTargetConstant(X86::COND_P, dl, MVT::i8);
           return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
-                             Cmp);
+                             Cmp, Op->getFlags());
         }
       }
     } else if (CC == ISD::SETUNE) {
@@ -24878,18 +24878,18 @@ SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
       // separate test.
       SDValue Cmp = DAG.getNode(X86ISD::FCMP, SDLoc(Cond), MVT::i32, LHS, RHS);
       SDValue CCVal = DAG.getTargetConstant(X86::COND_NE, dl, MVT::i8);
-      Chain =
-          DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal, Cmp);
+      Chain = DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
+                          Cmp, Op->getFlags());
       CCVal = DAG.getTargetConstant(X86::COND_P, dl, MVT::i8);
       return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
-                         Cmp);
+                         Cmp, Op->getFlags());
     } else {
       X86::CondCode X86Cond =
           TranslateX86CC(CC, dl, /*IsFP*/ true, LHS, RHS, DAG);
       SDValue Cmp = DAG.getNode(X86ISD::FCMP, SDLoc(Cond), MVT::i32, LHS, RHS);
       SDValue CCVal = DAG.getTargetConstant(X86Cond, dl, MVT::i8);
       return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
-                         Cmp);
+                         Cmp, Op->getFlags());
     }
   }
 
@@ -24900,7 +24900,7 @@ SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
 
     SDValue CCVal = DAG.getTargetConstant(X86Cond, dl, MVT::i8);
     return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
-                       Overflow);
+                       Overflow, Op->getFlags());
   }
 
   // Look past the truncate if the high bits are known zero.
@@ -24919,8 +24919,8 @@ SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
 
   SDValue CCVal;
   SDValue EFLAGS = emitFlagsForSetcc(LHS, RHS, ISD::SETNE, dl, DAG, CCVal);
-  return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
-                     EFLAGS);
+  return DAG.getNode(X86ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal, EFLAGS,
+                     Op->getFlags());
 }
 
 // Lower dynamic stack allocation to _alloca call for Cygwin/Mingw targets.
diff --git a/llvm/test/CodeGen/X86/unpredictable-brcond.ll b/llvm/test/CodeGen/X86/unpredictable-brcond.ll
index 6c894ea8277679c..12411f1c49f2dbb 100644
--- a/llvm/test/CodeGen/X86/unpredictable-brcond.ll
+++ b/llvm/test/CodeGen/X86/unpredictable-brcond.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
-; Currently, unpredictable metadata on conditional branches is lost during CodeGen.
+; Make sure MIR generated for conditional branch with unpredictable metadata has unpredictable flag.
 ; RUN: llc -mtriple=x86_64-unknown-linux-gnu -stop-after=finalize-isel < %s | FileCheck %s
 
 define void @cond_branch_1(i1 %cond) {
@@ -11,7 +11,7 @@ define void @cond_branch_1(i1 %cond) {
   ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gr32 = COPY $edi
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gr8 = COPY [[COPY]].sub_8bit
   ; CHECK-NEXT:   TEST8ri killed [[COPY1]], 1, implicit-def $eflags
-  ; CHECK-NEXT:   JCC_1 %bb.2, 4, implicit $eflags
+  ; CHECK-NEXT:   unpredictable JCC_1 %bb.2, 4, implicit $eflags
   ; CHECK-NEXT:   JMP_1 %bb.1
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.1.then:
@@ -51,7 +51,7 @@ define void @cond_branch_2(double %a, double %b, i32 %c, i32 %d) nounwind {
   ; CHECK-NEXT:   [[SETCCr1:%[0-9]+]]:gr8 = SETCCr 6, implicit $eflags
   ; CHECK-NEXT:   [[OR8rr:%[0-9]+]]:gr8 = OR8rr [[SETCCr]], killed [[SETCCr1]], implicit-def dead $eflags
   ; CHECK-NEXT:   TEST8rr [[OR8rr]], [[OR8rr]], implicit-def $eflags
-  ; CHECK-NEXT:   JCC_1 %bb.2, 5, implicit $eflags
+  ; CHECK-NEXT:   unpredictable JCC_1 %bb.2, 5, implicit $eflags
   ; CHECK-NEXT:   JMP_1 %bb.1
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.1.true:
@@ -89,8 +89,8 @@ define void @isint_branch(double %d) nounwind {
   ; CHECK-NEXT:   [[CVTDQ2PDrr:%[0-9]+]]:vr128 = CVTDQ2PDrr killed [[CVTTPD2DQrr]]
   ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:fr64 = COPY [[CVTDQ2PDrr]]
   ; CHECK-NEXT:   nofpexcept UCOMISDrr [[COPY]], killed [[COPY2]], implicit-def $eflags, implicit $mxcsr
-  ; CHECK-NEXT:   JCC_1 %bb.2, 5, implicit $eflags
-  ; CHECK-NEXT:   JCC_1 %bb.2, 10, implicit $eflags
+  ; CHECK-NEXT:   unpredictable JCC_1 %bb.2, 5, implicit $eflags
+  ; CHECK-NEXT:   unpredictable JCC_1 %bb.2, 10, implicit $eflags
   ; CHECK-NEXT:   JMP_1 %bb.1
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.1.true:

From 731ae694a3d8f4d39e855c9a82c97d4f170fd48a Mon Sep 17 00:00:00 2001
From: Noah Goldstein <goldstein.w.n@gmail.com>
Date: Sun, 18 Aug 2024 20:36:35 -0700
Subject: [PATCH 321/441] Revert "[CodeGenPrepare] Folding `urem` with loop
 invariant value"

This reverts commit c64ce8bf283120fd145a57d0e61f9697f719139d.

Seems to be causing stage2 failures on buildbots. Reverting while I
investigate.
---
 llvm/lib/CodeGen/CodeGenPrepare.cpp           | 131 -----------------
 llvm/test/CodeGen/X86/fold-loop-of-urem.ll    | 138 +++++++-----------
 .../CodeGenPrepare/X86/fold-loop-of-urem.ll   |  37 +----
 3 files changed, 61 insertions(+), 245 deletions(-)

diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp
index 72db1f4adabf76f..48253a613b41d2b 100644
--- a/llvm/lib/CodeGen/CodeGenPrepare.cpp
+++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp
@@ -472,7 +472,6 @@ class CodeGenPrepare {
   bool replaceMathCmpWithIntrinsic(BinaryOperator *BO, Value *Arg0, Value *Arg1,
                                    CmpInst *Cmp, Intrinsic::ID IID);
   bool optimizeCmp(CmpInst *Cmp, ModifyDT &ModifiedDT);
-  bool optimizeURem(Instruction *Rem);
   bool combineToUSubWithOverflow(CmpInst *Cmp, ModifyDT &ModifiedDT);
   bool combineToUAddWithOverflow(CmpInst *Cmp, ModifyDT &ModifiedDT);
   void verifyBFIUpdates(Function &F);
@@ -1976,132 +1975,6 @@ static bool foldFCmpToFPClassTest(CmpInst *Cmp, const TargetLowering &TLI,
   return true;
 }
 
-static bool isRemOfLoopIncrementWithLoopInvariant(Instruction *Rem,
-                                                  const LoopInfo *LI,
-                                                  Value *&RemAmtOut,
-                                                  PHINode *&LoopIncrPNOut) {
-  Value *Incr, *RemAmt;
-  // NB: If RemAmt is a power of 2 it *should* have been transformed by now.
-  if (!match(Rem, m_URem(m_Value(Incr), m_Value(RemAmt))))
-    return false;
-
-  // Find out loop increment PHI.
-  auto *PN = dyn_cast<PHINode>(Incr);
-  if (!PN)
-    return false;
-
-  // This isn't strictly necessary, what we really need is one increment and any
-  // amount of initial values all being the same.
-  if (PN->getNumIncomingValues() != 2)
-    return false;
-
-  // Only trivially analyzable loops.
-  Loop *L = LI->getLoopFor(Rem->getParent());
-  if (!L || !L->getLoopPreheader() || !L->getLoopLatch())
-    return false;
-
-  // Only works if the remainder amount is a loop invaraint
-  if (!L->isLoopInvariant(RemAmt))
-    return false;
-
-  // Is the PHI a loop increment?
-  auto LoopIncrInfo = getIVIncrement(PN, LI);
-  if (!LoopIncrInfo)
-    return false;
-
-  // getIVIncrement finds the loop at PN->getParent(). This might be a different
-  // loop from the loop with Rem->getParent().
-  if (L->getHeader() != PN->getParent())
-    return false;
-
-  // We need remainder_amount % increment_amount to be zero. Increment of one
-  // satisfies that without any special logic and is overwhelmingly the common
-  // case.
-  if (!match(LoopIncrInfo->second, m_One()))
-    return false;
-
-  // Need the increment to not overflow.
-  if (!match(LoopIncrInfo->first, m_NUWAdd(m_Value(), m_Value())))
-    return false;
-
-  // Set output variables.
-  RemAmtOut = RemAmt;
-  LoopIncrPNOut = PN;
-
-  return true;
-}
-
-// Try to transform:
-//
-// for(i = Start; i < End; ++i)
-//    Rem = (i nuw+ IncrLoopInvariant) u% RemAmtLoopInvariant;
-//
-// ->
-//
-// Rem = (Start nuw+ IncrLoopInvariant) % RemAmtLoopInvariant;
-// for(i = Start; i < End; ++i, ++rem)
-//    Rem = rem == RemAmtLoopInvariant ? 0 : Rem;
-//
-// Currently only implemented for `IncrLoopInvariant` being zero.
-static bool foldURemOfLoopIncrement(Instruction *Rem, const DataLayout *DL,
-                                    const LoopInfo *LI,
-                                    SmallSet<BasicBlock *, 32> &FreshBBs,
-                                    bool IsHuge) {
-  Value *RemAmt;
-  PHINode *LoopIncrPN;
-  if (!isRemOfLoopIncrementWithLoopInvariant(Rem, LI, RemAmt, LoopIncrPN))
-    return false;
-
-  // Only non-constant remainder as the extra IV is probably not profitable
-  // in that case.
-  //
-  // Potential TODO(1): `urem` of a const ends up as `mul` + `shift` + `add`. If
-  // we can rule out register pressure and ensure this `urem` is executed each
-  // iteration, its probably profitable to handle the const case as well.
-  //
-  // Potential TODO(2): Should we have a check for how "nested" this remainder
-  // operation is? The new code runs every iteration so if the remainder is
-  // guarded behind unlikely conditions this might not be worth it.
-  if (match(RemAmt, m_ImmConstant()))
-    return false;
-  Loop *L = LI->getLoopFor(Rem->getParent());
-
-  Value *Start = LoopIncrPN->getIncomingValueForBlock(L->getLoopPreheader());
-
-  // Create new remainder with induction variable.
-  Type *Ty = Rem->getType();
-  IRBuilder<> Builder(Rem->getContext());
-
-  Builder.SetInsertPoint(LoopIncrPN);
-  PHINode *NewRem = Builder.CreatePHI(Ty, 2);
-
-  Builder.SetInsertPoint(cast<Instruction>(
-      LoopIncrPN->getIncomingValueForBlock(L->getLoopLatch())));
-  // `(add (urem x, y), 1)` is always nuw.
-  Value *RemAdd = Builder.CreateNUWAdd(NewRem, ConstantInt::get(Ty, 1));
-  Value *RemCmp = Builder.CreateICmp(ICmpInst::ICMP_EQ, RemAdd, RemAmt);
-  Value *RemSel =
-      Builder.CreateSelect(RemCmp, Constant::getNullValue(Ty), RemAdd);
-
-  NewRem->addIncoming(Start, L->getLoopPreheader());
-  NewRem->addIncoming(RemSel, L->getLoopLatch());
-
-  // Insert all touched BBs.
-  FreshBBs.insert(LoopIncrPN->getParent());
-  FreshBBs.insert(L->getLoopLatch());
-  FreshBBs.insert(Rem->getParent());
-
-  replaceAllUsesWith(Rem, NewRem, FreshBBs, IsHuge);
-  Rem->eraseFromParent();
-  return true;
-}
-
-bool CodeGenPrepare::optimizeURem(Instruction *Rem) {
-  if (foldURemOfLoopIncrement(Rem, DL, LI, FreshBBs, IsHugeFunc))
-    return true;
-  return false;
-}
-
 bool CodeGenPrepare::optimizeCmp(CmpInst *Cmp, ModifyDT &ModifiedDT) {
   if (sinkCmpExpression(Cmp, *TLI))
     return true;
@@ -8485,10 +8358,6 @@ bool CodeGenPrepare::optimizeInst(Instruction *I, ModifyDT &ModifiedDT) {
     if (optimizeCmp(Cmp, ModifiedDT))
       return true;
 
-  if (match(I, m_URem(m_Value(), m_Value())))
-    if (optimizeURem(I))
-      return true;
-
   if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
     LI->setMetadata(LLVMContext::MD_invariant_group, nullptr);
     bool Modified = optimizeLoadExt(LI);
diff --git a/llvm/test/CodeGen/X86/fold-loop-of-urem.ll b/llvm/test/CodeGen/X86/fold-loop-of-urem.ll
index b9fe4d7c79c7fc2..aad2e0dd7bd248a 100644
--- a/llvm/test/CodeGen/X86/fold-loop-of-urem.ll
+++ b/llvm/test/CodeGen/X86/fold-loop-of-urem.ll
@@ -15,31 +15,25 @@ define void @simple_urem_to_sel(i32 %N, i32 %rem_amt) nounwind {
 ; CHECK-NEXT:    je .LBB0_4
 ; CHECK-NEXT:  # %bb.1: # %for.body.preheader
 ; CHECK-NEXT:    pushq %rbp
-; CHECK-NEXT:    pushq %r15
 ; CHECK-NEXT:    pushq %r14
-; CHECK-NEXT:    pushq %r12
 ; CHECK-NEXT:    pushq %rbx
 ; CHECK-NEXT:    movl %esi, %ebx
 ; CHECK-NEXT:    movl %edi, %ebp
-; CHECK-NEXT:    xorl %r15d, %r15d
 ; CHECK-NEXT:    xorl %r14d, %r14d
-; CHECK-NEXT:    xorl %r12d, %r12d
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  .LBB0_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    movl %r14d, %edi
+; CHECK-NEXT:    movl %r14d, %eax
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divl %ebx
+; CHECK-NEXT:    movl %edx, %edi
 ; CHECK-NEXT:    callq use.i32@PLT
 ; CHECK-NEXT:    incl %r14d
-; CHECK-NEXT:    cmpl %ebx, %r14d
-; CHECK-NEXT:    cmovel %r15d, %r14d
-; CHECK-NEXT:    incl %r12d
-; CHECK-NEXT:    cmpl %r12d, %ebp
+; CHECK-NEXT:    cmpl %r14d, %ebp
 ; CHECK-NEXT:    jne .LBB0_2
 ; CHECK-NEXT:  # %bb.3:
 ; CHECK-NEXT:    popq %rbx
-; CHECK-NEXT:    popq %r12
 ; CHECK-NEXT:    popq %r14
-; CHECK-NEXT:    popq %r15
 ; CHECK-NEXT:    popq %rbp
 ; CHECK-NEXT:  .LBB0_4: # %for.cond.cleanup
 ; CHECK-NEXT:    retq
@@ -66,28 +60,24 @@ define void @simple_urem_to_sel_nested2(i32 %N, i32 %rem_amt) nounwind {
 ; CHECK-NEXT:    je .LBB1_8
 ; CHECK-NEXT:  # %bb.1: # %for.body.preheader
 ; CHECK-NEXT:    pushq %rbp
-; CHECK-NEXT:    pushq %r15
 ; CHECK-NEXT:    pushq %r14
-; CHECK-NEXT:    pushq %r12
 ; CHECK-NEXT:    pushq %rbx
 ; CHECK-NEXT:    movl %esi, %ebx
 ; CHECK-NEXT:    movl %edi, %ebp
-; CHECK-NEXT:    xorl %r15d, %r15d
 ; CHECK-NEXT:    xorl %r14d, %r14d
-; CHECK-NEXT:    xorl %r12d, %r12d
 ; CHECK-NEXT:    jmp .LBB1_2
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  .LBB1_5: # %for.body1
 ; CHECK-NEXT:    # in Loop: Header=BB1_2 Depth=1
-; CHECK-NEXT:    movl %r14d, %edi
+; CHECK-NEXT:    movl %r14d, %eax
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divl %ebx
+; CHECK-NEXT:    movl %edx, %edi
 ; CHECK-NEXT:    callq use.i32@PLT
 ; CHECK-NEXT:  .LBB1_6: # %for.body.tail
 ; CHECK-NEXT:    # in Loop: Header=BB1_2 Depth=1
 ; CHECK-NEXT:    incl %r14d
-; CHECK-NEXT:    cmpl %ebx, %r14d
-; CHECK-NEXT:    cmovel %r15d, %r14d
-; CHECK-NEXT:    incl %r12d
-; CHECK-NEXT:    cmpl %r12d, %ebp
+; CHECK-NEXT:    cmpl %r14d, %ebp
 ; CHECK-NEXT:    je .LBB1_7
 ; CHECK-NEXT:  .LBB1_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
@@ -107,9 +97,7 @@ define void @simple_urem_to_sel_nested2(i32 %N, i32 %rem_amt) nounwind {
 ; CHECK-NEXT:    jmp .LBB1_6
 ; CHECK-NEXT:  .LBB1_7:
 ; CHECK-NEXT:    popq %rbx
-; CHECK-NEXT:    popq %r12
 ; CHECK-NEXT:    popq %r14
-; CHECK-NEXT:    popq %r15
 ; CHECK-NEXT:    popq %rbp
 ; CHECK-NEXT:  .LBB1_8: # %for.cond.cleanup
 ; CHECK-NEXT:    retq
@@ -225,36 +213,40 @@ for.body.tail:
 define void @simple_urem_to_sel_vec(<2 x i64> %rem_amt) nounwind {
 ; CHECK-LABEL: simple_urem_to_sel_vec:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    subq $56, %rsp
-; CHECK-NEXT:    movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    subq $24, %rsp
 ; CHECK-NEXT:    pxor %xmm1, %xmm1
-; CHECK-NEXT:    pxor %xmm0, %xmm0
-; CHECK-NEXT:    movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movdqa %xmm1, (%rsp) # 16-byte Spill
+; CHECK-NEXT:    movq %xmm0, %rbx
+; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; CHECK-NEXT:    movq %xmm0, %r14
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  .LBB3_1: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movdqa (%rsp), %xmm1 # 16-byte Reload
 ; CHECK-NEXT:    movdqa %xmm1, (%rsp) # 16-byte Spill
-; CHECK-NEXT:    movdqa %xmm1, %xmm0
+; CHECK-NEXT:    movq %xmm1, %rax
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divq %rbx
+; CHECK-NEXT:    movq %rdx, %xmm0
+; CHECK-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; CHECK-NEXT:    movq %xmm1, %rax
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divq %r14
+; CHECK-NEXT:    movq %rdx, %xmm1
+; CHECK-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; CHECK-NEXT:    callq use.2xi64@PLT
-; CHECK-NEXT:    pcmpeqd %xmm1, %xmm1
-; CHECK-NEXT:    movdqa (%rsp), %xmm2 # 16-byte Reload
-; CHECK-NEXT:    psubq %xmm1, %xmm2
-; CHECK-NEXT:    movdqa %xmm2, %xmm0
-; CHECK-NEXT:    movdqa %xmm2, %xmm3
-; CHECK-NEXT:    pcmpeqd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
-; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,0,3,2]
-; CHECK-NEXT:    pand %xmm0, %xmm2
-; CHECK-NEXT:    pandn %xmm3, %xmm2
-; CHECK-NEXT:    movdqa %xmm2, (%rsp) # 16-byte Spill
-; CHECK-NEXT:    movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; CHECK-NEXT:    psubq %xmm1, %xmm0
-; CHECK-NEXT:    movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    movdqa (%rsp), %xmm0 # 16-byte Reload
+; CHECK-NEXT:    psubq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; CHECK-NEXT:    movdqa %xmm0, (%rsp) # 16-byte Spill
 ; CHECK-NEXT:    callq get.i1@PLT
 ; CHECK-NEXT:    testb $1, %al
-; CHECK-NEXT:    movdqa (%rsp), %xmm1 # 16-byte Reload
 ; CHECK-NEXT:    je .LBB3_1
 ; CHECK-NEXT:  # %bb.2: # %for.cond.cleanup
-; CHECK-NEXT:    addq $56, %rsp
+; CHECK-NEXT:    addq $24, %rsp
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r14
 ; CHECK-NEXT:    retq
 entry:
   br label %for.body
@@ -344,33 +336,27 @@ define void @simple_urem_to_sel_second_acc(i32 %N, i32 %rem_amt) nounwind {
 ; CHECK-NEXT:    pushq %rbp
 ; CHECK-NEXT:    pushq %r15
 ; CHECK-NEXT:    pushq %r14
-; CHECK-NEXT:    pushq %r13
-; CHECK-NEXT:    pushq %r12
 ; CHECK-NEXT:    pushq %rbx
 ; CHECK-NEXT:    pushq %rax
 ; CHECK-NEXT:    movl %esi, %ebx
 ; CHECK-NEXT:    movl %edi, %ebp
 ; CHECK-NEXT:    movl $1, %r15d
-; CHECK-NEXT:    xorl %r12d, %r12d
 ; CHECK-NEXT:    xorl %r14d, %r14d
-; CHECK-NEXT:    xorl %r13d, %r13d
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  .LBB5_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    movl %r14d, %edi
+; CHECK-NEXT:    movl %r14d, %eax
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divl %ebx
+; CHECK-NEXT:    movl %edx, %edi
 ; CHECK-NEXT:    callq use.i32@PLT
 ; CHECK-NEXT:    incl %r14d
-; CHECK-NEXT:    cmpl %ebx, %r14d
-; CHECK-NEXT:    cmovel %r12d, %r14d
-; CHECK-NEXT:    incl %r13d
 ; CHECK-NEXT:    addl $2, %r15d
 ; CHECK-NEXT:    cmpl %ebp, %r15d
 ; CHECK-NEXT:    jbe .LBB5_2
 ; CHECK-NEXT:  # %bb.3:
 ; CHECK-NEXT:    addq $8, %rsp
 ; CHECK-NEXT:    popq %rbx
-; CHECK-NEXT:    popq %r12
-; CHECK-NEXT:    popq %r13
 ; CHECK-NEXT:    popq %r14
 ; CHECK-NEXT:    popq %r15
 ; CHECK-NEXT:    popq %rbp
@@ -690,27 +676,23 @@ define void @simple_urem_multi_latch_non_canonical(i32 %N, i32 %rem_amt) nounwin
 ; CHECK-NEXT:    pushq %rbp
 ; CHECK-NEXT:    pushq %r15
 ; CHECK-NEXT:    pushq %r14
-; CHECK-NEXT:    pushq %r13
-; CHECK-NEXT:    pushq %r12
 ; CHECK-NEXT:    pushq %rbx
 ; CHECK-NEXT:    pushq %rax
 ; CHECK-NEXT:    movl %esi, %ebx
 ; CHECK-NEXT:    movl %edi, %ebp
 ; CHECK-NEXT:    decl %ebp
-; CHECK-NEXT:    xorl %r12d, %r12d
 ; CHECK-NEXT:    xorl %r14d, %r14d
-; CHECK-NEXT:    xorl %r13d, %r13d
 ; CHECK-NEXT:    jmp .LBB12_2
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  .LBB12_3: # %for.body.backedge
 ; CHECK-NEXT:    # in Loop: Header=BB12_2 Depth=1
 ; CHECK-NEXT:    incl %r14d
-; CHECK-NEXT:    cmpl %ebx, %r14d
-; CHECK-NEXT:    cmovel %r12d, %r14d
-; CHECK-NEXT:    incl %r13d
 ; CHECK-NEXT:  .LBB12_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    movl %r14d, %edi
+; CHECK-NEXT:    movl %r14d, %eax
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divl %ebx
+; CHECK-NEXT:    movl %edx, %edi
 ; CHECK-NEXT:    callq use.i32@PLT
 ; CHECK-NEXT:    callq get.i1@PLT
 ; CHECK-NEXT:    movl %eax, %r15d
@@ -720,13 +702,11 @@ define void @simple_urem_multi_latch_non_canonical(i32 %N, i32 %rem_amt) nounwin
 ; CHECK-NEXT:  # %bb.4: # %for.body0
 ; CHECK-NEXT:    # in Loop: Header=BB12_2 Depth=1
 ; CHECK-NEXT:    callq do_stuff1@PLT
-; CHECK-NEXT:    cmpl %r13d, %ebp
+; CHECK-NEXT:    cmpl %r14d, %ebp
 ; CHECK-NEXT:    jne .LBB12_3
 ; CHECK-NEXT:  # %bb.5:
 ; CHECK-NEXT:    addq $8, %rsp
 ; CHECK-NEXT:    popq %rbx
-; CHECK-NEXT:    popq %r12
-; CHECK-NEXT:    popq %r13
 ; CHECK-NEXT:    popq %r14
 ; CHECK-NEXT:    popq %r15
 ; CHECK-NEXT:    popq %rbp
@@ -890,31 +870,25 @@ define void @simple_urem_to_sel_non_zero_start(i32 %N, i32 %rem_amt) nounwind {
 ; CHECK-NEXT:    jb .LBB16_4
 ; CHECK-NEXT:  # %bb.1: # %for.body.preheader
 ; CHECK-NEXT:    pushq %rbp
-; CHECK-NEXT:    pushq %r15
 ; CHECK-NEXT:    pushq %r14
-; CHECK-NEXT:    pushq %r12
 ; CHECK-NEXT:    pushq %rbx
 ; CHECK-NEXT:    movl %esi, %ebx
 ; CHECK-NEXT:    movl %edi, %ebp
 ; CHECK-NEXT:    movl $2, %r14d
-; CHECK-NEXT:    xorl %r15d, %r15d
-; CHECK-NEXT:    movl $2, %r12d
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  .LBB16_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    movl %r14d, %edi
+; CHECK-NEXT:    movl %r14d, %eax
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divl %ebx
+; CHECK-NEXT:    movl %edx, %edi
 ; CHECK-NEXT:    callq use.i32@PLT
 ; CHECK-NEXT:    incl %r14d
-; CHECK-NEXT:    cmpl %ebx, %r14d
-; CHECK-NEXT:    cmovel %r15d, %r14d
-; CHECK-NEXT:    incl %r12d
-; CHECK-NEXT:    cmpl %r12d, %ebp
+; CHECK-NEXT:    cmpl %r14d, %ebp
 ; CHECK-NEXT:    jne .LBB16_2
 ; CHECK-NEXT:  # %bb.3:
 ; CHECK-NEXT:    popq %rbx
-; CHECK-NEXT:    popq %r12
 ; CHECK-NEXT:    popq %r14
-; CHECK-NEXT:    popq %r15
 ; CHECK-NEXT:    popq %rbp
 ; CHECK-NEXT:  .LBB16_4: # %for.cond.cleanup
 ; CHECK-NEXT:    retq
@@ -1092,34 +1066,28 @@ define void @simple_urem_to_sel_non_zero_start_through_sub(i32 %N, i32 %rem_amt,
 ; CHECK-LABEL: simple_urem_to_sel_non_zero_start_through_sub:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    pushq %rbp
-; CHECK-NEXT:    pushq %r15
 ; CHECK-NEXT:    pushq %r14
-; CHECK-NEXT:    pushq %r12
 ; CHECK-NEXT:    pushq %rbx
 ; CHECK-NEXT:    movl %edi, %ebp
 ; CHECK-NEXT:    subl %edx, %ebp
 ; CHECK-NEXT:    jbe .LBB20_3
 ; CHECK-NEXT:  # %bb.1: # %for.body.preheader
 ; CHECK-NEXT:    movl %esi, %ebx
-; CHECK-NEXT:    xorl %r15d, %r15d
 ; CHECK-NEXT:    xorl %r14d, %r14d
-; CHECK-NEXT:    xorl %r12d, %r12d
 ; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  .LBB20_2: # %for.body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    movl %r14d, %edi
+; CHECK-NEXT:    movl %r14d, %eax
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    divl %ebx
+; CHECK-NEXT:    movl %edx, %edi
 ; CHECK-NEXT:    callq use.i32@PLT
 ; CHECK-NEXT:    incl %r14d
-; CHECK-NEXT:    cmpl %ebx, %r14d
-; CHECK-NEXT:    cmovel %r15d, %r14d
-; CHECK-NEXT:    incl %r12d
-; CHECK-NEXT:    cmpl %r12d, %ebp
+; CHECK-NEXT:    cmpl %r14d, %ebp
 ; CHECK-NEXT:    jne .LBB20_2
 ; CHECK-NEXT:  .LBB20_3: # %for.cond.cleanup
 ; CHECK-NEXT:    popq %rbx
-; CHECK-NEXT:    popq %r12
 ; CHECK-NEXT:    popq %r14
-; CHECK-NEXT:    popq %r15
 ; CHECK-NEXT:    popq %rbp
 ; CHECK-NEXT:    retq
 entry:
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/fold-loop-of-urem.ll b/llvm/test/Transforms/CodeGenPrepare/X86/fold-loop-of-urem.ll
index 304ae337ed41973..a019679e65905de 100644
--- a/llvm/test/Transforms/CodeGenPrepare/X86/fold-loop-of-urem.ll
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/fold-loop-of-urem.ll
@@ -19,12 +19,9 @@ define void @simple_urem_to_sel(i32 %N, i32 %rem_amt) nounwind {
 ; CHECK:       [[FOR_COND_CLEANUP]]:
 ; CHECK-NEXT:    ret void
 ; CHECK:       [[FOR_BODY]]:
-; CHECK-NEXT:    [[REM:%.*]] = phi i32 [ 0, %[[FOR_BODY_PREHEADER]] ], [ [[TMP3:%.*]], %[[FOR_BODY]] ]
 ; CHECK-NEXT:    [[I_04:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[REM:%.*]] = urem i32 [[I_04]], [[REM_AMT]]
 ; CHECK-NEXT:    tail call void @use.i32(i32 [[REM]])
-; CHECK-NEXT:    [[TMP1:%.*]] = add nuw i32 [[REM]], 1
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], [[REM_AMT]]
-; CHECK-NEXT:    [[TMP3]] = select i1 [[TMP2]], i32 0, i32 [[TMP1]]
 ; CHECK-NEXT:    [[INC]] = add nuw i32 [[I_04]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
 ; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
@@ -56,8 +53,7 @@ define void @simple_urem_to_sel_nested2(i32 %N, i32 %rem_amt) nounwind {
 ; CHECK:       [[FOR_COND_CLEANUP]]:
 ; CHECK-NEXT:    ret void
 ; CHECK:       [[FOR_BODY]]:
-; CHECK-NEXT:    [[REM:%.*]] = phi i32 [ 0, %[[FOR_BODY_PREHEADER]] ], [ [[TMP3:%.*]], %[[FOR_BODY_TAIL:.*]] ]
-; CHECK-NEXT:    [[I_04:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY_TAIL]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[I_04:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY_TAIL:.*]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
 ; CHECK-NEXT:    [[COND0:%.*]] = call i1 @get.i1()
 ; CHECK-NEXT:    br i1 [[COND0]], label %[[FOR_BODY0:.*]], label %[[FOR_BODY_TAIL]]
 ; CHECK:       [[FOR_BODY0]]:
@@ -67,12 +63,10 @@ define void @simple_urem_to_sel_nested2(i32 %N, i32 %rem_amt) nounwind {
 ; CHECK-NEXT:    [[COND2:%.*]] = call i1 @get.i1()
 ; CHECK-NEXT:    br i1 [[COND2]], label %[[FOR_BODY1]], label %[[FOR_BODY_TAIL]]
 ; CHECK:       [[FOR_BODY1]]:
+; CHECK-NEXT:    [[REM:%.*]] = urem i32 [[I_04]], [[REM_AMT]]
 ; CHECK-NEXT:    tail call void @use.i32(i32 [[REM]])
 ; CHECK-NEXT:    br label %[[FOR_BODY_TAIL]]
 ; CHECK:       [[FOR_BODY_TAIL]]:
-; CHECK-NEXT:    [[TMP1:%.*]] = add nuw i32 [[REM]], 1
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], [[REM_AMT]]
-; CHECK-NEXT:    [[TMP3]] = select i1 [[TMP2]], i32 0, i32 [[TMP1]]
 ; CHECK-NEXT:    [[INC]] = add nuw i32 [[I_04]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
 ; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
@@ -170,12 +164,9 @@ define void @simple_urem_to_sel_vec(<2 x i64> %rem_amt) nounwind {
 ; CHECK:       [[FOR_COND_CLEANUP:.*]]:
 ; CHECK-NEXT:    ret void
 ; CHECK:       [[FOR_BODY]]:
-; CHECK-NEXT:    [[REM:%.*]] = phi <2 x i64> [ zeroinitializer, %[[ENTRY]] ], [ [[TMP3:%.*]], %[[FOR_BODY]] ]
 ; CHECK-NEXT:    [[I_04:%.*]] = phi <2 x i64> [ [[INC:%.*]], %[[FOR_BODY]] ], [ zeroinitializer, %[[ENTRY]] ]
+; CHECK-NEXT:    [[REM:%.*]] = urem <2 x i64> [[I_04]], [[REM_AMT]]
 ; CHECK-NEXT:    tail call void @use.2xi64(<2 x i64> [[REM]])
-; CHECK-NEXT:    [[TMP1:%.*]] = add nuw <2 x i64> [[REM]], <i64 1, i64 1>
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq <2 x i64> [[TMP1]], [[REM_AMT]]
-; CHECK-NEXT:    [[TMP3]] = select <2 x i1> [[TMP2]], <2 x i64> zeroinitializer, <2 x i64> [[TMP1]]
 ; CHECK-NEXT:    [[INC]] = add nuw <2 x i64> [[I_04]], <i64 1, i64 1>
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = call i1 @get.i1()
 ; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
@@ -255,13 +246,10 @@ define void @simple_urem_to_sel_second_acc(i32 %N, i32 %rem_amt) nounwind {
 ; CHECK:       [[FOR_COND_CLEANUP]]:
 ; CHECK-NEXT:    ret void
 ; CHECK:       [[FOR_BODY]]:
-; CHECK-NEXT:    [[REM:%.*]] = phi i32 [ 0, %[[FOR_BODY_PREHEADER]] ], [ [[TMP3:%.*]], %[[FOR_BODY]] ]
 ; CHECK-NEXT:    [[I_04:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
 ; CHECK-NEXT:    [[I_05:%.*]] = phi i32 [ [[INC2:%.*]], %[[FOR_BODY]] ], [ 1, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[REM:%.*]] = urem i32 [[I_04]], [[REM_AMT]]
 ; CHECK-NEXT:    tail call void @use.i32(i32 [[REM]])
-; CHECK-NEXT:    [[TMP1:%.*]] = add nuw i32 [[REM]], 1
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], [[REM_AMT]]
-; CHECK-NEXT:    [[TMP3]] = select i1 [[TMP2]], i32 0, i32 [[TMP1]]
 ; CHECK-NEXT:    [[INC]] = add nuw i32 [[I_04]], 1
 ; CHECK-NEXT:    [[INC2]] = add nuw i32 [[I_05]], 2
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp ugt i32 [[INC2]], [[N]]
@@ -398,12 +386,9 @@ define void @simple_urem_non_zero_entry4(i32 %N, i32 %rem_amt) nounwind {
 ; CHECK:       [[FOR_COND_CLEANUP]]:
 ; CHECK-NEXT:    ret void
 ; CHECK:       [[FOR_BODY]]:
-; CHECK-NEXT:    [[REM:%.*]] = phi i32 [ 4, %[[FOR_BODY_PREHEADER]] ], [ [[TMP3:%.*]], %[[FOR_BODY]] ]
 ; CHECK-NEXT:    [[I_04:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 4, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[REM:%.*]] = urem i32 [[I_04]], [[REM_AMT]]
 ; CHECK-NEXT:    tail call void @use.i32(i32 [[REM]])
-; CHECK-NEXT:    [[TMP1:%.*]] = add nuw i32 [[REM]], 1
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], [[REM_AMT]]
-; CHECK-NEXT:    [[TMP3]] = select i1 [[TMP2]], i32 0, i32 [[TMP1]]
 ; CHECK-NEXT:    [[INC]] = add nuw i32 [[I_04]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
 ; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
@@ -472,12 +457,9 @@ define void @simple_urem_fail_no_preheader_non_canonical(i32 %N, i32 %rem_amt) n
 ; CHECK-NEXT:    [[I_04_PH:%.*]] = phi i32 [ 1, %[[FOR_BODY1]] ], [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
 ; CHECK:       [[FOR_BODY]]:
-; CHECK-NEXT:    [[REM:%.*]] = phi i32 [ [[I_04_PH]], %[[FOR_BODY_PREHEADER]] ], [ [[TMP3:%.*]], %[[FOR_BODY]] ]
 ; CHECK-NEXT:    [[I_04:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ [[I_04_PH]], %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[REM:%.*]] = urem i32 [[I_04]], [[REM_AMT]]
 ; CHECK-NEXT:    tail call void @use.i32(i32 [[REM]])
-; CHECK-NEXT:    [[TMP1:%.*]] = add nuw i32 [[REM]], 1
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], [[REM_AMT]]
-; CHECK-NEXT:    [[TMP3]] = select i1 [[TMP2]], i32 0, i32 [[TMP1]]
 ; CHECK-NEXT:    [[INC]] = add nuw i32 [[I_04]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
 ; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]
@@ -668,12 +650,9 @@ define void @simple_urem_to_sel_non_zero_start(i32 %N, i32 %rem_amt) nounwind {
 ; CHECK:       [[FOR_COND_CLEANUP]]:
 ; CHECK-NEXT:    ret void
 ; CHECK:       [[FOR_BODY]]:
-; CHECK-NEXT:    [[REM:%.*]] = phi i32 [ 2, %[[FOR_BODY_PREHEADER]] ], [ [[TMP3:%.*]], %[[FOR_BODY]] ]
 ; CHECK-NEXT:    [[I_04:%.*]] = phi i32 [ [[INC:%.*]], %[[FOR_BODY]] ], [ 2, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    [[REM:%.*]] = urem i32 [[I_04]], [[REM_AMT]]
 ; CHECK-NEXT:    tail call void @use.i32(i32 [[REM]])
-; CHECK-NEXT:    [[TMP1:%.*]] = add nuw i32 [[REM]], 1
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], [[REM_AMT]]
-; CHECK-NEXT:    [[TMP3]] = select i1 [[TMP2]], i32 0, i32 [[TMP1]]
 ; CHECK-NEXT:    [[INC]] = add nuw i32 [[I_04]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[N]]
 ; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]]

From a80a90e34b1f26422ebf56e922abe2c193607c81 Mon Sep 17 00:00:00 2001
From: Pengcheng Wang <wangpengcheng.pp@bytedance.com>
Date: Mon, 19 Aug 2024 11:50:32 +0800
Subject: [PATCH 322/441] [RISCV][MC] Support experimental extensions Zvbc32e
 and Zvkgs (#103709)

These two extensions add addtional instructions for carryless
multiplication with 32-bits elements and Vector-Scalar GCM
instructions.

Please see https://github.com/riscv/riscv-isa-manual/pull/1306.
---
 .../Driver/print-supported-extensions-riscv.c |  2 ++
 .../test/Preprocessor/riscv-target-features.c | 18 ++++++++++++++++
 llvm/docs/RISCVUsage.rst                      |  3 +++
 llvm/docs/ReleaseNotes.rst                    |  1 +
 llvm/lib/Target/RISCV/RISCVFeatures.td        | 16 ++++++++++++++
 llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td    | 12 +++++++++--
 llvm/lib/TargetParser/RISCVISAInfo.cpp        |  9 ++++++--
 llvm/test/CodeGen/RISCV/attributes.ll         |  8 +++++++
 llvm/test/MC/RISCV/rvv/zvbc.s                 | 17 +++++++++++----
 llvm/test/MC/RISCV/rvv/zvkgs.s                | 21 +++++++++++++++++++
 .../TargetParser/RISCVISAInfoTest.cpp         |  2 ++
 11 files changed, 101 insertions(+), 8 deletions(-)
 create mode 100644 llvm/test/MC/RISCV/rvv/zvkgs.s

diff --git a/clang/test/Driver/print-supported-extensions-riscv.c b/clang/test/Driver/print-supported-extensions-riscv.c
index 91f12b8416b2a42..132422393170a90 100644
--- a/clang/test/Driver/print-supported-extensions-riscv.c
+++ b/clang/test/Driver/print-supported-extensions-riscv.c
@@ -172,6 +172,8 @@
 // CHECK-NEXT:     zicfiss              1.0       'Zicfiss' (Shadow stack)
 // CHECK-NEXT:     zacas                1.0       'Zacas' (Atomic Compare-And-Swap Instructions)
 // CHECK-NEXT:     zalasr               0.1       'Zalasr' (Load-Acquire and Store-Release Instructions)
+// CHECK-NEXT:     zvbc32e              0.7       'Zvbc32e' (Vector Carryless Multiplication with 32-bits elements)
+// CHECK-NEXT:     zvkgs                0.7       'Zvkgs' (Vector-Scalar GCM instructions for Cryptography)
 // CHECK-NEXT:     smmpm                1.0       'Smmpm' (Machine-level Pointer Masking for M-mode)
 // CHECK-NEXT:     smnpm                1.0       'Smnpm' (Machine-level Pointer Masking for next lower privilege mode)
 // CHECK-NEXT:     ssnpm                1.0       'Ssnpm' (Supervisor-level Pointer Masking for next lower privilege mode)
diff --git a/clang/test/Preprocessor/riscv-target-features.c b/clang/test/Preprocessor/riscv-target-features.c
index 72131108cb5f6ac..517702fab5b919f 100644
--- a/clang/test/Preprocessor/riscv-target-features.c
+++ b/clang/test/Preprocessor/riscv-target-features.c
@@ -187,8 +187,10 @@
 // CHECK-NOT: __riscv_zicfilp {{.*$}}
 // CHECK-NOT: __riscv_zicfiss {{.*$}}
 // CHECK-NOT: __riscv_ztso {{.*$}}
+// CHECK-NOT: __riscv_zvbc32e {{.*$}}
 // CHECK-NOT: __riscv_zvfbfmin {{.*$}}
 // CHECK-NOT: __riscv_zvfbfwma {{.*$}}
+// CHECK-NOT: __riscv_zvkgs {{.*$}}
 
 // RUN: %clang --target=riscv32-unknown-linux-gnu \
 // RUN:   -march=rv32ia -E -dM %s \
@@ -1658,6 +1660,14 @@
 // RUN:   -o - | FileCheck --check-prefix=CHECK-ZTSO-EXT %s
 // CHECK-ZTSO-EXT: __riscv_ztso 1000000{{$}}
 
+// RUN: %clang --target=riscv32 -menable-experimental-extensions \
+// RUN:   -march=rv32i_zve32x_zvbc32e0p7 -E -dM %s \
+// RUN:   -o - | FileCheck --check-prefix=CHECK-ZVBC32E-EXT %s
+// RUN: %clang --target=riscv64 -menable-experimental-extensions \
+// RUN:   -march=rv64i_zve32x_zvbc32e0p7 -E -dM %s \
+// RUN:   -o - | FileCheck --check-prefix=CHECK-ZVBC32E-EXT %s
+// CHECK-ZVBC32E-EXT: __riscv_zvbc32e 7000{{$}}
+
 // RUN: %clang --target=riscv32 -menable-experimental-extensions \
 // RUN:   -march=rv32ifzvfbfmin1p0 -E -dM %s \
 // RUN:   -o - | FileCheck --check-prefix=CHECK-ZVFBFMIN-EXT %s
@@ -1674,6 +1684,14 @@
 // RUN:   -o - | FileCheck --check-prefix=CHECK-ZVFBFWMA-EXT %s
 // CHECK-ZVFBFWMA-EXT: __riscv_zvfbfwma 1000000{{$}}
 
+// RUN: %clang --target=riscv32 -menable-experimental-extensions \
+// RUN:   -march=rv32i_zve32x_zvkgs0p7 -E -dM %s \
+// RUN:   -o - | FileCheck --check-prefix=CHECK-ZVKGS-EXT %s
+// RUN: %clang --target=riscv64 -menable-experimental-extensions \
+// RUN:   -march=rv64i_zve32x_zvkgs0p7 -E -dM %s \
+// RUN:   -o - | FileCheck --check-prefix=CHECK-ZVKGS-EXT %s
+// CHECK-ZVKGS-EXT: __riscv_zvkgs 7000{{$}}
+
 // RUN: %clang -target riscv32 -menable-experimental-extensions \
 // RUN:   -march=rv32izicfiss1p0 -E -dM %s \
 // RUN:   -o - | FileCheck --check-prefix=CHECK-ZICFISS-EXT %s
diff --git a/llvm/docs/RISCVUsage.rst b/llvm/docs/RISCVUsage.rst
index b3c7b0e3883d01f..3af1428138c0216 100644
--- a/llvm/docs/RISCVUsage.rst
+++ b/llvm/docs/RISCVUsage.rst
@@ -302,6 +302,9 @@ The primary goal of experimental support is to assist in the process of ratifica
 ``experimental-zicfilp``, ``experimental-zicfiss``
   LLVM implements the `1.0 release specification <https://github.com/riscv/riscv-cfi/releases/tag/v1.0>`__.
 
+``experimental-zvbc32e``, ``experimental-zvkgs``
+  LLVM implements the `0.7 release specification <https://github.com/user-attachments/files/16450464/riscv-crypto-spec-vector-extra_v0.0.7.pdf>`__.
+
 To use an experimental extension from `clang`, you must add `-menable-experimental-extensions` to the command line, and specify the exact version of the experimental extension you are using.  To use an experimental extension with LLVM's internal developer tools (e.g. `llc`, `llvm-objdump`, `llvm-mc`), you must prefix the extension name with `experimental-`.  Note that you don't need to specify the version with internal tools, and shouldn't include the `experimental-` prefix with `clang`.
 
 Vendor Extensions
diff --git a/llvm/docs/ReleaseNotes.rst b/llvm/docs/ReleaseNotes.rst
index 42c0ad976089f7b..057bd5345fba289 100644
--- a/llvm/docs/ReleaseNotes.rst
+++ b/llvm/docs/ReleaseNotes.rst
@@ -111,6 +111,7 @@ Changes to the RISC-V Backend
 * Fixed length vector support using RVV instructions now requires VLEN>=64. This
   means Zve32x and Zve32f will also require Zvl64b. The prior support was
   largely untested.
+* The ``Zvbc32e`` and ``Zvkgs`` extensions are now supported experimentally.
 
 Changes to the WebAssembly Backend
 ----------------------------------
diff --git a/llvm/lib/Target/RISCV/RISCVFeatures.td b/llvm/lib/Target/RISCV/RISCVFeatures.td
index f2610ea7c9768ee..a439e75a3d5af43 100644
--- a/llvm/lib/Target/RISCV/RISCVFeatures.td
+++ b/llvm/lib/Target/RISCV/RISCVFeatures.td
@@ -754,6 +754,14 @@ def HasStdExtZvbc : Predicate<"Subtarget->hasStdExtZvbc()">,
                     AssemblerPredicate<(all_of FeatureStdExtZvbc),
                         "'Zvbc' (Vector Carryless Multiplication)">;
 
+def FeatureStdExtZvbc32e
+    : RISCVExperimentalExtension<"zvbc32e", 0, 7,
+                                 "'Zvbc32e' (Vector Carryless Multiplication with 32-bits elements)">;
+
+def HasStdExtZvbcOrZvbc32e : Predicate<"Subtarget->hasStdExtZvbc() || Subtarget->hasStdExtZvbc32e()">,
+                             AssemblerPredicate<(any_of FeatureStdExtZvbc, FeatureStdExtZvbc32e),
+                             "'Zvbc' or 'Zvbc32e' (Vector Carryless Multiplication)">;
+
 def FeatureStdExtZvkg
     : RISCVExtension<"zvkg", 1, 0,
                      "'Zvkg' (Vector GCM instructions for Cryptography)">,
@@ -762,6 +770,14 @@ def HasStdExtZvkg : Predicate<"Subtarget->hasStdExtZvkg()">,
                     AssemblerPredicate<(all_of FeatureStdExtZvkg),
                         "'Zvkg' (Vector GCM instructions for Cryptography)">;
 
+def FeatureStdExtZvkgs
+    : RISCVExperimentalExtension<"zvkgs", 0, 7,
+                                 "'Zvkgs' (Vector-Scalar GCM instructions for Cryptography)",
+                                 [FeatureStdExtZvkg]>;
+def HasStdExtZvkgs : Predicate<"Subtarget->hasStdExtZvkgs()">,
+                     AssemblerPredicate<(all_of FeatureStdExtZvkgs),
+                        "'Zvkgs' (Vector-Scalar GCM instructions for Cryptography)">;
+
 def FeatureStdExtZvkned
     : RISCVExtension<"zvkned", 1, 0,
                      "'Zvkned' (Vector AES Encryption & Decryption (Single Round))">,
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td
index bf9caa9b30b674e..cafd259031746d5 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td
@@ -127,10 +127,10 @@ let Predicates = [HasStdExtZvbb] in {
   defm VWSLL_V  : VSHT_IV_V_X_I<"vwsll", 0b110101>;
 } // Predicates = [HasStdExtZvbb]
 
-let Predicates = [HasStdExtZvbc] in {
+let Predicates = [HasStdExtZvbcOrZvbc32e] in {
   defm VCLMUL_V  : VCLMUL_MV_V_X<"vclmul", 0b001100>;
   defm VCLMULH_V : VCLMUL_MV_V_X<"vclmulh", 0b001101>;
-} // Predicates = [HasStdExtZvbc]
+} // Predicates = [HasStdExtZvbcOrZvbc32e]
 
 let Predicates = [HasStdExtZvkb] in {
   defm VANDN_V  : VALU_IV_V_X<"vandn", 0b000001>;
@@ -150,6 +150,14 @@ let Predicates = [HasStdExtZvkg], RVVConstraint = NoConstraint in {
                  SchedBinaryMC<"WriteVGMULV", "ReadVGMULV", "ReadVGMULV">;
 } // Predicates = [HasStdExtZvkg]
 
+let Predicates = [HasStdExtZvkgs], RVVConstraint = VS2Constraint in {
+  def VGHSH_VS : PALUVVNoVmTernary<0b100011, OPMVV, "vghsh.vs">,
+                 SchedTernaryMC<"WriteVGHSHV", "ReadVGHSHV", "ReadVGHSHV",
+                                "ReadVGHSHV">;
+  def VGMUL_VS : PALUVs2NoVmBinary<0b101001, 0b10001, OPMVV, "vgmul.vs">,
+                 SchedBinaryMC<"WriteVGMULV", "ReadVGMULV", "ReadVGMULV">;
+} // Predicates = [HasStdExtZvkgs]
+
 let Predicates = [HasStdExtZvknhaOrZvknhb], RVVConstraint = Sha2Constraint in {
   def VSHA2CH_VV : PALUVVNoVmTernary<0b101110, OPMVV, "vsha2ch.vv">,
                    SchedTernaryMC<"WriteVSHA2CHV", "ReadVSHA2CHV", "ReadVSHA2CHV",
diff --git a/llvm/lib/TargetParser/RISCVISAInfo.cpp b/llvm/lib/TargetParser/RISCVISAInfo.cpp
index 59df02daf632652..156ee0debab87be 100644
--- a/llvm/lib/TargetParser/RISCVISAInfo.cpp
+++ b/llvm/lib/TargetParser/RISCVISAInfo.cpp
@@ -750,8 +750,13 @@ Error RISCVISAInfo::checkDependency() {
     return getError(
         "'zvbc' requires 'v' or 'zve64*' extension to also be specified");
 
-  if ((Exts.count("zvkb") || Exts.count("zvkg") || Exts.count("zvkned") ||
-       Exts.count("zvknha") || Exts.count("zvksed") || Exts.count("zvksh")) &&
+  if (Exts.count("zvbc32e") && !Exts.count("zve32x"))
+    return getError(
+        "'zvbc32e' requires 'v' or 'zve*' extension to also be specified");
+
+  if ((Exts.count("zvkb") || Exts.count("zvkg") || Exts.count("zvkgs") ||
+       Exts.count("zvkned") || Exts.count("zvknha") || Exts.count("zvksed") ||
+       Exts.count("zvksh")) &&
       !HasVector)
     return getError(
         "'zvk*' requires 'v' or 'zve*' extension to also be specified");
diff --git a/llvm/test/CodeGen/RISCV/attributes.ll b/llvm/test/CodeGen/RISCV/attributes.ll
index 4f841683a868c4e..3aee484beeaa357 100644
--- a/llvm/test/CodeGen/RISCV/attributes.ll
+++ b/llvm/test/CodeGen/RISCV/attributes.ll
@@ -125,6 +125,8 @@
 ; RUN: llc -mtriple=riscv32 -mattr=+zama16b %s -o - | FileCheck --check-prefixes=CHECK,RV32ZAMA16B %s
 ; RUN: llc -mtriple=riscv32 -mattr=+experimental-zicfilp %s -o - | FileCheck --check-prefix=RV32ZICFILP %s
 ; RUN: llc -mtriple=riscv32 -mattr=+a,+zabha %s -o - | FileCheck --check-prefix=RV32ZABHA %s
+; RUN: llc -mtriple=riscv32 -mattr=+zve32x -mattr=+experimental-zvbc32e  %s -o - | FileCheck --check-prefix=RV32ZVBC32E %s
+; RUN: llc -mtriple=riscv32 -mattr=+zve32x -mattr=+experimental-zvkgs  %s -o - | FileCheck --check-prefix=RV32ZVKGS %s
 ; RUN: llc -mtriple=riscv32 -mattr=+experimental-ssnpm  %s -o - | FileCheck --check-prefix=RV32SSNPM %s
 ; RUN: llc -mtriple=riscv32 -mattr=+experimental-smnpm  %s -o - | FileCheck --check-prefix=RV32SMNPM %s
 ; RUN: llc -mtriple=riscv32 -mattr=+experimental-smmpm %s -o - | FileCheck --check-prefix=RV32SMMPM %s
@@ -263,6 +265,8 @@
 ; RUN: llc -mtriple=riscv64 -mattr=+experimental-zalasr %s -o - | FileCheck --check-prefix=RV64ZALASR %s
 ; RUN: llc -mtriple=riscv64 -mattr=+experimental-zicfilp %s -o - | FileCheck --check-prefix=RV64ZICFILP %s
 ; RUN: llc -mtriple=riscv64 -mattr=+a,+zabha %s -o - | FileCheck --check-prefix=RV64ZABHA %s
+; RUN: llc -mtriple=riscv64 -mattr=+zve32x -mattr=+experimental-zvbc32e  %s -o - | FileCheck --check-prefix=RV64ZVBC32E %s
+; RUN: llc -mtriple=riscv64 -mattr=+zve32x -mattr=+experimental-zvkgs  %s -o - | FileCheck --check-prefix=RV64ZVKGS %s
 ; RUN: llc -mtriple=riscv64 -mattr=+experimental-ssnpm  %s -o - | FileCheck --check-prefix=RV64SSNPM %s
 ; RUN: llc -mtriple=riscv64 -mattr=+experimental-smnpm  %s -o - | FileCheck --check-prefix=RV64SMNPM %s
 ; RUN: llc -mtriple=riscv64 -mattr=+experimental-smmpm %s -o - | FileCheck --check-prefix=RV64SMMPM %s
@@ -409,6 +413,8 @@
 ; RV32ZAMA16B: .attribute 5, "rv32i2p1_zama16b1p0"
 ; RV32ZICFILP: .attribute 5, "rv32i2p1_zicfilp1p0_zicsr2p0"
 ; RV32ZABHA: .attribute 5, "rv32i2p1_a2p1_zabha1p0"
+; RV32ZVBC32E: .attribute 5, "rv32i2p1_zicsr2p0_zvbc32e0p7_zve32x1p0_zvl32b1p0"
+; RV32ZVKGS: .attribute 5, "rv32i2p1_zicsr2p0_zve32x1p0_zvkg1p0_zvkgs0p7_zvl32b1p0"
 ; RV32SSNPM: .attribute 5, "rv32i2p1_ssnpm1p0"
 ; RV32SMNPM: .attribute 5, "rv32i2p1_smnpm1p0"
 ; RV32SMMPM: .attribute 5, "rv32i2p1_smmpm1p0"
@@ -545,6 +551,8 @@
 ; RV64ZALASR: .attribute 5, "rv64i2p1_zalasr0p1"
 ; RV64ZICFILP: .attribute 5, "rv64i2p1_zicfilp1p0_zicsr2p0"
 ; RV64ZABHA: .attribute 5, "rv64i2p1_a2p1_zabha1p0"
+; RV64ZVBC32E: .attribute 5, "rv64i2p1_zicsr2p0_zvbc32e0p7_zve32x1p0_zvl32b1p0"
+; RV64ZVKGS: .attribute 5, "rv64i2p1_zicsr2p0_zve32x1p0_zvkg1p0_zvkgs0p7_zvl32b1p0"
 ; RV64SSNPM: .attribute 5, "rv64i2p1_ssnpm1p0"
 ; RV64SMNPM: .attribute 5, "rv64i2p1_smnpm1p0"
 ; RV64SMMPM: .attribute 5, "rv64i2p1_smmpm1p0"
diff --git a/llvm/test/MC/RISCV/rvv/zvbc.s b/llvm/test/MC/RISCV/rvv/zvbc.s
index 0eb02d153b79b61..324d6e04a308fe5 100644
--- a/llvm/test/MC/RISCV/rvv/zvbc.s
+++ b/llvm/test/MC/RISCV/rvv/zvbc.s
@@ -7,27 +7,36 @@
 # RUN:        | FileCheck %s --check-prefix=CHECK-INST
 # RUN: llvm-mc -triple=riscv32 -filetype=obj --mattr=+zve64x --mattr=+zvbc %s \
 # RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+# RUN: llvm-mc -triple=riscv32 -show-encoding --mattr=+zve32x --mattr=+experimental-zvbc32e %s \
+# RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+# RUN: not llvm-mc -triple=riscv32 -show-encoding %s 2>&1 \
+# RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+# RUN: llvm-mc -triple=riscv32 -filetype=obj --mattr=+zve32x --mattr=+experimental-zvbc32e %s \
+# RUN:        | llvm-objdump -d --mattr=+zve32x --mattr=+experimental-zvbc32e  - \
+# RUN:        | FileCheck %s --check-prefix=CHECK-INST
+# RUN: llvm-mc -triple=riscv32 -filetype=obj --mattr=+zve32x --mattr=+experimental-zvbc32e %s \
+# RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
 
 vclmul.vv v10, v9, v8
 # CHECK-INST: vclmul.vv v10, v9, v8
 # CHECK-ENCODING: [0x57,0x25,0x94,0x32]
-# CHECK-ERROR: instruction requires the following: 'Zvbc' (Vector Carryless Multiplication){{$}}
+# CHECK-ERROR: instruction requires the following: 'Zvbc' or 'Zvbc32e' (Vector Carryless Multiplication){{$}}
 # CHECK-UNKNOWN: 32942557 <unknown>
 
 vclmul.vx v10, v9, a0
 # CHECK-INST: vclmul.vx v10, v9, a0
 # CHECK-ENCODING: [0x57,0x65,0x95,0x32]
-# CHECK-ERROR: instruction requires the following: 'Zvbc' (Vector Carryless Multiplication){{$}}
+# CHECK-ERROR: instruction requires the following: 'Zvbc' or 'Zvbc32e' (Vector Carryless Multiplication){{$}}
 # CHECK-UNKNOWN: 32956557 <unknown>
 
 vclmulh.vv v10, v9, v8
 # CHECK-INST: vclmulh.vv v10, v9, v8
 # CHECK-ENCODING: [0x57,0x25,0x94,0x36]
-# CHECK-ERROR: instruction requires the following: 'Zvbc' (Vector Carryless Multiplication){{$}}
+# CHECK-ERROR: instruction requires the following: 'Zvbc' or 'Zvbc32e' (Vector Carryless Multiplication){{$}}
 # CHECK-UNKNOWN: 36942557 <unknown>
 
 vclmulh.vx v10, v9, a0
 # CHECK-INST: vclmulh.vx v10, v9, a0
 # CHECK-ENCODING: [0x57,0x65,0x95,0x36]
-# CHECK-ERROR: instruction requires the following: 'Zvbc' (Vector Carryless Multiplication){{$}}
+# CHECK-ERROR: instruction requires the following: 'Zvbc' or 'Zvbc32e' (Vector Carryless Multiplication){{$}}
 # CHECK-UNKNOWN: 36956557 <unknown>
diff --git a/llvm/test/MC/RISCV/rvv/zvkgs.s b/llvm/test/MC/RISCV/rvv/zvkgs.s
new file mode 100644
index 000000000000000..5af4e8bb49ccbb1
--- /dev/null
+++ b/llvm/test/MC/RISCV/rvv/zvkgs.s
@@ -0,0 +1,21 @@
+# RUN: llvm-mc -triple=riscv32 -show-encoding --mattr=+zve32x --mattr=+experimental-zvkgs %s \
+# RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+# RUN: not llvm-mc -triple=riscv32 -show-encoding %s 2>&1 \
+# RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+# RUN: llvm-mc -triple=riscv32 -filetype=obj --mattr=+zve32x --mattr=+experimental-zvkgs %s \
+# RUN:        | llvm-objdump -d --mattr=+zve32x --mattr=+experimental-zvkgs  - \
+# RUN:        | FileCheck %s --check-prefix=CHECK-INST
+# RUN: llvm-mc -triple=riscv32 -filetype=obj --mattr=+zve32x --mattr=+experimental-zvkgs %s \
+# RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+
+vghsh.vs v10, v9, v8
+# CHECK-INST: vghsh.vs v10, v9, v8
+# CHECK-ENCODING: [0x77,0x25,0x94,0x8e]
+# CHECK-ERROR: instruction requires the following: 'Zvkgs' (Vector-Scalar GCM instructions for Cryptography){{$}}
+# CHECK-UNKNOWN: 8e942577 <unknown>
+
+vgmul.vs v10, v9
+# CHECK-INST: vgmul.vs v10, v9
+# CHECK-ENCODING: [0x77,0xa5,0x98,0xa6]
+# CHECK-ERROR: instruction requires the following: 'Zvkgs' (Vector-Scalar GCM instructions for Cryptography){{$}}
+# CHECK-UNKNOWN: a698a577 <unknown>
diff --git a/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp b/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
index bede4e64696c55c..426dca506135110 100644
--- a/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
+++ b/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
@@ -1029,6 +1029,8 @@ Experimental extensions
     zicfiss              1.0
     zacas                1.0
     zalasr               0.1
+    zvbc32e              0.7
+    zvkgs                0.7
     smmpm                1.0
     smnpm                1.0
     ssnpm                1.0

From 82fdfd4aa7f60b1f8e715211b925a7f2bfe57ea9 Mon Sep 17 00:00:00 2001
From: Akshat Oke <76596238+Akshat-Oke@users.noreply.github.com>
Date: Mon, 19 Aug 2024 09:59:47 +0530
Subject: [PATCH 323/441] [TableGen] Print Error and not crash on dumping
 non-string values (#104568)

Co-authored-by: Akshat Oke <Akshat.Oke@amd.com>
---
 llvm/lib/TableGen/Error.cpp | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/TableGen/Error.cpp b/llvm/lib/TableGen/Error.cpp
index dabb265ef80ca1f..4b99ff89da3b939 100644
--- a/llvm/lib/TableGen/Error.cpp
+++ b/llvm/lib/TableGen/Error.cpp
@@ -173,8 +173,11 @@ void CheckAssert(SMLoc Loc, Init *Condition, Init *Message) {
 // Dump a message to stderr.
 void dumpMessage(SMLoc Loc, Init *Message) {
   auto *MessageInit = dyn_cast<StringInit>(Message);
-  assert(MessageInit && "no debug message to print");
-  PrintNote(Loc, MessageInit->getValue());
+  if (!MessageInit) {
+    PrintError(Loc, "dump value is not of type string");
+  } else {
+    PrintNote(Loc, MessageInit->getValue());
+  }
 }
 
 } // end namespace llvm

From 7faf1a0868cb6a6a05e454044239292c454e0a71 Mon Sep 17 00:00:00 2001
From: Daniel Cederman <cederman@gaisler.com>
Date: Mon, 19 Aug 2024 07:59:58 +0200
Subject: [PATCH 324/441] [Sparc] Add errata workaround pass for GR712RC and
 UT700 (#103843)

This patch adds a pass that provides workarounds for the errata
described in GRLIB-TN-0009, GRLIB-TN-0010, GRLIB-TN-0011, GRLIB-TN-0012,
and GRLIB-TN-0013, that are applicable to the GR712RC and UT700. The
documents are available for download from here:

https://www.gaisler.com/index.php/information/app-tech-notes

The pass will detect certain sensitive instruction sequences and prevent
them from occurring by inserting NOP instruction. Below is an overview
of each of the workarounds. A similar implementation is available in
GCC.

GRLIB-TN-0009:

* Insert NOPs to prevent the sequence (stb/sth/st/stf) -> (single
non-store/load instruction) -> (any store)

* Insert NOPs to prevent the sequence (std/stdf) -> (any store)

GRLIB-TN-0010:

* Insert a NOP between load instruction and atomic instruction (swap and
casa).

* Insert a NOP at branch target if load in delay slot and atomic
instruction at branch target.

* Do not allow functions to begin with atomic instruction.

GRLIB-TN-0011:

* Insert .p2align 4 before atomic instructions (swap and casa).

GRLIB-TN-0012:

* Place a NOP at the branch target of an integer branch if it is a
floating-point operation or a floating-point branch.

GRLIB-TN-0013:

* Prevent (div/sqrt) instructions in the delay slot.

* Insert NOPs to prevent the sequence (div/sqrt) -> (two or three
floating point operations or loads) -> (div/sqrt).

* Do not insert NOPs if any of the floating point operations have a
dependency on the destination register of the first (div/sqrt).

* Do not insert NOPs if one of the floating point operations is a
(div/sqrt).

* Insert NOPs to prevent (div/sqrt) followed by a branch.
---
 llvm/lib/Target/Sparc/DelaySlotFiller.cpp    |  14 +
 llvm/lib/Target/Sparc/LeonFeatures.td        |  35 +++
 llvm/lib/Target/Sparc/LeonPasses.cpp         | 304 +++++++++++++++++++
 llvm/lib/Target/Sparc/LeonPasses.h           |  27 ++
 llvm/lib/Target/Sparc/Sparc.h                |   1 +
 llvm/lib/Target/Sparc/SparcAsmPrinter.cpp    |   6 +
 llvm/lib/Target/Sparc/SparcTargetMachine.cpp |   2 +
 llvm/test/CodeGen/SPARC/tn0009.mir           |  26 ++
 llvm/test/CodeGen/SPARC/tn0010.mir           |  79 +++++
 llvm/test/CodeGen/SPARC/tn0011.ll            |  26 ++
 llvm/test/CodeGen/SPARC/tn0012.mir           |  65 ++++
 llvm/test/CodeGen/SPARC/tn0013.mir           |  93 ++++++
 12 files changed, 678 insertions(+)
 create mode 100644 llvm/test/CodeGen/SPARC/tn0009.mir
 create mode 100644 llvm/test/CodeGen/SPARC/tn0010.mir
 create mode 100644 llvm/test/CodeGen/SPARC/tn0011.ll
 create mode 100644 llvm/test/CodeGen/SPARC/tn0012.mir
 create mode 100644 llvm/test/CodeGen/SPARC/tn0013.mir

diff --git a/llvm/lib/Target/Sparc/DelaySlotFiller.cpp b/llvm/lib/Target/Sparc/DelaySlotFiller.cpp
index 7e129101fefc790..a74bfe490b90189 100644
--- a/llvm/lib/Target/Sparc/DelaySlotFiller.cpp
+++ b/llvm/lib/Target/Sparc/DelaySlotFiller.cpp
@@ -282,6 +282,20 @@ bool Filler::delayHasHazard(MachineBasicBlock::iterator candidate,
       Opcode >=  SP::FDIVD && Opcode <= SP::FSQRTD)
     return true;
 
+  if (Subtarget->fixTN0009() && candidate->mayStore())
+    return true;
+
+  if (Subtarget->fixTN0013()) {
+    switch (Opcode) {
+    case SP::FDIVS:
+    case SP::FDIVD:
+    case SP::FSQRTS:
+    case SP::FSQRTD:
+      return true;
+    default:
+      break;
+    }
+  }
 
   return false;
 }
diff --git a/llvm/lib/Target/Sparc/LeonFeatures.td b/llvm/lib/Target/Sparc/LeonFeatures.td
index 75273eff186867f..ac2ad5a577f4b3f 100644
--- a/llvm/lib/Target/Sparc/LeonFeatures.td
+++ b/llvm/lib/Target/Sparc/LeonFeatures.td
@@ -61,3 +61,38 @@ def FixAllFDIVSQRT : SubtargetFeature<
 def LeonCycleCounter
   : SubtargetFeature<"leoncyclecounter", "HasLeonCycleCounter", "true",
                      "Use the Leon cycle counter register">;
+
+def FixTN0009 : SubtargetFeature<
+  "fix-tn0009",
+  "FixTN0009",
+  "true",
+  "Enable workaround for errata described in GRLIB-TN-0009"
+>;
+
+def FixTN0010 : SubtargetFeature<
+  "fix-tn0010",
+  "FixTN0010",
+  "true",
+  "Enable workaround for errata described in GRLIB-TN-0010"
+>;
+
+def FixTN0011 : SubtargetFeature<
+  "fix-tn0011",
+  "FixTN0011",
+  "true",
+  "Enable workaround for errata described in GRLIB-TN-0011"
+>;
+
+def FixTN0012 : SubtargetFeature<
+  "fix-tn0012",
+  "FixTN0012",
+  "true",
+  "Enable workaround for errata described in GRLIB-TN-0012"
+>;
+
+def FixTN0013 : SubtargetFeature<
+  "fix-tn0013",
+  "FixTN0013",
+  "true",
+  "Enable workaround for errata described in GRLIB-TN-0013"
+>;
diff --git a/llvm/lib/Target/Sparc/LeonPasses.cpp b/llvm/lib/Target/Sparc/LeonPasses.cpp
index 45a46c131d21eaf..018d8126f7a7e9a 100644
--- a/llvm/lib/Target/Sparc/LeonPasses.cpp
+++ b/llvm/lib/Target/Sparc/LeonPasses.cpp
@@ -19,6 +19,310 @@
 
 using namespace llvm;
 
+char ErrataWorkaround::ID = 0;
+
+ErrataWorkaround::ErrataWorkaround() : MachineFunctionPass(ID) {
+  initializeErrataWorkaroundPass(*PassRegistry::getPassRegistry());
+}
+
+INITIALIZE_PASS(ErrataWorkaround, "errata-workaround", "Errata workaround pass",
+                false, false)
+
+// Move iterator to the next instruction in the function, ignoring
+// meta instructions and inline assembly. Returns false when reaching
+// the end of the function.
+bool ErrataWorkaround::moveNext(MachineBasicBlock::iterator &I) {
+
+  MachineBasicBlock *MBB = I->getParent();
+
+  do {
+    I++;
+
+    while (I == MBB->end()) {
+      if (MBB->getFallThrough() == nullptr)
+        return false;
+      MBB = MBB->getFallThrough();
+      I = MBB->begin();
+    }
+  } while (I->isMetaInstruction() || I->isInlineAsm());
+
+  return true;
+}
+
+void ErrataWorkaround::insertNop(MachineBasicBlock::iterator I) {
+  BuildMI(*I->getParent(), I, I->getDebugLoc(), TII->get(SP::NOP));
+}
+
+bool ErrataWorkaround::isFloat(MachineBasicBlock::iterator I) {
+  if (I->getNumOperands() == 0)
+    return false;
+
+  if (!I->getOperand(0).isReg())
+    return false;
+
+  unsigned reg = I->getOperand(0).getReg();
+
+  if (!SP::FPRegsRegClass.contains(reg) && !SP::DFPRegsRegClass.contains(reg))
+    return false;
+
+  return true;
+}
+
+bool ErrataWorkaround::isDivSqrt(MachineBasicBlock::iterator I) {
+  switch (I->getOpcode()) {
+  case SP::FDIVS:
+  case SP::FDIVD:
+  case SP::FSQRTS:
+  case SP::FSQRTD:
+    return true;
+  }
+  return false;
+}
+
+// Prevents the following code sequence from being generated:
+// (stb/sth/st/stf) -> (single non-store/load instruction) -> (any store)
+// If the sequence is detected a NOP instruction is inserted after
+// the first store instruction.
+bool ErrataWorkaround::checkSeqTN0009A(MachineBasicBlock::iterator I) {
+  switch (I->getOpcode()) {
+  case SP::STrr:
+  case SP::STri:
+  case SP::STBrr:
+  case SP::STBri:
+  case SP::STHrr:
+  case SP::STHri:
+  case SP::STFrr:
+  case SP::STFri:
+    break;
+  default:
+    return false;
+  }
+
+  MachineBasicBlock::iterator MI = I;
+  if (!moveNext(MI))
+    return false;
+
+  if (MI->mayStore() || MI->mayLoad())
+    return false;
+
+  MachineBasicBlock::iterator PatchHere = MI;
+
+  if (!moveNext(MI))
+    return false;
+
+  if (!MI->mayStore())
+    return false;
+
+  insertNop(PatchHere);
+  return true;
+}
+
+// Prevents the following code sequence from being generated:
+// (std/stdf) -> (any store)
+// If the sequence is detected a NOP instruction is inserted after
+// the first store instruction.
+bool ErrataWorkaround::checkSeqTN0009B(MachineBasicBlock::iterator I) {
+
+  switch (I->getOpcode()) {
+  case SP::STDrr:
+  case SP::STDri:
+  case SP::STDFrr:
+  case SP::STDFri:
+    break;
+  default:
+    return false;
+  }
+
+  MachineBasicBlock::iterator MI = I;
+
+  if (!moveNext(MI))
+    return false;
+
+  if (!MI->mayStore())
+    return false;
+
+  insertNop(MI);
+  return true;
+}
+
+// Insert a NOP at branch target if load in delay slot and atomic
+// instruction at branch target. Also insert a NOP between load
+// instruction and atomic instruction (swap or casa).
+bool ErrataWorkaround::checkSeqTN0010(MachineBasicBlock::iterator I) {
+
+  // Check for load instruction or branch bundled with load instruction
+  if (!I->mayLoad())
+    return false;
+
+  // Check for branch to atomic instruction with load in delay slot
+  if (I->isBranch()) {
+    MachineBasicBlock *TargetMBB = I->getOperand(0).getMBB();
+    MachineBasicBlock::iterator MI = TargetMBB->begin();
+
+    while (MI != TargetMBB->end() && MI->isMetaInstruction())
+      MI++;
+
+    if (MI == TargetMBB->end())
+      return false;
+
+    switch (MI->getOpcode()) {
+    case SP::SWAPrr:
+    case SP::SWAPri:
+    case SP::CASArr:
+      insertNop(MI);
+      break;
+    default:
+      break;
+    }
+  }
+
+  // Check for load followed by atomic instruction
+  MachineBasicBlock::iterator MI = I;
+  if (!moveNext(MI))
+    return false;
+
+  switch (MI->getOpcode()) {
+  case SP::SWAPrr:
+  case SP::SWAPri:
+  case SP::CASArr:
+    break;
+  default:
+    return false;
+  }
+  insertNop(MI);
+  return true;
+}
+
+// Do not allow functions to begin with an atomic instruction
+bool ErrataWorkaround::checkSeqTN0010First(MachineBasicBlock &MBB) {
+  MachineBasicBlock::iterator I = MBB.begin();
+  while (I != MBB.end() && I->isMetaInstruction())
+    I++;
+  switch (I->getOpcode()) {
+  case SP::SWAPrr:
+  case SP::SWAPri:
+  case SP::CASArr:
+    break;
+  default:
+    return false;
+  }
+  insertNop(I);
+  return true;
+}
+
+// Inserts a NOP instruction at the target of an integer branch if the
+// target is a floating-point instruction or floating-point branch.
+bool ErrataWorkaround::checkSeqTN0012(MachineBasicBlock::iterator I) {
+
+  if (I->getOpcode() != SP::BCOND && I->getOpcode() != SP::BCONDA)
+    return false;
+
+  MachineBasicBlock *TargetMBB = I->getOperand(0).getMBB();
+  MachineBasicBlock::iterator MI = TargetMBB->begin();
+
+  while (MI != TargetMBB->end() && MI->isMetaInstruction())
+    MI++;
+
+  if (MI == TargetMBB->end())
+    return false;
+
+  if (!isFloat(MI) && MI->getOpcode() != SP::FBCOND)
+    return false;
+
+  insertNop(MI);
+  return true;
+}
+
+// Prevents the following code sequence from being generated:
+// (div/sqrt) -> (2 to 3 floating-point operations or loads) -> (div/sqrt)
+// If the sequence is detected one or two NOP instruction are inserted after
+// the first div/sqrt instruction. No NOPs are inserted if one of the floating-
+// point instructions in the middle of the sequence is a (div/sqrt), or if
+// they have dependency on the destination register of the first (div/sqrt).
+//
+// The function also prevents the following code sequence from being generated,
+// (div/sqrt) -> (branch), by inserting a NOP instruction after the (div/sqrt).
+bool ErrataWorkaround::checkSeqTN0013(MachineBasicBlock::iterator I) {
+
+  if (!isDivSqrt(I))
+    return false;
+
+  unsigned dstReg = I->getOperand(0).getReg();
+
+  MachineBasicBlock::iterator MI = I;
+  if (!moveNext(MI))
+    return false;
+
+  if (MI->isBranch()) {
+    insertNop(MI);
+    return true;
+  }
+
+  MachineBasicBlock::iterator PatchHere = MI;
+
+  unsigned fpFound = 0;
+  for (unsigned i = 0; i < 4; i++) {
+
+    if (!isFloat(MI)) {
+      if (!moveNext(MI))
+        return false;
+      continue;
+    }
+
+    if (MI->readsRegister(dstReg, TRI))
+      return false;
+
+    if (isDivSqrt(MI)) {
+      if (i < 2)
+        return false;
+      if (fpFound < 2)
+        return false;
+
+      insertNop(PatchHere);
+      if (i == 2)
+        insertNop(PatchHere);
+      return true;
+    }
+
+    fpFound++;
+    if (!moveNext(MI))
+      return false;
+  }
+
+  return false;
+}
+
+bool ErrataWorkaround::runOnMachineFunction(MachineFunction &MF) {
+  bool Changed = false;
+  ST = &MF.getSubtarget<SparcSubtarget>();
+
+  if (!(ST->fixTN0009() || ST->fixTN0010() || ST->fixTN0012() ||
+        ST->fixTN0013()))
+    return false;
+
+  TII = ST->getInstrInfo();
+  TRI = ST->getRegisterInfo();
+
+  if (ST->fixTN0010())
+    Changed |= checkSeqTN0010First(MF.front());
+
+  for (auto &MBB : MF) {
+    for (auto &I : MBB) {
+      if (ST->fixTN0009()) {
+        Changed |= checkSeqTN0009A(I);
+        Changed |= checkSeqTN0009B(I);
+      }
+      if (ST->fixTN0010())
+        Changed |= checkSeqTN0010(I);
+      if (ST->fixTN0012())
+        Changed |= checkSeqTN0012(I);
+      if (ST->fixTN0013())
+        Changed |= checkSeqTN0013(I);
+    }
+  }
+  return Changed;
+}
+
 LEONMachineFunctionPass::LEONMachineFunctionPass(char &ID)
     : MachineFunctionPass(ID) {}
 
diff --git a/llvm/lib/Target/Sparc/LeonPasses.h b/llvm/lib/Target/Sparc/LeonPasses.h
index 9bc4569a12984ac..261b283fadaeb31 100644
--- a/llvm/lib/Target/Sparc/LeonPasses.h
+++ b/llvm/lib/Target/Sparc/LeonPasses.h
@@ -37,6 +37,33 @@ class LLVM_LIBRARY_VISIBILITY LEONMachineFunctionPass
   }
 };
 
+class LLVM_LIBRARY_VISIBILITY ErrataWorkaround : public MachineFunctionPass {
+protected:
+  const SparcSubtarget *ST;
+  const TargetInstrInfo *TII;
+  const TargetRegisterInfo *TRI;
+
+  bool checkSeqTN0009A(MachineBasicBlock::iterator I);
+  bool checkSeqTN0009B(MachineBasicBlock::iterator I);
+  bool checkSeqTN0010First(MachineBasicBlock &MBB);
+  bool checkSeqTN0010(MachineBasicBlock::iterator I);
+  bool checkSeqTN0012(MachineBasicBlock::iterator I);
+  bool checkSeqTN0013(MachineBasicBlock::iterator I);
+
+  bool moveNext(MachineBasicBlock::iterator &I);
+  bool isFloat(MachineBasicBlock::iterator I);
+  bool isDivSqrt(MachineBasicBlock::iterator I);
+  void insertNop(MachineBasicBlock::iterator I);
+
+public:
+  static char ID;
+
+  ErrataWorkaround();
+  bool runOnMachineFunction(MachineFunction &MF) override;
+
+  StringRef getPassName() const override { return "Errata workaround pass"; };
+};
+
 class LLVM_LIBRARY_VISIBILITY InsertNOPLoad : public LEONMachineFunctionPass {
 public:
   static char ID;
diff --git a/llvm/lib/Target/Sparc/Sparc.h b/llvm/lib/Target/Sparc/Sparc.h
index 33a803469603b3b..afcbc473295411e 100644
--- a/llvm/lib/Target/Sparc/Sparc.h
+++ b/llvm/lib/Target/Sparc/Sparc.h
@@ -32,6 +32,7 @@ FunctionPass *createSparcDelaySlotFillerPass();
 void LowerSparcMachineInstrToMCInst(const MachineInstr *MI, MCInst &OutMI,
                                     AsmPrinter &AP);
 void initializeSparcDAGToDAGISelLegacyPass(PassRegistry &);
+void initializeErrataWorkaroundPass(PassRegistry &);
 } // namespace llvm
 
 namespace llvm {
diff --git a/llvm/lib/Target/Sparc/SparcAsmPrinter.cpp b/llvm/lib/Target/Sparc/SparcAsmPrinter.cpp
index 182779fa93e5033..87d76ebf268c41f 100644
--- a/llvm/lib/Target/Sparc/SparcAsmPrinter.cpp
+++ b/llvm/lib/Target/Sparc/SparcAsmPrinter.cpp
@@ -276,6 +276,12 @@ void SparcAsmPrinter::emitInstruction(const MachineInstr *MI) {
   case TargetOpcode::DBG_VALUE:
     // FIXME: Debug Value.
     return;
+  case SP::CASArr:
+  case SP::SWAPrr:
+  case SP::SWAPri:
+    if (MF->getSubtarget<SparcSubtarget>().fixTN0011())
+      OutStreamer->emitCodeAlignment(Align(16), &getSubtargetInfo());
+    break;
   case SP::GETPCX:
     LowerGETPCXAndEmitMCInsts(MI, getSubtargetInfo());
     return;
diff --git a/llvm/lib/Target/Sparc/SparcTargetMachine.cpp b/llvm/lib/Target/Sparc/SparcTargetMachine.cpp
index ea403230bea8938..fec2d3a35ae6d29 100644
--- a/llvm/lib/Target/Sparc/SparcTargetMachine.cpp
+++ b/llvm/lib/Target/Sparc/SparcTargetMachine.cpp
@@ -29,6 +29,7 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeSparcTarget() {
 
   PassRegistry &PR = *PassRegistry::getPassRegistry();
   initializeSparcDAGToDAGISelLegacyPass(PR);
+  initializeErrataWorkaroundPass(PR);
 }
 
 static cl::opt<bool>
@@ -193,6 +194,7 @@ void SparcPassConfig::addPreEmitPass(){
   addPass(new InsertNOPLoad());
   addPass(new DetectRoundChange());
   addPass(new FixAllFDIVSQRT());
+  addPass(new ErrataWorkaround());
 }
 
 void SparcV8TargetMachine::anchor() { }
diff --git a/llvm/test/CodeGen/SPARC/tn0009.mir b/llvm/test/CodeGen/SPARC/tn0009.mir
new file mode 100644
index 000000000000000..f3807940cd30ec4
--- /dev/null
+++ b/llvm/test/CodeGen/SPARC/tn0009.mir
@@ -0,0 +1,26 @@
+# RUN: llc %s -mattr=+fix-tn0009 -march=sparc -run-pass=errata-workaround -o - \
+# RUN:   | FileCheck %s
+
+---
+# CHECK:      STrr $o1, $g0, $o0
+# CHECK-NEXT: NOP
+# CHECK-NEXT: NOP
+# CHECK-NEXT: STrr $o1, $g0, $o0
+name:            seqa
+body:             |
+  bb.0.entry:
+    STrr $o1, $g0, $o0
+    NOP
+    STrr $o1, $g0, $o0
+
+---
+# CHECK:      STDrr $i0, $g0, $i0_i1
+# CHECK-NEXT: NOP
+# CHECK-NEXT: STDrr $i0, $g0, $i0_i1
+name:            seqb
+body:             |
+  bb.0.entry:
+    STDrr $i0, $g0, $i0_i1
+    STDrr $i0, $g0, $i0_i1
+
+...
diff --git a/llvm/test/CodeGen/SPARC/tn0010.mir b/llvm/test/CodeGen/SPARC/tn0010.mir
new file mode 100644
index 000000000000000..416dd740cf21cac
--- /dev/null
+++ b/llvm/test/CodeGen/SPARC/tn0010.mir
@@ -0,0 +1,79 @@
+# RUN: llc %s -mattr=+fix-tn0010 -march=sparc -run-pass=errata-workaround -o - \
+# RUN:   | FileCheck %s
+
+---
+# CHECK:      LDrr
+# CHECK-NEXT: NOP
+# CHECK-NEXT: CASArr
+# CHECK:      LDrr
+# CHECK-NEXT: NOP
+# CHECK-NEXT: SWAPrr
+# CHECK:      LDrr
+# CHECK-NEXT: NOP
+# CHECK-NEXT: SWAPri
+name:            er-5-1
+registers:
+  - { id: 0, class: intregs }
+body:             |
+  bb.0.entry:
+    %0 = LDrr $o0, $g0
+    %0 = CASArr $o2, $o3, %0, 10
+    %0 = LDrr $g0, $o0
+    %0 = SWAPrr $o2, $o3, %0
+    %0 = LDrr $g0, $o0
+    %0 = SWAPri $o2, 10, %0
+
+---
+# CHECK:      bb.2:
+# CHECK-NEXT: NOP
+# CHECK-NEXT: CASArr
+name:            er-5-2
+registers:
+  - { id: 0, class: intregs }
+body:             |
+  bb.0.entry:
+    successors: %bb.1, %bb.2
+    BCOND %bb.2, 10, implicit $icc {
+      %0 = LDrr $g0, $o0
+    }
+
+  bb.1.entry:
+    successors: %bb.2
+    NOP
+
+  bb.2:
+    %0 = CASArr $o1, $o2, %0, 10
+
+---
+# CHECK:      bb.0.entry:
+# CHECK-NEXT: NOP
+# CHECK-NEXT: CASArr
+name:            er-5-3
+registers:
+  - { id: 0, class: intregs }
+body:             |
+  bb.0.entry:
+    %0 = CASArr $o1, $o2, %0, 10
+
+---
+# CHECK:      bb.1.entry:
+# CHECK:      NOP
+# CHECK-NEXT: CASArr
+name:            er-5-4
+registers:
+  - { id: 0, class: intregs }
+body:             |
+  bb.0.entry:
+    successors: %bb.1, %bb.2
+    BCOND %bb.2, 10, implicit $icc {
+      %0 = LDrr $g0, $o0
+    }
+
+  bb.1.entry:
+    successors: %bb.2
+    %0 = CASArr $o1, $o2, %0, 10
+
+  bb.2:
+    NOP
+
+...
diff --git a/llvm/test/CodeGen/SPARC/tn0011.ll b/llvm/test/CodeGen/SPARC/tn0011.ll
new file mode 100644
index 000000000000000..e13379eb54f4a98
--- /dev/null
+++ b/llvm/test/CodeGen/SPARC/tn0011.ll
@@ -0,0 +1,26 @@
+; RUN: llc < %s -mtriple=sparc -mattr=fix-tn0011,hasleoncasa | FileCheck %s
+
+; CHECK: .p2align 4
+; CHECK-NEXT: casa
+define i32 @test_casarr(i32* %p, i32 %v) {
+entry:
+  %0 = atomicrmw nand i32* %p, i32 %v seq_cst
+  ret i32 %0
+}
+
+; CHECK: .p2align 4
+; CHECK-NEXT: swap
+define i32 @test_swaprr(i32* %p, i32 %v) {
+entry:
+  %0 = atomicrmw xchg i32* %p, i32 %v seq_cst
+  ret i32 %0
+}
+
+; CHECK: .p2align 4
+; CHECK-NEXT: swap
+define i32 @test_swapri(i32* %p, i32 %v) {
+entry:
+  %1 = getelementptr inbounds i32, ptr %p, i32 1
+  %2 = atomicrmw xchg ptr %1, i32 %v seq_cst, align 4
+  ret i32 %2
+}
diff --git a/llvm/test/CodeGen/SPARC/tn0012.mir b/llvm/test/CodeGen/SPARC/tn0012.mir
new file mode 100644
index 000000000000000..f6c55b0d9a770d8
--- /dev/null
+++ b/llvm/test/CodeGen/SPARC/tn0012.mir
@@ -0,0 +1,65 @@
+# RUN: llc %s -mattr=+fix-tn0012 -march=sparc -run-pass=errata-workaround -o - \
+# RUN:   | FileCheck %s
+
+---
+# CHECK:      bb.2:
+# CHECK:      NOP
+# CHECK-NEXT: $f0 = FADDS $f0, $f0
+name:            er-5-1
+body:             |
+  bb.0.entry:
+    successors: %bb.1, %bb.2
+    BCOND %bb.2, 10, implicit $icc {
+      NOP
+    }
+
+  bb.1.entry:
+    successors: %bb.2
+    NOP
+
+  bb.2:
+    $f0 = FADDS $f0, $f0
+
+---
+# CHECK:      bb.2:
+# CHECK:      NOP
+# CHECK-NEXT: FBCOND %bb.3, 22, implicit $fcc0 {
+name:            er-5-2
+body:             |
+  bb.0.entry:
+    successors: %bb.2, %bb.1
+    BCOND %bb.1, 10, implicit $icc {
+      NOP
+    }
+
+  bb.2.entry:
+    successors: %bb.1
+    NOP
+
+  bb.1:
+    successors: %bb.3
+    FBCOND %bb.3, 22, implicit $fcc0 {
+      NOP
+    }
+  bb.3:
+   NOP
+
+---
+# CHECK:      bb.2:
+# CHECK:      NOP
+# CHECK-NEXT: FCMPS $f0, $f0, implicit-def $fcc0
+name:            er-5-3
+body:             |
+  bb.0.entry:
+    successors: %bb.1, %bb.2
+    BCOND %bb.2, 10, implicit $icc {
+      NOP
+    }
+
+  bb.1.entry:
+    successors: %bb.2
+    NOP
+
+  bb.2:
+    FCMPS $f0, $f0, implicit-def $fcc0
+...
diff --git a/llvm/test/CodeGen/SPARC/tn0013.mir b/llvm/test/CodeGen/SPARC/tn0013.mir
new file mode 100644
index 000000000000000..068bd0f02b34829
--- /dev/null
+++ b/llvm/test/CodeGen/SPARC/tn0013.mir
@@ -0,0 +1,93 @@
+# RUN: llc %s -mattr=+fix-tn0013 -march=sparc -run-pass=errata-workaround -o - \
+# RUN:   | FileCheck %s
+
+---
+# CHECK:      $f0 = FSQRTS $f0
+# CHECK-NEXT: NOP
+# CHECK-NEXT: NOP
+# CHECK-NEXT: $f3 = FADDS $f1, $f2
+# CHECK-NEXT: $f3 = FADDS $f1, $f2
+# CHECK-NEXT: $f0 = FDIVS $f4, $f5
+name:            er-8-1
+body:             |
+  bb.0.entry:
+    $f0 = FSQRTS $f0
+    $f3 = FADDS $f1, $f2
+    $f3 = FADDS $f1, $f2
+    $f0 = FDIVS $f4, $f5
+
+---
+# CHECK:      $f0 = FDIVS $f0, $f0
+# CHECK-NEXT: NOP
+# CHECK-NEXT: $f3 = FADDS $f1, $f2
+# CHECK-NEXT: $f3 = FADDS $f1, $f2
+# CHECK-NEXT: $f3 = FADDS $f1, $f2
+# CHECK-NEXT: $f0 = FSQRTS $f4
+name:            er-8-2
+body:             |
+  bb.0.entry:
+    $f0 = FDIVS $f0, $f0
+    $f3 = FADDS $f1, $f2
+    $f3 = FADDS $f1, $f2
+    $f3 = FADDS $f1, $f2
+    $f0 = FSQRTS $f4
+
+---
+# CHECK-NOT:  NOP
+name:            er-9
+body:             |
+  bb.0.entry:
+    $f0 = FSQRTS $f0
+    $f3 = FADDS $f1, $f2
+    $f3 = FADDS $f0, $f2
+    $f3 = FADDS $f1, $f2
+    $f0 = FSQRTS $f0
+
+---
+# CHECK-NOT:  NOP
+name:            er-10
+body:             |
+  bb.0.entry:
+    $f0 = FSQRTS $f0
+    $f4 = FSQRTS $f4
+    $f3 = FADDS $f1, $f2
+    $f0 = FSQRTS $f0
+
+---
+# CHECK:      er-11
+# CHECK:      $f0 = FSQRTS $f0
+# CHECK-NEXT: NOP
+# CHECK:      $f0 = FDIVS $f0, $f0
+# CHECK-NEXT: NOP
+name:            er-11
+body:             |
+  bb.0.entry:
+    successors: %bb.3, %bb.1
+    $f0 = FSQRTS $f0
+    FBCOND %bb.3, 22, implicit $fcc0 {
+      NOP
+    }
+  bb.1:
+    successors: %bb.3, %bb.4
+    $f0 = FDIVS $f0, $f0
+    BCOND %bb.4, 10, implicit $icc {
+      NOP
+    }
+  bb.3:
+    NOP
+  bb.4:
+    NOP
+
+---
+# CHECK:      $f1 = FDIVS $f0, $f1
+# CHECK-NEXT: NOP
+# CHECK-NEXT: STri $i6, -84, $i2
+name:            er-8-3
+body:             |
+  bb.0.entry:
+    $f1 = FDIVS $f0, $f1
+    STri $i6, -84, $i2
+    $f0 = LDFri $i6, -84
+    $f0 = FITOS $f0
+    $f5 = FDIVS $f4, $f0
+...

From f802c39cea29d587ba4db6f71352e3f11a994a89 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Fri, 16 Aug 2024 17:14:57 -0700
Subject: [PATCH 325/441] [RISCV] Add more tests for
 RISCVISAInfo::checkDependency(). NFC

---
 .../TargetParser/RISCVISAInfoTest.cpp         | 88 +++++++++++++++++++
 1 file changed, 88 insertions(+)

diff --git a/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp b/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
index 426dca506135110..4c5087b570fc93a 100644
--- a/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
+++ b/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
@@ -620,6 +620,94 @@ TEST(ParseArchString, RejectsConflictingExtensions) {
     EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
               "'zcf' is only supported for 'rv32'");
   }
+
+  for (StringRef Input : {"rv64i_xwchc"}) {
+    EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
+              "'Xwchc' is only supported for 'rv32'");
+  }
+
+  for (StringRef Input : {"rv32id_xwchc"}) {
+    EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
+              "'D' and 'Xwchc' extensions are incompatible");
+  }
+
+  for (StringRef Input : {"rv32i_zcb_xwchc"}) {
+    EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
+              "'Xwchc' and 'Zcb' extensions are incompatible");
+  }
+}
+
+TEST(ParseArchString, MissingDepency) {
+  for (StringRef Input : {"rv32i_zvl32b", "rv64i_zvl128b"}) {
+    EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
+              "'zvl*b' requires 'v' or 'zve*' extension to also be specified");
+  }
+
+  for (StringRef Input : {"rv32i_zvbb"}) {
+    EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
+              "'zvbb' requires 'v' or 'zve*' extension to also be specified");
+  }
+
+  for (StringRef Input : {"rv32i_zvbc32e0p7"}) {
+    EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
+              "'zvbc32e' requires 'v' or 'zve*' extension to also be specified");
+  }
+
+  for (StringRef Input : {"rv32i_zvbc"}) {
+    EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
+              "'zvbc' requires 'v' or 'zve64*' extension to also be specified");
+  }
+
+  for (StringRef Input : {"rv32i_zvkb"}) {
+    EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
+              "'zvk*' requires 'v' or 'zve*' extension to also be specified");
+  }
+
+  for (StringRef Input : {"rv32i_zvkg"}) {
+    EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
+              "'zvk*' requires 'v' or 'zve*' extension to also be specified");
+  }
+
+  for (StringRef Input : {"rv32i_zvkgs0p7"}) {
+    EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
+              "'zvk*' requires 'v' or 'zve*' extension to also be specified");
+  }
+
+  for (StringRef Input : {"rv32i_zvkned"}) {
+    EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
+              "'zvk*' requires 'v' or 'zve*' extension to also be specified");
+  }
+
+  for (StringRef Input : {"rv32i_zvknha"}) {
+    EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
+              "'zvk*' requires 'v' or 'zve*' extension to also be specified");
+  }
+
+  for (StringRef Input : {"rv32i_zvksed"}) {
+    EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
+              "'zvk*' requires 'v' or 'zve*' extension to also be specified");
+  }
+
+  for (StringRef Input : {"rv32i_zvksh"}) {
+    EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
+              "'zvk*' requires 'v' or 'zve*' extension to also be specified");
+  }
+
+  for (StringRef Input : {"rv32i_zvknhb"}) {
+    EXPECT_EQ(
+        toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
+        "'zvknhb' requires 'v' or 'zve64*' extension to also be specified");
+  }
+
+  for (StringRef Input : {"rv32i_zacas1p0"}) {
+    EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
+              "'zacas' requires 'a' or 'zaamo' extension to also be specified");
+  }
+
+  for (StringRef Input : {"rv32i_zabha"}) {
+    EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
+              "'zabha' requires 'a' or 'zaamo' extension to also be specified");
+  }
 }
 
 TEST(ParseArchString, RejectsUnrecognizedProfileNames) {

From 164d1230f78b32647e1a6e948245e75f557a8068 Mon Sep 17 00:00:00 2001
From: Pengcheng Wang <wangpengcheng.pp@bytedance.com>
Date: Mon, 19 Aug 2024 15:16:51 +0800
Subject: [PATCH 326/441] [RISCV] Simplify reserse fixed regs (#104736)

Add a macro to simplify some codes.
---
 clang/lib/Driver/ToolChains/Arch/RISCV.cpp | 99 ++++++++--------------
 1 file changed, 36 insertions(+), 63 deletions(-)

diff --git a/clang/lib/Driver/ToolChains/Arch/RISCV.cpp b/clang/lib/Driver/ToolChains/Arch/RISCV.cpp
index 149a31f58e75d21..6935904a24edbfd 100644
--- a/clang/lib/Driver/ToolChains/Arch/RISCV.cpp
+++ b/clang/lib/Driver/ToolChains/Arch/RISCV.cpp
@@ -95,69 +95,42 @@ void riscv::getRISCVTargetFeatures(const Driver &D, const llvm::Triple &Triple,
       CPUFastVectorUnaligned = true;
   }
 
-  // Handle features corresponding to "-ffixed-X" options
-  if (Args.hasArg(options::OPT_ffixed_x1))
-    Features.push_back("+reserve-x1");
-  if (Args.hasArg(options::OPT_ffixed_x2))
-    Features.push_back("+reserve-x2");
-  if (Args.hasArg(options::OPT_ffixed_x3))
-    Features.push_back("+reserve-x3");
-  if (Args.hasArg(options::OPT_ffixed_x4))
-    Features.push_back("+reserve-x4");
-  if (Args.hasArg(options::OPT_ffixed_x5))
-    Features.push_back("+reserve-x5");
-  if (Args.hasArg(options::OPT_ffixed_x6))
-    Features.push_back("+reserve-x6");
-  if (Args.hasArg(options::OPT_ffixed_x7))
-    Features.push_back("+reserve-x7");
-  if (Args.hasArg(options::OPT_ffixed_x8))
-    Features.push_back("+reserve-x8");
-  if (Args.hasArg(options::OPT_ffixed_x9))
-    Features.push_back("+reserve-x9");
-  if (Args.hasArg(options::OPT_ffixed_x10))
-    Features.push_back("+reserve-x10");
-  if (Args.hasArg(options::OPT_ffixed_x11))
-    Features.push_back("+reserve-x11");
-  if (Args.hasArg(options::OPT_ffixed_x12))
-    Features.push_back("+reserve-x12");
-  if (Args.hasArg(options::OPT_ffixed_x13))
-    Features.push_back("+reserve-x13");
-  if (Args.hasArg(options::OPT_ffixed_x14))
-    Features.push_back("+reserve-x14");
-  if (Args.hasArg(options::OPT_ffixed_x15))
-    Features.push_back("+reserve-x15");
-  if (Args.hasArg(options::OPT_ffixed_x16))
-    Features.push_back("+reserve-x16");
-  if (Args.hasArg(options::OPT_ffixed_x17))
-    Features.push_back("+reserve-x17");
-  if (Args.hasArg(options::OPT_ffixed_x18))
-    Features.push_back("+reserve-x18");
-  if (Args.hasArg(options::OPT_ffixed_x19))
-    Features.push_back("+reserve-x19");
-  if (Args.hasArg(options::OPT_ffixed_x20))
-    Features.push_back("+reserve-x20");
-  if (Args.hasArg(options::OPT_ffixed_x21))
-    Features.push_back("+reserve-x21");
-  if (Args.hasArg(options::OPT_ffixed_x22))
-    Features.push_back("+reserve-x22");
-  if (Args.hasArg(options::OPT_ffixed_x23))
-    Features.push_back("+reserve-x23");
-  if (Args.hasArg(options::OPT_ffixed_x24))
-    Features.push_back("+reserve-x24");
-  if (Args.hasArg(options::OPT_ffixed_x25))
-    Features.push_back("+reserve-x25");
-  if (Args.hasArg(options::OPT_ffixed_x26))
-    Features.push_back("+reserve-x26");
-  if (Args.hasArg(options::OPT_ffixed_x27))
-    Features.push_back("+reserve-x27");
-  if (Args.hasArg(options::OPT_ffixed_x28))
-    Features.push_back("+reserve-x28");
-  if (Args.hasArg(options::OPT_ffixed_x29))
-    Features.push_back("+reserve-x29");
-  if (Args.hasArg(options::OPT_ffixed_x30))
-    Features.push_back("+reserve-x30");
-  if (Args.hasArg(options::OPT_ffixed_x31))
-    Features.push_back("+reserve-x31");
+// Handle features corresponding to "-ffixed-X" options
+#define RESERVE_REG(REG)                                                       \
+  if (Args.hasArg(options::OPT_ffixed_##REG))                                  \
+    Features.push_back("+reserve-" #REG);
+  RESERVE_REG(x1)
+  RESERVE_REG(x2)
+  RESERVE_REG(x3)
+  RESERVE_REG(x4)
+  RESERVE_REG(x5)
+  RESERVE_REG(x6)
+  RESERVE_REG(x7)
+  RESERVE_REG(x8)
+  RESERVE_REG(x9)
+  RESERVE_REG(x10)
+  RESERVE_REG(x11)
+  RESERVE_REG(x12)
+  RESERVE_REG(x13)
+  RESERVE_REG(x14)
+  RESERVE_REG(x15)
+  RESERVE_REG(x16)
+  RESERVE_REG(x17)
+  RESERVE_REG(x18)
+  RESERVE_REG(x19)
+  RESERVE_REG(x20)
+  RESERVE_REG(x21)
+  RESERVE_REG(x22)
+  RESERVE_REG(x23)
+  RESERVE_REG(x24)
+  RESERVE_REG(x25)
+  RESERVE_REG(x26)
+  RESERVE_REG(x27)
+  RESERVE_REG(x28)
+  RESERVE_REG(x29)
+  RESERVE_REG(x30)
+  RESERVE_REG(x31)
+#undef RESERVE_REG
 
   // -mrelax is default, unless -mno-relax is specified.
   if (Args.hasFlag(options::OPT_mrelax, options::OPT_mno_relax, true)) {

From d489b7ccb7b43fe54dfb4b2884d7f1eb2cb5806e Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Fri, 16 Aug 2024 14:20:49 -0700
Subject: [PATCH 327/441] [RISCV] Merge some ISA error reporting together and
 make some errors more precise.

Loop over the extension names that have the same error message.

Print the name of Zvk* extensions instead of 'zvk*'.
---
 llvm/lib/TargetParser/RISCVISAInfo.cpp        | 55 ++++++++-----------
 .../TargetParser/RISCVISAInfoTest.cpp         | 14 ++---
 2 files changed, 30 insertions(+), 39 deletions(-)

diff --git a/llvm/lib/TargetParser/RISCVISAInfo.cpp b/llvm/lib/TargetParser/RISCVISAInfo.cpp
index 156ee0debab87be..e8d912486efc6b4 100644
--- a/llvm/lib/TargetParser/RISCVISAInfo.cpp
+++ b/llvm/lib/TargetParser/RISCVISAInfo.cpp
@@ -739,31 +739,23 @@ Error RISCVISAInfo::checkDependency() {
     return getError("'f' and 'zfinx' extensions are incompatible");
 
   if (HasZvl && !HasVector)
-    return getError(
-        "'zvl*b' requires 'v' or 'zve*' extension to also be specified");
-
-  if (Exts.count("zvbb") && !HasVector)
-    return getError(
-        "'zvbb' requires 'v' or 'zve*' extension to also be specified");
-
-  if (Exts.count("zvbc") && !Exts.count("zve64x"))
-    return getError(
-        "'zvbc' requires 'v' or 'zve64*' extension to also be specified");
-
-  if (Exts.count("zvbc32e") && !Exts.count("zve32x"))
-    return getError(
-        "'zvbc32e' requires 'v' or 'zve*' extension to also be specified");
-
-  if ((Exts.count("zvkb") || Exts.count("zvkg") || Exts.count("zvkgs") ||
-       Exts.count("zvkned") || Exts.count("zvknha") || Exts.count("zvksed") ||
-       Exts.count("zvksh")) &&
-      !HasVector)
-    return getError(
-        "'zvk*' requires 'v' or 'zve*' extension to also be specified");
-
-  if (Exts.count("zvknhb") && !Exts.count("zve64x"))
-    return getError(
-        "'zvknhb' requires 'v' or 'zve64*' extension to also be specified");
+    return getError(Twine("'") + "zvl*b" +
+                    "' requires 'v' or 'zve*' extension to also be specified");
+
+  if (!HasVector)
+    for (auto Ext :
+         {"zvbb", "zvbc32e", "zvkb", "zvkg", "zvkgs", "zvkned", "zvknha", "zvksed", "zvksh"})
+      if (Exts.count(Ext))
+        return getError(
+            Twine("'") + Ext +
+            "' requires 'v' or 'zve*' extension to also be specified");
+
+  if (!Exts.count("zve64x"))
+    for (auto Ext : {"zvknhb", "zvbc"})
+      if (Exts.count(Ext))
+        return getError(
+            Twine("'") + Ext +
+            "' requires 'v' or 'zve64*' extension to also be specified");
 
   if ((HasZcmt || Exts.count("zcmp")) && HasD && (HasC || Exts.count("zcd")))
     return getError(Twine("'") + (HasZcmt ? "zcmt" : "zcmp") +
@@ -774,13 +766,12 @@ Error RISCVISAInfo::checkDependency() {
   if (XLen != 32 && Exts.count("zcf"))
     return getError("'zcf' is only supported for 'rv32'");
 
-  if (Exts.count("zacas") && !(Exts.count("a") || Exts.count("zaamo")))
-    return getError(
-        "'zacas' requires 'a' or 'zaamo' extension to also be specified");
-
-  if (Exts.count("zabha") && !(Exts.count("a") || Exts.count("zaamo")))
-    return getError(
-        "'zabha' requires 'a' or 'zaamo' extension to also be specified");
+  if (!(Exts.count("a") || Exts.count("zaamo")))
+    for (auto Ext : {"zacas", "zabha"})
+      if (Exts.count(Ext))
+        return getError(
+            Twine("'") + Ext +
+            "' requires 'a' or 'zaamo' extension to also be specified");
 
   if (Exts.count("xwchc") != 0) {
     if (XLen != 32)
diff --git a/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp b/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
index 4c5087b570fc93a..0d6a354101a345b 100644
--- a/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
+++ b/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
@@ -660,37 +660,37 @@ TEST(ParseArchString, MissingDepency) {
 
   for (StringRef Input : {"rv32i_zvkb"}) {
     EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
-              "'zvk*' requires 'v' or 'zve*' extension to also be specified");
+              "'zvkb' requires 'v' or 'zve*' extension to also be specified");
   }
 
   for (StringRef Input : {"rv32i_zvkg"}) {
     EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
-              "'zvk*' requires 'v' or 'zve*' extension to also be specified");
+              "'zvkg' requires 'v' or 'zve*' extension to also be specified");
   }
 
   for (StringRef Input : {"rv32i_zvkgs0p7"}) {
     EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
-              "'zvk*' requires 'v' or 'zve*' extension to also be specified");
+              "'zvkg' requires 'v' or 'zve*' extension to also be specified");
   }
 
   for (StringRef Input : {"rv32i_zvkned"}) {
     EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
-              "'zvk*' requires 'v' or 'zve*' extension to also be specified");
+              "'zvkned' requires 'v' or 'zve*' extension to also be specified");
   }
 
   for (StringRef Input : {"rv32i_zvknha"}) {
     EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
-              "'zvk*' requires 'v' or 'zve*' extension to also be specified");
+              "'zvknha' requires 'v' or 'zve*' extension to also be specified");
   }
 
   for (StringRef Input : {"rv32i_zvksed"}) {
     EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
-              "'zvk*' requires 'v' or 'zve*' extension to also be specified");
+              "'zvksed' requires 'v' or 'zve*' extension to also be specified");
   }
 
   for (StringRef Input : {"rv32i_zvksh"}) {
     EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
-              "'zvk*' requires 'v' or 'zve*' extension to also be specified");
+              "'zvksh' requires 'v' or 'zve*' extension to also be specified");
   }
 
   for (StringRef Input : {"rv32i_zvknhb"}) {

From 10a4f1ef9e1f30729920d2dd22d75d1002e66d0d Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Fri, 16 Aug 2024 16:06:28 -0700
Subject: [PATCH 328/441] [RISCV] Add helper functions to exploit similarity of
 some RISCVISAInfo::checkDependency() error strings. NFC

---
 llvm/lib/TargetParser/RISCVISAInfo.cpp | 33 ++++++++++++++------------
 1 file changed, 18 insertions(+), 15 deletions(-)

diff --git a/llvm/lib/TargetParser/RISCVISAInfo.cpp b/llvm/lib/TargetParser/RISCVISAInfo.cpp
index e8d912486efc6b4..ac0bf8c5cec9df3 100644
--- a/llvm/lib/TargetParser/RISCVISAInfo.cpp
+++ b/llvm/lib/TargetParser/RISCVISAInfo.cpp
@@ -721,6 +721,16 @@ RISCVISAInfo::parseArchString(StringRef Arch, bool EnableExperimentalExtension,
   return RISCVISAInfo::postProcessAndChecking(std::move(ISAInfo));
 }
 
+static Error getIncompatibleError(StringRef Ext1, StringRef Ext2) {
+  return getError("'" + Ext1 + "' and '" + Ext2 +
+                  "' extensions are incompatible");
+}
+
+static Error getExtensionRequiresError(StringRef Ext, StringRef ReqExt) {
+  return getError("'" + Ext + "' requires '" + ReqExt +
+                  "' extension to also be specified");
+}
+
 Error RISCVISAInfo::checkDependency() {
   bool HasE = Exts.count("e") != 0;
   bool HasI = Exts.count("i") != 0;
@@ -733,29 +743,24 @@ Error RISCVISAInfo::checkDependency() {
   bool HasZcmt = Exts.count("zcmt") != 0;
 
   if (HasI && HasE)
-    return getError("'I' and 'E' extensions are incompatible");
+    return getIncompatibleError("I", "E");
 
   if (HasF && HasZfinx)
-    return getError("'f' and 'zfinx' extensions are incompatible");
+    return getIncompatibleError("f", "zfinx");
 
   if (HasZvl && !HasVector)
-    return getError(Twine("'") + "zvl*b" +
-                    "' requires 'v' or 'zve*' extension to also be specified");
+    return getExtensionRequiresError("zvl*b", "v' or 'zve*");
 
   if (!HasVector)
     for (auto Ext :
          {"zvbb", "zvbc32e", "zvkb", "zvkg", "zvkgs", "zvkned", "zvknha", "zvksed", "zvksh"})
       if (Exts.count(Ext))
-        return getError(
-            Twine("'") + Ext +
-            "' requires 'v' or 'zve*' extension to also be specified");
+        return getExtensionRequiresError(Ext, "v' or 'zve*");
 
   if (!Exts.count("zve64x"))
     for (auto Ext : {"zvknhb", "zvbc"})
       if (Exts.count(Ext))
-        return getError(
-            Twine("'") + Ext +
-            "' requires 'v' or 'zve64*' extension to also be specified");
+        return getExtensionRequiresError(Ext, "v' or 'zve64*");
 
   if ((HasZcmt || Exts.count("zcmp")) && HasD && (HasC || Exts.count("zcd")))
     return getError(Twine("'") + (HasZcmt ? "zcmt" : "zcmp") +
@@ -769,19 +774,17 @@ Error RISCVISAInfo::checkDependency() {
   if (!(Exts.count("a") || Exts.count("zaamo")))
     for (auto Ext : {"zacas", "zabha"})
       if (Exts.count(Ext))
-        return getError(
-            Twine("'") + Ext +
-            "' requires 'a' or 'zaamo' extension to also be specified");
+        return getExtensionRequiresError(Ext, "a' or 'zaamo");
 
   if (Exts.count("xwchc") != 0) {
     if (XLen != 32)
       return getError("'Xwchc' is only supported for 'rv32'");
 
     if (HasD)
-      return getError("'D' and 'Xwchc' extensions are incompatible");
+      return getIncompatibleError("D", "Xwchc");
 
     if (Exts.count("zcb") != 0)
-      return getError("'Xwchc' and 'Zcb' extensions are incompatible");
+      return getIncompatibleError("Xwchc", "Zcb");
   }
 
   return Error::success();

From 371f936c456bca2b2af52f0d7fb9e7c9d1c0a1b7 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Fri, 16 Aug 2024 19:00:25 -0700
Subject: [PATCH 329/441] [RISCV] Make extension names lower case in
 RISCVISAInfo::checkDependency() error messages.

---
 llvm/lib/TargetParser/RISCVISAInfo.cpp           | 8 ++++----
 llvm/unittests/TargetParser/RISCVISAInfoTest.cpp | 6 +++---
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/llvm/lib/TargetParser/RISCVISAInfo.cpp b/llvm/lib/TargetParser/RISCVISAInfo.cpp
index ac0bf8c5cec9df3..d82501ebf8d2fb8 100644
--- a/llvm/lib/TargetParser/RISCVISAInfo.cpp
+++ b/llvm/lib/TargetParser/RISCVISAInfo.cpp
@@ -743,7 +743,7 @@ Error RISCVISAInfo::checkDependency() {
   bool HasZcmt = Exts.count("zcmt") != 0;
 
   if (HasI && HasE)
-    return getIncompatibleError("I", "E");
+    return getIncompatibleError("i", "e");
 
   if (HasF && HasZfinx)
     return getIncompatibleError("f", "zfinx");
@@ -778,13 +778,13 @@ Error RISCVISAInfo::checkDependency() {
 
   if (Exts.count("xwchc") != 0) {
     if (XLen != 32)
-      return getError("'Xwchc' is only supported for 'rv32'");
+      return getError("'xwchc' is only supported for 'rv32'");
 
     if (HasD)
-      return getIncompatibleError("D", "Xwchc");
+      return getIncompatibleError("d", "xwchc");
 
     if (Exts.count("zcb") != 0)
-      return getIncompatibleError("Xwchc", "Zcb");
+      return getIncompatibleError("xwchc", "zcb");
   }
 
   return Error::success();
diff --git a/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp b/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
index 0d6a354101a345b..d1de96477a5ca06 100644
--- a/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
+++ b/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
@@ -623,17 +623,17 @@ TEST(ParseArchString, RejectsConflictingExtensions) {
 
   for (StringRef Input : {"rv64i_xwchc"}) {
     EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
-              "'Xwchc' is only supported for 'rv32'");
+              "'xwchc' is only supported for 'rv32'");
   }
 
   for (StringRef Input : {"rv32id_xwchc"}) {
     EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
-              "'D' and 'Xwchc' extensions are incompatible");
+              "'d' and 'xwchc' extensions are incompatible");
   }
 
   for (StringRef Input : {"rv32i_zcb_xwchc"}) {
     EXPECT_EQ(toString(RISCVISAInfo::parseArchString(Input, true).takeError()),
-              "'Xwchc' and 'Zcb' extensions are incompatible");
+              "'xwchc' and 'zcb' extensions are incompatible");
   }
 }
 

From 985d64b03accbed8500a85372d716367d89b61be Mon Sep 17 00:00:00 2001
From: hev <wangrui@loongson.cn>
Date: Mon, 19 Aug 2024 15:23:05 +0800
Subject: [PATCH 330/441] [LoongArch] Merge base and offset for LSX/LASX memory
 accesses (#104452)

---
 .../LoongArch/LoongArchMergeBaseOffset.cpp    | 24 ++++++-
 .../LoongArch/can-not-realign-stack.ll        | 12 ++--
 .../CodeGen/LoongArch/lasx/build-vector.ll    | 18 ++---
 .../lasx/ir-instruction/shuffle-as-xvshuf.ll  | 15 ++---
 llvm/test/CodeGen/LoongArch/lasx/vselect.ll   |  3 +-
 .../CodeGen/LoongArch/lsx/build-vector.ll     | 18 ++---
 .../lsx/ir-instruction/shuffle-as-vshuf.ll    | 18 ++---
 llvm/test/CodeGen/LoongArch/lsx/vselect.ll    |  3 +-
 .../CodeGen/LoongArch/merge-base-offset.ll    | 66 +++++++------------
 9 files changed, 74 insertions(+), 103 deletions(-)

diff --git a/llvm/lib/Target/LoongArch/LoongArchMergeBaseOffset.cpp b/llvm/lib/Target/LoongArch/LoongArchMergeBaseOffset.cpp
index ae50b7a6f923e3a..47071d29c2cd44e 100644
--- a/llvm/lib/Target/LoongArch/LoongArchMergeBaseOffset.cpp
+++ b/llvm/lib/Target/LoongArch/LoongArchMergeBaseOffset.cpp
@@ -381,6 +381,14 @@ static unsigned getNewOpc(unsigned Op, bool isLarge) {
     return isLarge ? LoongArch::FLDX_S : LoongArch::FLD_S;
   case LoongArch::FLD_D:
     return isLarge ? LoongArch::FLDX_D : LoongArch::FLD_D;
+  case LoongArch::VLD:
+    return isLarge ? LoongArch::VLDX : LoongArch::VLD;
+  case LoongArch::XVLD:
+    return isLarge ? LoongArch::XVLDX : LoongArch::XVLD;
+  case LoongArch::VLDREPL_B:
+    return LoongArch::VLDREPL_B;
+  case LoongArch::XVLDREPL_B:
+    return LoongArch::XVLDREPL_B;
   case LoongArch::ST_B:
     return isLarge ? LoongArch::STX_B : LoongArch::ST_B;
   case LoongArch::ST_H:
@@ -395,6 +403,10 @@ static unsigned getNewOpc(unsigned Op, bool isLarge) {
     return isLarge ? LoongArch::FSTX_S : LoongArch::FST_S;
   case LoongArch::FST_D:
     return isLarge ? LoongArch::FSTX_D : LoongArch::FST_D;
+  case LoongArch::VST:
+    return isLarge ? LoongArch::VSTX : LoongArch::VST;
+  case LoongArch::XVST:
+    return isLarge ? LoongArch::XVSTX : LoongArch::XVST;
   default:
     llvm_unreachable("Unexpected opcode for replacement");
   }
@@ -444,6 +456,12 @@ bool LoongArchMergeBaseOffsetOpt::foldIntoMemoryOps(MachineInstr &Hi20,
     default:
       LLVM_DEBUG(dbgs() << "Not a load or store instruction: " << UseMI);
       return false;
+    case LoongArch::VLDREPL_B:
+    case LoongArch::XVLDREPL_B:
+      // We can't do this for large pattern.
+      if (Last)
+        return false;
+      [[fallthrough]];
     case LoongArch::LD_B:
     case LoongArch::LD_H:
     case LoongArch::LD_W:
@@ -455,6 +473,8 @@ bool LoongArchMergeBaseOffsetOpt::foldIntoMemoryOps(MachineInstr &Hi20,
     case LoongArch::LDPTR_D:
     case LoongArch::FLD_S:
     case LoongArch::FLD_D:
+    case LoongArch::VLD:
+    case LoongArch::XVLD:
     case LoongArch::ST_B:
     case LoongArch::ST_H:
     case LoongArch::ST_W:
@@ -462,7 +482,9 @@ bool LoongArchMergeBaseOffsetOpt::foldIntoMemoryOps(MachineInstr &Hi20,
     case LoongArch::STPTR_W:
     case LoongArch::STPTR_D:
     case LoongArch::FST_S:
-    case LoongArch::FST_D: {
+    case LoongArch::FST_D:
+    case LoongArch::VST:
+    case LoongArch::XVST: {
       if (UseMI.getOperand(1).isFI())
         return false;
       // Register defined by Lo should not be the value register.
diff --git a/llvm/test/CodeGen/LoongArch/can-not-realign-stack.ll b/llvm/test/CodeGen/LoongArch/can-not-realign-stack.ll
index 49155a3966a8429..6d5cf5a94293120 100644
--- a/llvm/test/CodeGen/LoongArch/can-not-realign-stack.ll
+++ b/llvm/test/CodeGen/LoongArch/can-not-realign-stack.ll
@@ -17,20 +17,16 @@ define dso_local noundef signext i32 @main() nounwind {
 ; CHECK-NEXT:    addi.d $sp, $sp, -272
 ; CHECK-NEXT:    st.d $ra, $sp, 264 # 8-byte Folded Spill
 ; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI0_0)
-; CHECK-NEXT:    addi.d $a0, $a0, %pc_lo12(.LCPI0_0)
-; CHECK-NEXT:    xvld $xr0, $a0, 0
+; CHECK-NEXT:    xvld $xr0, $a0, %pc_lo12(.LCPI0_0)
 ; CHECK-NEXT:    xvst $xr0, $sp, 96 # 32-byte Folded Spill
 ; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI0_1)
-; CHECK-NEXT:    addi.d $a0, $a0, %pc_lo12(.LCPI0_1)
-; CHECK-NEXT:    xvld $xr1, $a0, 0
+; CHECK-NEXT:    xvld $xr1, $a0, %pc_lo12(.LCPI0_1)
 ; CHECK-NEXT:    xvst $xr1, $sp, 64 # 32-byte Folded Spill
 ; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI0_2)
-; CHECK-NEXT:    addi.d $a0, $a0, %pc_lo12(.LCPI0_2)
-; CHECK-NEXT:    xvld $xr2, $a0, 0
+; CHECK-NEXT:    xvld $xr2, $a0, %pc_lo12(.LCPI0_2)
 ; CHECK-NEXT:    xvst $xr2, $sp, 32 # 32-byte Folded Spill
 ; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI0_3)
-; CHECK-NEXT:    addi.d $a0, $a0, %pc_lo12(.LCPI0_3)
-; CHECK-NEXT:    xvld $xr3, $a0, 0
+; CHECK-NEXT:    xvld $xr3, $a0, %pc_lo12(.LCPI0_3)
 ; CHECK-NEXT:    xvst $xr3, $sp, 0 # 32-byte Folded Spill
 ; CHECK-NEXT:    xvst $xr0, $sp, 136
 ; CHECK-NEXT:    xvst $xr1, $sp, 168
diff --git a/llvm/test/CodeGen/LoongArch/lasx/build-vector.ll b/llvm/test/CodeGen/LoongArch/lasx/build-vector.ll
index 3a0fd0758cb324f..b06f6523e977c5f 100644
--- a/llvm/test/CodeGen/LoongArch/lasx/build-vector.ll
+++ b/llvm/test/CodeGen/LoongArch/lasx/build-vector.ll
@@ -153,8 +153,7 @@ define void @buildvector_v32i8_const(ptr %dst) nounwind {
 ; CHECK-LABEL: buildvector_v32i8_const:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    pcalau12i $a1, %pc_hi20(.LCPI12_0)
-; CHECK-NEXT:    addi.d $a1, $a1, %pc_lo12(.LCPI12_0)
-; CHECK-NEXT:    xvld $xr0, $a1, 0
+; CHECK-NEXT:    xvld $xr0, $a1, %pc_lo12(.LCPI12_0)
 ; CHECK-NEXT:    xvst $xr0, $a0, 0
 ; CHECK-NEXT:    ret
 entry:
@@ -166,8 +165,7 @@ define void @buildvector_v16i16_const(ptr %dst) nounwind {
 ; CHECK-LABEL: buildvector_v16i16_const:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    pcalau12i $a1, %pc_hi20(.LCPI13_0)
-; CHECK-NEXT:    addi.d $a1, $a1, %pc_lo12(.LCPI13_0)
-; CHECK-NEXT:    xvld $xr0, $a1, 0
+; CHECK-NEXT:    xvld $xr0, $a1, %pc_lo12(.LCPI13_0)
 ; CHECK-NEXT:    xvst $xr0, $a0, 0
 ; CHECK-NEXT:    ret
 entry:
@@ -179,8 +177,7 @@ define void @buildvector_v8i32_const(ptr %dst) nounwind {
 ; CHECK-LABEL: buildvector_v8i32_const:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    pcalau12i $a1, %pc_hi20(.LCPI14_0)
-; CHECK-NEXT:    addi.d $a1, $a1, %pc_lo12(.LCPI14_0)
-; CHECK-NEXT:    xvld $xr0, $a1, 0
+; CHECK-NEXT:    xvld $xr0, $a1, %pc_lo12(.LCPI14_0)
 ; CHECK-NEXT:    xvst $xr0, $a0, 0
 ; CHECK-NEXT:    ret
 entry:
@@ -192,8 +189,7 @@ define void @buildvector_v4i64_const(ptr %dst) nounwind {
 ; CHECK-LABEL: buildvector_v4i64_const:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    pcalau12i $a1, %pc_hi20(.LCPI15_0)
-; CHECK-NEXT:    addi.d $a1, $a1, %pc_lo12(.LCPI15_0)
-; CHECK-NEXT:    xvld $xr0, $a1, 0
+; CHECK-NEXT:    xvld $xr0, $a1, %pc_lo12(.LCPI15_0)
 ; CHECK-NEXT:    xvst $xr0, $a0, 0
 ; CHECK-NEXT:    ret
 entry:
@@ -205,8 +201,7 @@ define void @buildvector_v2f32_const(ptr %dst) nounwind {
 ; CHECK-LABEL: buildvector_v2f32_const:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    pcalau12i $a1, %pc_hi20(.LCPI16_0)
-; CHECK-NEXT:    addi.d $a1, $a1, %pc_lo12(.LCPI16_0)
-; CHECK-NEXT:    xvld $xr0, $a1, 0
+; CHECK-NEXT:    xvld $xr0, $a1, %pc_lo12(.LCPI16_0)
 ; CHECK-NEXT:    xvst $xr0, $a0, 0
 ; CHECK-NEXT:    ret
 entry:
@@ -218,8 +213,7 @@ define void @buildvector_v4f64_const(ptr %dst) nounwind {
 ; CHECK-LABEL: buildvector_v4f64_const:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    pcalau12i $a1, %pc_hi20(.LCPI17_0)
-; CHECK-NEXT:    addi.d $a1, $a1, %pc_lo12(.LCPI17_0)
-; CHECK-NEXT:    xvld $xr0, $a1, 0
+; CHECK-NEXT:    xvld $xr0, $a1, %pc_lo12(.LCPI17_0)
 ; CHECK-NEXT:    xvst $xr0, $a0, 0
 ; CHECK-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/shuffle-as-xvshuf.ll b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/shuffle-as-xvshuf.ll
index 4cc819018f0a8d1..6a8880514871572 100644
--- a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/shuffle-as-xvshuf.ll
+++ b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/shuffle-as-xvshuf.ll
@@ -6,8 +6,7 @@ define <32 x i8> @shufflevector_v32i8(<32 x i8> %a, <32 x i8> %b) {
 ; CHECK-LABEL: shufflevector_v32i8:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI0_0)
-; CHECK-NEXT:    addi.d $a0, $a0, %pc_lo12(.LCPI0_0)
-; CHECK-NEXT:    xvld $xr2, $a0, 0
+; CHECK-NEXT:    xvld $xr2, $a0, %pc_lo12(.LCPI0_0)
 ; CHECK-NEXT:    xvshuf.b $xr0, $xr1, $xr0, $xr2
 ; CHECK-NEXT:    ret
     %c = shufflevector <32 x i8> %a, <32 x i8> %b, <32 x i32> <i32 1, i32 3, i32 5, i32 7, i32 8, i32 10, i32 12, i32 15, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39,
@@ -20,8 +19,7 @@ define <16 x i16> @shufflevector_v16i16(<16 x i16> %a, <16 x i16> %b) {
 ; CHECK-LABEL: shufflevector_v16i16:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI1_0)
-; CHECK-NEXT:    addi.d $a0, $a0, %pc_lo12(.LCPI1_0)
-; CHECK-NEXT:    xvld $xr2, $a0, 0
+; CHECK-NEXT:    xvld $xr2, $a0, %pc_lo12(.LCPI1_0)
 ; CHECK-NEXT:    xvpermi.d $xr0, $xr0, 78
 ; CHECK-NEXT:    xvpermi.d $xr1, $xr1, 78
 ; CHECK-NEXT:    xvshuf.h $xr2, $xr1, $xr0
@@ -37,8 +35,7 @@ define <8 x i32> @shufflevector_v8i32(<8 x i32> %a, <8 x i32> %b) {
 ; CHECK-LABEL: shufflevector_v8i32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI2_0)
-; CHECK-NEXT:    addi.d $a0, $a0, %pc_lo12(.LCPI2_0)
-; CHECK-NEXT:    xvld $xr2, $a0, 0
+; CHECK-NEXT:    xvld $xr2, $a0, %pc_lo12(.LCPI2_0)
 ; CHECK-NEXT:    xvpermi.d $xr0, $xr0, 68
 ; CHECK-NEXT:    xvpermi.d $xr1, $xr1, 68
 ; CHECK-NEXT:    xvshuf.w $xr2, $xr1, $xr0
@@ -53,8 +50,7 @@ define <4 x i64> @shufflevector_v4i64(<4 x i64> %a, <4 x i64> %b) {
 ; CHECK-LABEL: shufflevector_v4i64:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_0)
-; CHECK-NEXT:    addi.d $a0, $a0, %pc_lo12(.LCPI3_0)
-; CHECK-NEXT:    xvld $xr2, $a0, 0
+; CHECK-NEXT:    xvld $xr2, $a0, %pc_lo12(.LCPI3_0)
 ; CHECK-NEXT:    xvpermi.d $xr0, $xr0, 238
 ; CHECK-NEXT:    xvpermi.d $xr1, $xr1, 238
 ; CHECK-NEXT:    xvshuf.d $xr2, $xr1, $xr0
@@ -69,8 +65,7 @@ define <8 x float> @shufflevector_v8f32(<8 x float> %a, <8 x float> %b) {
 ; CHECK-LABEL: shufflevector_v8f32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI4_0)
-; CHECK-NEXT:    addi.d $a0, $a0, %pc_lo12(.LCPI4_0)
-; CHECK-NEXT:    xvld $xr2, $a0, 0
+; CHECK-NEXT:    xvld $xr2, $a0, %pc_lo12(.LCPI4_0)
 ; CHECK-NEXT:    xvshuf.w $xr2, $xr1, $xr0
 ; CHECK-NEXT:    xvori.b $xr0, $xr2, 0
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/LoongArch/lasx/vselect.ll b/llvm/test/CodeGen/LoongArch/lasx/vselect.ll
index 7b4c7ced4b5f363..17ba28afc81f5f9 100644
--- a/llvm/test/CodeGen/LoongArch/lasx/vselect.ll
+++ b/llvm/test/CodeGen/LoongArch/lasx/vselect.ll
@@ -72,8 +72,7 @@ define void @select_v4i64(ptr %res, ptr %a0, ptr %a1) nounwind {
 ; CHECK-NEXT:    xvld $xr0, $a1, 0
 ; CHECK-NEXT:    xvld $xr1, $a2, 0
 ; CHECK-NEXT:    pcalau12i $a1, %pc_hi20(.LCPI4_0)
-; CHECK-NEXT:    addi.d $a1, $a1, %pc_lo12(.LCPI4_0)
-; CHECK-NEXT:    xvld $xr2, $a1, 0
+; CHECK-NEXT:    xvld $xr2, $a1, %pc_lo12(.LCPI4_0)
 ; CHECK-NEXT:    xvbitsel.v $xr0, $xr1, $xr0, $xr2
 ; CHECK-NEXT:    xvst $xr0, $a0, 0
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/LoongArch/lsx/build-vector.ll b/llvm/test/CodeGen/LoongArch/lsx/build-vector.ll
index c04d7ca889f7ef6..ba19fe75d7570c9 100644
--- a/llvm/test/CodeGen/LoongArch/lsx/build-vector.ll
+++ b/llvm/test/CodeGen/LoongArch/lsx/build-vector.ll
@@ -153,8 +153,7 @@ define void @buildvector_v16i8_const(ptr %dst) nounwind {
 ; CHECK-LABEL: buildvector_v16i8_const:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    pcalau12i $a1, %pc_hi20(.LCPI12_0)
-; CHECK-NEXT:    addi.d $a1, $a1, %pc_lo12(.LCPI12_0)
-; CHECK-NEXT:    vld $vr0, $a1, 0
+; CHECK-NEXT:    vld $vr0, $a1, %pc_lo12(.LCPI12_0)
 ; CHECK-NEXT:    vst $vr0, $a0, 0
 ; CHECK-NEXT:    ret
 entry:
@@ -166,8 +165,7 @@ define void @buildvector_v8i16_const(ptr %dst) nounwind {
 ; CHECK-LABEL: buildvector_v8i16_const:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    pcalau12i $a1, %pc_hi20(.LCPI13_0)
-; CHECK-NEXT:    addi.d $a1, $a1, %pc_lo12(.LCPI13_0)
-; CHECK-NEXT:    vld $vr0, $a1, 0
+; CHECK-NEXT:    vld $vr0, $a1, %pc_lo12(.LCPI13_0)
 ; CHECK-NEXT:    vst $vr0, $a0, 0
 ; CHECK-NEXT:    ret
 entry:
@@ -179,8 +177,7 @@ define void @buildvector_v4i32_const(ptr %dst) nounwind {
 ; CHECK-LABEL: buildvector_v4i32_const:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    pcalau12i $a1, %pc_hi20(.LCPI14_0)
-; CHECK-NEXT:    addi.d $a1, $a1, %pc_lo12(.LCPI14_0)
-; CHECK-NEXT:    vld $vr0, $a1, 0
+; CHECK-NEXT:    vld $vr0, $a1, %pc_lo12(.LCPI14_0)
 ; CHECK-NEXT:    vst $vr0, $a0, 0
 ; CHECK-NEXT:    ret
 entry:
@@ -192,8 +189,7 @@ define void @buildvector_v2i64_const(ptr %dst) nounwind {
 ; CHECK-LABEL: buildvector_v2i64_const:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    pcalau12i $a1, %pc_hi20(.LCPI15_0)
-; CHECK-NEXT:    addi.d $a1, $a1, %pc_lo12(.LCPI15_0)
-; CHECK-NEXT:    vld $vr0, $a1, 0
+; CHECK-NEXT:    vld $vr0, $a1, %pc_lo12(.LCPI15_0)
 ; CHECK-NEXT:    vst $vr0, $a0, 0
 ; CHECK-NEXT:    ret
 entry:
@@ -205,8 +201,7 @@ define void @buildvector_v2f32_const(ptr %dst) nounwind {
 ; CHECK-LABEL: buildvector_v2f32_const:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    pcalau12i $a1, %pc_hi20(.LCPI16_0)
-; CHECK-NEXT:    addi.d $a1, $a1, %pc_lo12(.LCPI16_0)
-; CHECK-NEXT:    vld $vr0, $a1, 0
+; CHECK-NEXT:    vld $vr0, $a1, %pc_lo12(.LCPI16_0)
 ; CHECK-NEXT:    vst $vr0, $a0, 0
 ; CHECK-NEXT:    ret
 entry:
@@ -218,8 +213,7 @@ define void @buildvector_v2f64_const(ptr %dst) nounwind {
 ; CHECK-LABEL: buildvector_v2f64_const:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    pcalau12i $a1, %pc_hi20(.LCPI17_0)
-; CHECK-NEXT:    addi.d $a1, $a1, %pc_lo12(.LCPI17_0)
-; CHECK-NEXT:    vld $vr0, $a1, 0
+; CHECK-NEXT:    vld $vr0, $a1, %pc_lo12(.LCPI17_0)
 ; CHECK-NEXT:    vst $vr0, $a0, 0
 ; CHECK-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/shuffle-as-vshuf.ll b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/shuffle-as-vshuf.ll
index 55800b31446b3d8..ac78a26ba43673f 100644
--- a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/shuffle-as-vshuf.ll
+++ b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/shuffle-as-vshuf.ll
@@ -5,8 +5,7 @@ define <16 x i8> @shufflevector_v16i8(<16 x i8> %a, <16 x i8> %b) {
 ; CHECK-LABEL: shufflevector_v16i8:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI0_0)
-; CHECK-NEXT:    addi.d $a0, $a0, %pc_lo12(.LCPI0_0)
-; CHECK-NEXT:    vld $vr2, $a0, 0
+; CHECK-NEXT:    vld $vr2, $a0, %pc_lo12(.LCPI0_0)
 ; CHECK-NEXT:    vshuf.b $vr0, $vr1, $vr0, $vr2
 ; CHECK-NEXT:    ret
     %c = shufflevector <16 x i8> %a, <16 x i8> %b, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 8, i32 10, i32 12, i32 15, i32 2, i32 4, i32 6, i32 8, i32 25, i32 30, i32 31, i32 31>
@@ -18,8 +17,7 @@ define <8 x i16> @shufflevector_v8i16(<8 x i16> %a, <8 x i16> %b) {
 ; CHECK-LABEL: shufflevector_v8i16:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI1_0)
-; CHECK-NEXT:    addi.d $a0, $a0, %pc_lo12(.LCPI1_0)
-; CHECK-NEXT:    vld $vr2, $a0, 0
+; CHECK-NEXT:    vld $vr2, $a0, %pc_lo12(.LCPI1_0)
 ; CHECK-NEXT:    vshuf.h $vr2, $vr1, $vr0
 ; CHECK-NEXT:    vori.b $vr0, $vr2, 0
 ; CHECK-NEXT:    ret
@@ -32,8 +30,7 @@ define <4 x i32> @shufflevector_v4i32(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-LABEL: shufflevector_v4i32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI2_0)
-; CHECK-NEXT:    addi.d $a0, $a0, %pc_lo12(.LCPI2_0)
-; CHECK-NEXT:    vld $vr2, $a0, 0
+; CHECK-NEXT:    vld $vr2, $a0, %pc_lo12(.LCPI2_0)
 ; CHECK-NEXT:    vshuf.w $vr2, $vr1, $vr0
 ; CHECK-NEXT:    vori.b $vr0, $vr2, 0
 ; CHECK-NEXT:    ret
@@ -46,8 +43,7 @@ define <2 x i64> @shufflevector_v2i64(<2 x i64> %a, <2 x i64> %b) {
 ; CHECK-LABEL: shufflevector_v2i64:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_0)
-; CHECK-NEXT:    addi.d $a0, $a0, %pc_lo12(.LCPI3_0)
-; CHECK-NEXT:    vld $vr2, $a0, 0
+; CHECK-NEXT:    vld $vr2, $a0, %pc_lo12(.LCPI3_0)
 ; CHECK-NEXT:    vshuf.d $vr2, $vr1, $vr0
 ; CHECK-NEXT:    vori.b $vr0, $vr2, 0
 ; CHECK-NEXT:    ret
@@ -60,8 +56,7 @@ define <4 x float> @shufflevector_v4f32(<4 x float> %a, <4 x float> %b) {
 ; CHECK-LABEL: shufflevector_v4f32:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI4_0)
-; CHECK-NEXT:    addi.d $a0, $a0, %pc_lo12(.LCPI4_0)
-; CHECK-NEXT:    vld $vr2, $a0, 0
+; CHECK-NEXT:    vld $vr2, $a0, %pc_lo12(.LCPI4_0)
 ; CHECK-NEXT:    vshuf.w $vr2, $vr1, $vr0
 ; CHECK-NEXT:    vori.b $vr0, $vr2, 0
 ; CHECK-NEXT:    ret
@@ -74,8 +69,7 @@ define <2 x double> @shufflevector_v2f64(<2 x double> %a, <2 x double> %b) {
 ; CHECK-LABEL: shufflevector_v2f64:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI5_0)
-; CHECK-NEXT:    addi.d $a0, $a0, %pc_lo12(.LCPI5_0)
-; CHECK-NEXT:    vld $vr2, $a0, 0
+; CHECK-NEXT:    vld $vr2, $a0, %pc_lo12(.LCPI5_0)
 ; CHECK-NEXT:    vshuf.d $vr2, $vr1, $vr0
 ; CHECK-NEXT:    vori.b $vr0, $vr2, 0
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/LoongArch/lsx/vselect.ll b/llvm/test/CodeGen/LoongArch/lsx/vselect.ll
index 48ef3c14a4bf513..1c10e6c3087ad76 100644
--- a/llvm/test/CodeGen/LoongArch/lsx/vselect.ll
+++ b/llvm/test/CodeGen/LoongArch/lsx/vselect.ll
@@ -72,8 +72,7 @@ define void @select_v2i64(ptr %res, ptr %a0, ptr %a1) nounwind {
 ; CHECK-NEXT:    vld $vr0, $a1, 0
 ; CHECK-NEXT:    vld $vr1, $a2, 0
 ; CHECK-NEXT:    pcalau12i $a1, %pc_hi20(.LCPI4_0)
-; CHECK-NEXT:    addi.d $a1, $a1, %pc_lo12(.LCPI4_0)
-; CHECK-NEXT:    vld $vr2, $a1, 0
+; CHECK-NEXT:    vld $vr2, $a1, %pc_lo12(.LCPI4_0)
 ; CHECK-NEXT:    vbitsel.v $vr0, $vr1, $vr0, $vr2
 ; CHECK-NEXT:    vst $vr0, $a0, 0
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/LoongArch/merge-base-offset.ll b/llvm/test/CodeGen/LoongArch/merge-base-offset.ll
index 58a8e5d77c63fe7..1e7a79beb62c610 100644
--- a/llvm/test/CodeGen/LoongArch/merge-base-offset.ll
+++ b/llvm/test/CodeGen/LoongArch/merge-base-offset.ll
@@ -542,21 +542,17 @@ define dso_local void @copy_i32x4() nounwind {
 ; LA32-LABEL: copy_i32x4:
 ; LA32:       # %bb.0: # %entry
 ; LA32-NEXT:    pcalau12i $a0, %pc_hi20(g_i32x4_src)
-; LA32-NEXT:    addi.w $a0, $a0, %pc_lo12(g_i32x4_src)
-; LA32-NEXT:    vld $vr0, $a0, 0
+; LA32-NEXT:    vld $vr0, $a0, %pc_lo12(g_i32x4_src)
 ; LA32-NEXT:    pcalau12i $a0, %pc_hi20(g_i32x4_dst)
-; LA32-NEXT:    addi.w $a0, $a0, %pc_lo12(g_i32x4_dst)
-; LA32-NEXT:    vst $vr0, $a0, 0
+; LA32-NEXT:    vst $vr0, $a0, %pc_lo12(g_i32x4_dst)
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: copy_i32x4:
 ; LA64:       # %bb.0: # %entry
 ; LA64-NEXT:    pcalau12i $a0, %pc_hi20(g_i32x4_src)
-; LA64-NEXT:    addi.d $a0, $a0, %pc_lo12(g_i32x4_src)
-; LA64-NEXT:    vld $vr0, $a0, 0
+; LA64-NEXT:    vld $vr0, $a0, %pc_lo12(g_i32x4_src)
 ; LA64-NEXT:    pcalau12i $a0, %pc_hi20(g_i32x4_dst)
-; LA64-NEXT:    addi.d $a0, $a0, %pc_lo12(g_i32x4_dst)
-; LA64-NEXT:    vst $vr0, $a0, 0
+; LA64-NEXT:    vst $vr0, $a0, %pc_lo12(g_i32x4_dst)
 ; LA64-NEXT:    ret
 ;
 ; LA64-LARGE-LABEL: copy_i32x4:
@@ -565,14 +561,12 @@ define dso_local void @copy_i32x4() nounwind {
 ; LA64-LARGE-NEXT:    addi.d $a1, $zero, %pc_lo12(g_i32x4_src)
 ; LA64-LARGE-NEXT:    lu32i.d $a1, %pc64_lo20(g_i32x4_src)
 ; LA64-LARGE-NEXT:    lu52i.d $a1, $a1, %pc64_hi12(g_i32x4_src)
-; LA64-LARGE-NEXT:    add.d $a0, $a1, $a0
-; LA64-LARGE-NEXT:    vld $vr0, $a0, 0
+; LA64-LARGE-NEXT:    vldx $vr0, $a1, $a0
 ; LA64-LARGE-NEXT:    pcalau12i $a0, %pc_hi20(g_i32x4_dst)
 ; LA64-LARGE-NEXT:    addi.d $a1, $zero, %pc_lo12(g_i32x4_dst)
 ; LA64-LARGE-NEXT:    lu32i.d $a1, %pc64_lo20(g_i32x4_dst)
 ; LA64-LARGE-NEXT:    lu52i.d $a1, $a1, %pc64_hi12(g_i32x4_dst)
-; LA64-LARGE-NEXT:    add.d $a0, $a1, $a0
-; LA64-LARGE-NEXT:    vst $vr0, $a0, 0
+; LA64-LARGE-NEXT:    vstx $vr0, $a1, $a0
 ; LA64-LARGE-NEXT:    ret
 entry:
   %0 = load <4 x i32>, ptr @g_i32x4_src, align 16
@@ -587,21 +581,17 @@ define dso_local void @copy_i32x8() nounwind {
 ; LA32-LABEL: copy_i32x8:
 ; LA32:       # %bb.0: # %entry
 ; LA32-NEXT:    pcalau12i $a0, %pc_hi20(g_i32x4_src)
-; LA32-NEXT:    addi.w $a0, $a0, %pc_lo12(g_i32x4_src)
-; LA32-NEXT:    xvld $xr0, $a0, 0
+; LA32-NEXT:    xvld $xr0, $a0, %pc_lo12(g_i32x4_src)
 ; LA32-NEXT:    pcalau12i $a0, %pc_hi20(g_i32x4_dst)
-; LA32-NEXT:    addi.w $a0, $a0, %pc_lo12(g_i32x4_dst)
-; LA32-NEXT:    xvst $xr0, $a0, 0
+; LA32-NEXT:    xvst $xr0, $a0, %pc_lo12(g_i32x4_dst)
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: copy_i32x8:
 ; LA64:       # %bb.0: # %entry
 ; LA64-NEXT:    pcalau12i $a0, %pc_hi20(g_i32x4_src)
-; LA64-NEXT:    addi.d $a0, $a0, %pc_lo12(g_i32x4_src)
-; LA64-NEXT:    xvld $xr0, $a0, 0
+; LA64-NEXT:    xvld $xr0, $a0, %pc_lo12(g_i32x4_src)
 ; LA64-NEXT:    pcalau12i $a0, %pc_hi20(g_i32x4_dst)
-; LA64-NEXT:    addi.d $a0, $a0, %pc_lo12(g_i32x4_dst)
-; LA64-NEXT:    xvst $xr0, $a0, 0
+; LA64-NEXT:    xvst $xr0, $a0, %pc_lo12(g_i32x4_dst)
 ; LA64-NEXT:    ret
 ;
 ; LA64-LARGE-LABEL: copy_i32x8:
@@ -610,14 +600,12 @@ define dso_local void @copy_i32x8() nounwind {
 ; LA64-LARGE-NEXT:    addi.d $a1, $zero, %pc_lo12(g_i32x4_src)
 ; LA64-LARGE-NEXT:    lu32i.d $a1, %pc64_lo20(g_i32x4_src)
 ; LA64-LARGE-NEXT:    lu52i.d $a1, $a1, %pc64_hi12(g_i32x4_src)
-; LA64-LARGE-NEXT:    add.d $a0, $a1, $a0
-; LA64-LARGE-NEXT:    xvld $xr0, $a0, 0
+; LA64-LARGE-NEXT:    xvldx $xr0, $a1, $a0
 ; LA64-LARGE-NEXT:    pcalau12i $a0, %pc_hi20(g_i32x4_dst)
 ; LA64-LARGE-NEXT:    addi.d $a1, $zero, %pc_lo12(g_i32x4_dst)
 ; LA64-LARGE-NEXT:    lu32i.d $a1, %pc64_lo20(g_i32x4_dst)
 ; LA64-LARGE-NEXT:    lu52i.d $a1, $a1, %pc64_hi12(g_i32x4_dst)
-; LA64-LARGE-NEXT:    add.d $a0, $a1, $a0
-; LA64-LARGE-NEXT:    xvst $xr0, $a0, 0
+; LA64-LARGE-NEXT:    xvstx $xr0, $a1, $a0
 ; LA64-LARGE-NEXT:    ret
 entry:
   %0 = load <8 x i32>, ptr @g_i32x4_src, align 32
@@ -631,21 +619,17 @@ define void @copy_i8_to_i8x16() {
 ; LA32-LABEL: copy_i8_to_i8x16:
 ; LA32:       # %bb.0: # %entry
 ; LA32-NEXT:    pcalau12i $a0, %pc_hi20(g_i8)
-; LA32-NEXT:    addi.w $a0, $a0, %pc_lo12(g_i8)
-; LA32-NEXT:    vldrepl.b $vr0, $a0, 0
+; LA32-NEXT:    vldrepl.b $vr0, $a0, %pc_lo12(g_i8)
 ; LA32-NEXT:    pcalau12i $a0, %pc_hi20(g_i8x16)
-; LA32-NEXT:    addi.w $a0, $a0, %pc_lo12(g_i8x16)
-; LA32-NEXT:    vst $vr0, $a0, 0
+; LA32-NEXT:    vst $vr0, $a0, %pc_lo12(g_i8x16)
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: copy_i8_to_i8x16:
 ; LA64:       # %bb.0: # %entry
 ; LA64-NEXT:    pcalau12i $a0, %pc_hi20(g_i8)
-; LA64-NEXT:    addi.d $a0, $a0, %pc_lo12(g_i8)
-; LA64-NEXT:    vldrepl.b $vr0, $a0, 0
+; LA64-NEXT:    vldrepl.b $vr0, $a0, %pc_lo12(g_i8)
 ; LA64-NEXT:    pcalau12i $a0, %pc_hi20(g_i8x16)
-; LA64-NEXT:    addi.d $a0, $a0, %pc_lo12(g_i8x16)
-; LA64-NEXT:    vst $vr0, $a0, 0
+; LA64-NEXT:    vst $vr0, $a0, %pc_lo12(g_i8x16)
 ; LA64-NEXT:    ret
 ;
 ; LA64-LARGE-LABEL: copy_i8_to_i8x16:
@@ -660,8 +644,7 @@ define void @copy_i8_to_i8x16() {
 ; LA64-LARGE-NEXT:    addi.d $a1, $zero, %pc_lo12(g_i8x16)
 ; LA64-LARGE-NEXT:    lu32i.d $a1, %pc64_lo20(g_i8x16)
 ; LA64-LARGE-NEXT:    lu52i.d $a1, $a1, %pc64_hi12(g_i8x16)
-; LA64-LARGE-NEXT:    add.d $a0, $a1, $a0
-; LA64-LARGE-NEXT:    vst $vr0, $a0, 0
+; LA64-LARGE-NEXT:    vstx $vr0, $a1, $a0
 ; LA64-LARGE-NEXT:    ret
 entry:
   %0 = call <16 x i8> @llvm.loongarch.lsx.vldrepl.b(ptr @g_i8, i32 0)
@@ -675,21 +658,17 @@ define void @copy_i8_to_i8x32() {
 ; LA32-LABEL: copy_i8_to_i8x32:
 ; LA32:       # %bb.0: # %entry
 ; LA32-NEXT:    pcalau12i $a0, %pc_hi20(g_i8)
-; LA32-NEXT:    addi.w $a0, $a0, %pc_lo12(g_i8)
-; LA32-NEXT:    xvldrepl.b $xr0, $a0, 0
+; LA32-NEXT:    xvldrepl.b $xr0, $a0, %pc_lo12(g_i8)
 ; LA32-NEXT:    pcalau12i $a0, %pc_hi20(g_i8x32)
-; LA32-NEXT:    addi.w $a0, $a0, %pc_lo12(g_i8x32)
-; LA32-NEXT:    xvst $xr0, $a0, 0
+; LA32-NEXT:    xvst $xr0, $a0, %pc_lo12(g_i8x32)
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: copy_i8_to_i8x32:
 ; LA64:       # %bb.0: # %entry
 ; LA64-NEXT:    pcalau12i $a0, %pc_hi20(g_i8)
-; LA64-NEXT:    addi.d $a0, $a0, %pc_lo12(g_i8)
-; LA64-NEXT:    xvldrepl.b $xr0, $a0, 0
+; LA64-NEXT:    xvldrepl.b $xr0, $a0, %pc_lo12(g_i8)
 ; LA64-NEXT:    pcalau12i $a0, %pc_hi20(g_i8x32)
-; LA64-NEXT:    addi.d $a0, $a0, %pc_lo12(g_i8x32)
-; LA64-NEXT:    xvst $xr0, $a0, 0
+; LA64-NEXT:    xvst $xr0, $a0, %pc_lo12(g_i8x32)
 ; LA64-NEXT:    ret
 ;
 ; LA64-LARGE-LABEL: copy_i8_to_i8x32:
@@ -704,8 +683,7 @@ define void @copy_i8_to_i8x32() {
 ; LA64-LARGE-NEXT:    addi.d $a1, $zero, %pc_lo12(g_i8x32)
 ; LA64-LARGE-NEXT:    lu32i.d $a1, %pc64_lo20(g_i8x32)
 ; LA64-LARGE-NEXT:    lu52i.d $a1, $a1, %pc64_hi12(g_i8x32)
-; LA64-LARGE-NEXT:    add.d $a0, $a1, $a0
-; LA64-LARGE-NEXT:    xvst $xr0, $a0, 0
+; LA64-LARGE-NEXT:    xvstx $xr0, $a1, $a0
 ; LA64-LARGE-NEXT:    ret
 entry:
   %0 = call <32 x i8> @llvm.loongarch.lasx.xvldrepl.b(ptr @g_i8, i32 0)

From 83879f4f5311af334550c54c8279397a8aa33e7b Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Mon, 19 Aug 2024 09:55:30 +0200
Subject: [PATCH 331/441] [SimplifyCFG] Don't block sinking for allocas if no
 phi created (#104579)

SimplifyCFG sinking currently does not sink loads/stores of allocas,
because historically SROA was unable to handle the resulting IR. Since
then, SROA both learned to speculate loads/stores over selects and phis,
*and* SimplifyCFG sinking has been deferred to the end of the function
simplification pipeline, which means that SROA happens before it.

As such, I believe that this workaround should no longer be necessary.
Given how sensitive SimplifyCFG sinking seems to be, this patch takes a
very conservative step towards removing this, by allowing sinking if we
don't actually need to form a phi over the pointer argument.

This fixes https://github.com/llvm/llvm-project/issues/104567, where
sinking a store to an escaped alloca allows converting a switch into
arithmetic.
---
 llvm/lib/Transforms/Utils/SimplifyCFG.cpp     | 43 +++++++++----------
 llvm/test/CodeGen/Hexagon/block-addr.ll       |  2 +-
 .../single-constant-use-preserves-dbgloc.ll   |  2 +-
 .../SimplifyCFG/X86/sink-common-code.ll       | 18 ++++----
 .../SimplifyCFG/sink-and-convert-switch.ll    | 19 +-------
 5 files changed, 34 insertions(+), 50 deletions(-)

diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
index 37c8761ca9383d0..ebdf760bda7f1ab 100644
--- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
@@ -1994,28 +1994,6 @@ static bool canSinkInstructions(
       return false;
   }
 
-  // Because SROA can't handle speculating stores of selects, try not to sink
-  // loads, stores or lifetime markers of allocas when we'd have to create a
-  // PHI for the address operand. Also, because it is likely that loads or
-  // stores of allocas will disappear when Mem2Reg/SROA is run, don't sink
-  // them.
-  // This can cause code churn which can have unintended consequences down
-  // the line - see https://llvm.org/bugs/show_bug.cgi?id=30244.
-  // FIXME: This is a workaround for a deficiency in SROA - see
-  // https://llvm.org/bugs/show_bug.cgi?id=30188
-  if (isa<StoreInst>(I0) && any_of(Insts, [](const Instruction *I) {
-        return isa<AllocaInst>(I->getOperand(1)->stripPointerCasts());
-      }))
-    return false;
-  if (isa<LoadInst>(I0) && any_of(Insts, [](const Instruction *I) {
-        return isa<AllocaInst>(I->getOperand(0)->stripPointerCasts());
-      }))
-    return false;
-  if (isLifeTimeMarker(I0) && any_of(Insts, [](const Instruction *I) {
-        return isa<AllocaInst>(I->getOperand(1)->stripPointerCasts());
-      }))
-    return false;
-
   // For calls to be sinkable, they must all be indirect, or have same callee.
   // I.e. if we have two direct calls to different callees, we don't want to
   // turn that into an indirect call. Likewise, if we have an indirect call,
@@ -2053,6 +2031,27 @@ static bool canSinkInstructions(
       return I->getOperand(OI) == I0->getOperand(OI);
     };
     if (!all_of(Insts, SameAsI0)) {
+      // Because SROA historically couldn't handle speculating stores of
+      // selects, we try not to sink loads, stores or lifetime markers of
+      // allocas when we'd have to create a PHI for the address operand.
+      // TODO: SROA supports speculation for loads and stores now -- remove
+      // this hack?
+      if (isa<StoreInst>(I0) && OI == 1 &&
+          any_of(Insts, [](const Instruction *I) {
+            return isa<AllocaInst>(I->getOperand(1)->stripPointerCasts());
+          }))
+        return false;
+      if (isa<LoadInst>(I0) && OI == 0 &&
+          any_of(Insts, [](const Instruction *I) {
+            return isa<AllocaInst>(I->getOperand(0)->stripPointerCasts());
+          }))
+        return false;
+      if (isLifeTimeMarker(I0) && OI == 1 &&
+          any_of(Insts, [](const Instruction *I) {
+            return isa<AllocaInst>(I->getOperand(1)->stripPointerCasts());
+          }))
+        return false;
+
       if ((isa<Constant>(Op) && !replacingOperandWithVariableIsCheap(I0, OI)) ||
           !canReplaceOperandWithVariable(I0, OI))
         // We can't create a PHI from this GEP.
diff --git a/llvm/test/CodeGen/Hexagon/block-addr.ll b/llvm/test/CodeGen/Hexagon/block-addr.ll
index cbe824643aca28a..5d028bba946407d 100644
--- a/llvm/test/CodeGen/Hexagon/block-addr.ll
+++ b/llvm/test/CodeGen/Hexagon/block-addr.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon < %s | FileCheck %s
+; RUN: llc -march=hexagon -hexagon-initial-cfg-cleanup=0 < %s | FileCheck %s
 
 ; CHECK-DAG: r[[REG:[0-9]+]] = memw(r{{[0-9]+<<#[0-9]+}}+##.LJTI{{.*}})
 ; CHECK-DAG: jumpr r[[REG]]
diff --git a/llvm/test/DebugInfo/ARM/single-constant-use-preserves-dbgloc.ll b/llvm/test/DebugInfo/ARM/single-constant-use-preserves-dbgloc.ll
index bcfba68baa88f79..ca61770b5eddbd7 100644
--- a/llvm/test/DebugInfo/ARM/single-constant-use-preserves-dbgloc.ll
+++ b/llvm/test/DebugInfo/ARM/single-constant-use-preserves-dbgloc.ll
@@ -1,4 +1,4 @@
-; RUN: llc -filetype=asm -asm-verbose=0 < %s | FileCheck %s
+; RUN: llc -filetype=asm -asm-verbose=0 -arm-atomic-cfg-tidy=0 < %s | FileCheck %s
 
 ; int main()
 ; {
diff --git a/llvm/test/Transforms/SimplifyCFG/X86/sink-common-code.ll b/llvm/test/Transforms/SimplifyCFG/X86/sink-common-code.ll
index cb2bbb8e0b9317e..cd26d949836e681 100644
--- a/llvm/test/Transforms/SimplifyCFG/X86/sink-common-code.ll
+++ b/llvm/test/Transforms/SimplifyCFG/X86/sink-common-code.ll
@@ -838,15 +838,14 @@ define i32 @test_pr30188a(i1 zeroext %flag, i32 %x) {
 ; CHECK:       if.then:
 ; CHECK-NEXT:    call void @g()
 ; CHECK-NEXT:    [[ONE:%.*]] = load i32, ptr [[Y]], align 4
-; CHECK-NEXT:    [[TWO:%.*]] = add i32 [[ONE]], 2
-; CHECK-NEXT:    store i32 [[TWO]], ptr [[Y]], align 4
 ; CHECK-NEXT:    br label [[IF_END:%.*]]
 ; CHECK:       if.else:
 ; CHECK-NEXT:    [[THREE:%.*]] = load i32, ptr [[Z]], align 4
-; CHECK-NEXT:    [[FOUR:%.*]] = add i32 [[THREE]], 2
-; CHECK-NEXT:    store i32 [[FOUR]], ptr [[Y]], align 4
 ; CHECK-NEXT:    br label [[IF_END]]
 ; CHECK:       if.end:
+; CHECK-NEXT:    [[THREE_SINK:%.*]] = phi i32 [ [[THREE]], [[IF_ELSE]] ], [ [[ONE]], [[IF_THEN]] ]
+; CHECK-NEXT:    [[FOUR:%.*]] = add i32 [[THREE_SINK]], 2
+; CHECK-NEXT:    store i32 [[FOUR]], ptr [[Y]], align 4
 ; CHECK-NEXT:    ret i32 1
 ;
 entry:
@@ -914,15 +913,16 @@ define zeroext i1 @test_pr30244(i1 zeroext %flag, i1 zeroext %flag2, i32 %blksA,
 ; CHECK-NEXT:    br i1 [[FLAG:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
 ; CHECK:       if.then:
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i32 [[BLKSA:%.*]], [[NBLKS:%.*]]
-; CHECK-NEXT:    [[FROMBOOL1:%.*]] = zext i1 [[CMP]] to i8
-; CHECK-NEXT:    store i8 [[FROMBOOL1]], ptr [[P]], align 1
-; CHECK-NEXT:    br label [[IF_END:%.*]]
+; CHECK-NEXT:    br label [[IF_END_SINK_SPLIT:%.*]]
 ; CHECK:       if.else:
-; CHECK-NEXT:    br i1 [[FLAG2:%.*]], label [[IF_THEN2:%.*]], label [[IF_END]]
+; CHECK-NEXT:    br i1 [[FLAG2:%.*]], label [[IF_THEN2:%.*]], label [[IF_END:%.*]]
 ; CHECK:       if.then2:
 ; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[NBLKS]], [[BLKSB:%.*]]
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp ule i32 [[ADD]], [[BLKSA]]
-; CHECK-NEXT:    [[FROMBOOL3:%.*]] = zext i1 [[CMP2]] to i8
+; CHECK-NEXT:    br label [[IF_END_SINK_SPLIT]]
+; CHECK:       if.end.sink.split:
+; CHECK-NEXT:    [[CMP2_SINK:%.*]] = phi i1 [ [[CMP2]], [[IF_THEN2]] ], [ [[CMP]], [[IF_THEN]] ]
+; CHECK-NEXT:    [[FROMBOOL3:%.*]] = zext i1 [[CMP2_SINK]] to i8
 ; CHECK-NEXT:    store i8 [[FROMBOOL3]], ptr [[P]], align 1
 ; CHECK-NEXT:    br label [[IF_END]]
 ; CHECK:       if.end:
diff --git a/llvm/test/Transforms/SimplifyCFG/sink-and-convert-switch.ll b/llvm/test/Transforms/SimplifyCFG/sink-and-convert-switch.ll
index 4944554f4b49ec1..4c93837f1422a9d 100644
--- a/llvm/test/Transforms/SimplifyCFG/sink-and-convert-switch.ll
+++ b/llvm/test/Transforms/SimplifyCFG/sink-and-convert-switch.ll
@@ -9,23 +9,8 @@ define void @pr104567(i8 %x, ptr %f) {
 ; CHECK-NEXT:  [[START:.*:]]
 ; CHECK-NEXT:    [[Y:%.*]] = alloca [1 x i8], align 1
 ; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 1, ptr nonnull [[Y]])
-; CHECK-NEXT:    switch i8 [[X]], label %[[DEFAULT_UNREACHABLE:.*]] [
-; CHECK-NEXT:      i8 0, label %[[BB4:.*]]
-; CHECK-NEXT:      i8 1, label %[[BB3:.*]]
-; CHECK-NEXT:      i8 2, label %[[BB2:.*]]
-; CHECK-NEXT:    ]
-; CHECK:       [[DEFAULT_UNREACHABLE]]:
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB4]]:
-; CHECK-NEXT:    store i8 4, ptr [[Y]], align 1
-; CHECK-NEXT:    br label %[[BB5:.*]]
-; CHECK:       [[BB3]]:
-; CHECK-NEXT:    store i8 5, ptr [[Y]], align 1
-; CHECK-NEXT:    br label %[[BB5]]
-; CHECK:       [[BB2]]:
-; CHECK-NEXT:    store i8 6, ptr [[Y]], align 1
-; CHECK-NEXT:    br label %[[BB5]]
-; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[SWITCH_OFFSET:%.*]] = add nsw i8 [[X]], 4
+; CHECK-NEXT:    store i8 [[SWITCH_OFFSET]], ptr [[Y]], align 1
 ; CHECK-NEXT:    call void [[F]](ptr [[Y]])
 ; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 1, ptr nonnull [[Y]])
 ; CHECK-NEXT:    ret void

From 5795f9e27390fe5c322853476b3c9ba8376e8541 Mon Sep 17 00:00:00 2001
From: Matthias Springer <me@m-sp.org>
Date: Mon, 19 Aug 2024 10:23:56 +0200
Subject: [PATCH 332/441] [mlir][docs] Update Bytecode documentation (#99854)

There were some discrepancies between the dialect section documentation
and the implementation.
---
 mlir/docs/BytecodeFormat.md | 48 +++++++++++++++++++++----------------
 1 file changed, 27 insertions(+), 21 deletions(-)

diff --git a/mlir/docs/BytecodeFormat.md b/mlir/docs/BytecodeFormat.md
index 869b2d8f0a00b66..ebc94c9f0d8ba6f 100644
--- a/mlir/docs/BytecodeFormat.md
+++ b/mlir/docs/BytecodeFormat.md
@@ -1,10 +1,10 @@
 # MLIR Bytecode Format
 
-This documents describes the MLIR bytecode format and its encoding.
+This document describes the MLIR bytecode format and its encoding.
 This format is versioned and stable: we don't plan to ever break
-compatibility, that is a dialect should be able to deserialize and
-older bytecode. Similarly, we support back-deployment we an older
-version of the format can be targetted.
+compatibility, that is a dialect should be able to deserialize any
+older bytecode. Similarly, we support back-deployment so that an
+older version of the format can be targetted.
 
 That said, it is important to realize that the promises of the
 bytecode format are made assuming immutable dialects: the format
@@ -19,7 +19,7 @@ information while decoding the input IR, and gives an opportunity
 to each dialect for which a version is present to perform IR
 upgrades post-parsing through the `upgradeFromVersion` method.
 There is no restriction on what kind of information a dialect
-is allowed to encode to model its versioning
+is allowed to encode to model its versioning.
 
 [TOC]
 
@@ -172,16 +172,13 @@ dialects that were also referenced.
 ```
 dialect_section {
   numDialects: varint,
-  dialectNames: varint[],
-  numTotalOpNames: varint,
-  opNames: op_name_group[]
+  dialectNames: dialect_name_group[],
+  opNames: dialect_ops_group[]  // ops grouped by dialect
 }
 
-op_name_group {
-  dialect: varint // (dialectID << 1) | (hasVersion),
-  version : dialect_version_section
-  numOpNames: varint,
-  opNames: varint[]
+dialect_name_group {
+  nameAndIsVersioned: varint  // (dialectID << 1) | (hasVersion),
+  version: dialect_version_section  // only if versioned
 }
 
 dialect_version_section {
@@ -189,6 +186,15 @@ dialect_version_section {
   version: byte[]
 }
 
+dialect_ops_group {
+  dialect: varint,
+  numOpNames: varint,
+  opNames: op_name_group[]
+}
+
+op_name_group {
+  nameAndIsRegistered: varint  // (nameID << 1) | (isRegisteredOp)
+}
 ```
 
 Dialects are encoded as a `varint` containing the index to the name string
@@ -196,7 +202,7 @@ within the string section, plus a flag indicating whether the dialect is
 versioned. Operation names are encoded in groups by dialect, with each group
 containing the dialect, the number of operation names, and the array of indexes
 to each name within the string section. The version is encoded as a nested
-section.
+section for each dialect.
 
 ### Attribute/Type Sections
 
@@ -249,9 +255,9 @@ its assembly format, or via a custom dialect defined encoding.
 
 In the case where a dialect does not define a method for encoding the attribute
 or type, the textual assembly format of that attribute or type is used as a
-fallback. For example, a type of `!bytecode.type` would be encoded as the null
-terminated string "!bytecode.type". This ensures that every attribute and type
-may be encoded, even if the owning dialect has not yet opted in to a more
+fallback. For example, a type `!bytecode.type<42>` would be encoded as the null
+terminated string "!bytecode.type<42>". This ensures that every attribute and
+type can be encoded, even if the owning dialect has not yet opted in to a more
 efficient serialization.
 
 TODO: We shouldn't redundantly encode the dialect name here, we should use a
@@ -259,9 +265,9 @@ reference to the parent dialect instead.
 
 ##### Dialect Defined Encoding
 
-In addition to the assembly format fallback, dialects may also provide a custom
-encoding for their attributes and types. Custom encodings are very beneficial in
-that they are significantly smaller and faster to read and write.
+As an alternative to the assembly format fallback, dialects may also provide a
+custom encoding for their attributes and types. Custom encodings are very
+beneficial in that they are significantly smaller and faster to read and write.
 
 Dialects can opt-in to providing custom encodings by implementing the
 `BytecodeDialectInterface`. This interface provides hooks, namely
@@ -377,7 +383,7 @@ uselist {
 
 The encoding of an operation is important because this is generally the most
 commonly appearing structure in the bytecode. A single encoding is used for
-every type of operation. Given this prevelance, many of the fields of an
+every type of operation. Given this prevalence, many of the fields of an
 operation are optional. The `encodingMask` field is a bitmask which indicates
 which of the components of the operation are present.
 

From cd60d10a10732154d95892219f70f0784f5a2249 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Mon, 19 Aug 2024 09:58:46 +0100
Subject: [PATCH 333/441] [VPlan] Move some LoopVectorizationPlanner helpers to
 VPlan.cpp (NFC).

Members not requiring access to LoopVectorizationLegality or
LoopVectorizationCostModel can safely be moved out of the very large
LoopVectorization.cpp and are more accurately placed in VPlan.cpp
---
 .../Transforms/Vectorize/LoopVectorize.cpp    | 58 ------------------
 llvm/lib/Transforms/Vectorize/VPlan.cpp       | 59 +++++++++++++++++++
 2 files changed, 59 insertions(+), 58 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 55c0ba3dd8f9bc1..e4307bf32804f7b 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -380,10 +380,6 @@ cl::opt<bool> llvm::EnableLoopVectorization(
     "vectorize-loops", cl::init(true), cl::Hidden,
     cl::desc("Run the Loop vectorization passes"));
 
-static cl::opt<bool> PrintVPlansInDotFormat(
-    "vplan-print-in-dot-format", cl::Hidden,
-    cl::desc("Use dot format instead of plain text when dumping VPlans"));
-
 static cl::opt<cl::boolOrDefault> ForceSafeDivisor(
     "force-widen-divrem-via-safe-divisor", cl::Hidden,
     cl::desc(
@@ -7302,19 +7298,6 @@ ElementCount LoopVectorizationPlanner::getBestVF() {
   return BestFactor.Width;
 }
 
-VPlan &LoopVectorizationPlanner::getBestPlanFor(ElementCount VF) const {
-  assert(count_if(VPlans,
-                  [VF](const VPlanPtr &Plan) { return Plan->hasVF(VF); }) ==
-             1 &&
-         "Best VF has not a single VPlan.");
-
-  for (const VPlanPtr &Plan : VPlans) {
-    if (Plan->hasVF(VF))
-      return *Plan.get();
-  }
-  llvm_unreachable("No plan found!");
-}
-
 static void AddRuntimeUnrollDisableMetaData(Loop *L) {
   SmallVector<Metadata *, 4> MDs;
   // Reserve first location for self reference to the LoopID metadata node.
@@ -7559,16 +7542,6 @@ LoopVectorizationPlanner::executePlan(
   return {State.ExpandedSCEVs, ReductionResumeValues};
 }
 
-#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
-void LoopVectorizationPlanner::printPlans(raw_ostream &O) {
-  for (const auto &Plan : VPlans)
-    if (PrintVPlansInDotFormat)
-      Plan->printDOT(O);
-    else
-      Plan->print(O);
-}
-#endif
-
 //===--------------------------------------------------------------------===//
 // EpilogueVectorizerMainLoop
 //===--------------------------------------------------------------------===//
@@ -7858,37 +7831,6 @@ void EpilogueVectorizerEpilogueLoop::printDebugTracesAtEnd() {
   });
 }
 
-bool LoopVectorizationPlanner::getDecisionAndClampRange(
-    const std::function<bool(ElementCount)> &Predicate, VFRange &Range) {
-  assert(!Range.isEmpty() && "Trying to test an empty VF range.");
-  bool PredicateAtRangeStart = Predicate(Range.Start);
-
-  for (ElementCount TmpVF : VFRange(Range.Start * 2, Range.End))
-    if (Predicate(TmpVF) != PredicateAtRangeStart) {
-      Range.End = TmpVF;
-      break;
-    }
-
-  return PredicateAtRangeStart;
-}
-
-/// Build VPlans for the full range of feasible VF's = {\p MinVF, 2 * \p MinVF,
-/// 4 * \p MinVF, ..., \p MaxVF} by repeatedly building a VPlan for a sub-range
-/// of VF's starting at a given VF and extending it as much as possible. Each
-/// vectorization decision can potentially shorten this sub-range during
-/// buildVPlan().
-void LoopVectorizationPlanner::buildVPlans(ElementCount MinVF,
-                                           ElementCount MaxVF) {
-  auto MaxVFTimes2 = MaxVF * 2;
-  for (ElementCount VF = MinVF; ElementCount::isKnownLT(VF, MaxVFTimes2);) {
-    VFRange SubRange = {VF, MaxVFTimes2};
-    auto Plan = buildVPlan(SubRange);
-    VPlanTransforms::optimize(*Plan, *PSE.getSE());
-    VPlans.push_back(std::move(Plan));
-    VF = SubRange.End;
-  }
-}
-
 iterator_range<mapped_iterator<Use *, std::function<VPValue *(Value *)>>>
 VPRecipeBuilder::mapToVPValues(User::op_range Operands) {
   std::function<VPValue *(Value *)> Fn = [this](Value *Op) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 2a9e4e12190cdc2..fe5edcedff54abd 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -21,6 +21,7 @@
 #include "VPlanCFG.h"
 #include "VPlanDominatorTree.h"
 #include "VPlanPatternMatch.h"
+#include "VPlanTransforms.h"
 #include "llvm/ADT/PostOrderIterator.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallVector.h"
@@ -55,6 +56,10 @@ namespace llvm {
 extern cl::opt<bool> EnableVPlanNativePath;
 }
 
+static cl::opt<bool> PrintVPlansInDotFormat(
+    "vplan-print-in-dot-format", cl::Hidden,
+    cl::desc("Use dot format instead of plain text when dumping VPlans"));
+
 #define DEBUG_TYPE "vplan"
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
@@ -1643,3 +1648,57 @@ bool vputils::isHeaderMask(const VPValue *V, VPlan &Plan) {
   return match(V, m_Binary<Instruction::ICmp>(m_VPValue(A), m_VPValue(B))) &&
          IsWideCanonicalIV(A) && B == Plan.getOrCreateBackedgeTakenCount();
 }
+
+bool LoopVectorizationPlanner::getDecisionAndClampRange(
+    const std::function<bool(ElementCount)> &Predicate, VFRange &Range) {
+  assert(!Range.isEmpty() && "Trying to test an empty VF range.");
+  bool PredicateAtRangeStart = Predicate(Range.Start);
+
+  for (ElementCount TmpVF : VFRange(Range.Start * 2, Range.End))
+    if (Predicate(TmpVF) != PredicateAtRangeStart) {
+      Range.End = TmpVF;
+      break;
+    }
+
+  return PredicateAtRangeStart;
+}
+
+/// Build VPlans for the full range of feasible VF's = {\p MinVF, 2 * \p MinVF,
+/// 4 * \p MinVF, ..., \p MaxVF} by repeatedly building a VPlan for a sub-range
+/// of VF's starting at a given VF and extending it as much as possible. Each
+/// vectorization decision can potentially shorten this sub-range during
+/// buildVPlan().
+void LoopVectorizationPlanner::buildVPlans(ElementCount MinVF,
+                                           ElementCount MaxVF) {
+  auto MaxVFTimes2 = MaxVF * 2;
+  for (ElementCount VF = MinVF; ElementCount::isKnownLT(VF, MaxVFTimes2);) {
+    VFRange SubRange = {VF, MaxVFTimes2};
+    auto Plan = buildVPlan(SubRange);
+    VPlanTransforms::optimize(*Plan, *PSE.getSE());
+    VPlans.push_back(std::move(Plan));
+    VF = SubRange.End;
+  }
+}
+
+VPlan &LoopVectorizationPlanner::getBestPlanFor(ElementCount VF) const {
+  assert(count_if(VPlans,
+                  [VF](const VPlanPtr &Plan) { return Plan->hasVF(VF); }) ==
+             1 &&
+         "Best VF has not a single VPlan.");
+
+  for (const VPlanPtr &Plan : VPlans) {
+    if (Plan->hasVF(VF))
+      return *Plan.get();
+  }
+  llvm_unreachable("No plan found!");
+}
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+void LoopVectorizationPlanner::printPlans(raw_ostream &O) {
+  for (const auto &Plan : VPlans)
+    if (PrintVPlansInDotFormat)
+      Plan->printDOT(O);
+    else
+      Plan->print(O);
+}
+#endif

From 5ab65a6c1c38eef2a096a6b9231281fb4876c387 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Tue, 13 Aug 2024 09:54:43 +0100
Subject: [PATCH 334/441] [X86] VPERM2*128 instructions aren't microcoded on
 znver2

This appears to be a copy+paste error from znver1 (which isn't really microcoded either - but it is rather complex!).

Confirmed with Agner + uops.info.
---
 llvm/lib/Target/X86/X86ScheduleZnver2.td          | 15 ++++++++++++---
 .../tools/llvm-mca/X86/Znver2/resources-avx1.s    | 10 +++++-----
 .../tools/llvm-mca/X86/Znver2/resources-avx2.s    | 10 +++++-----
 3 files changed, 22 insertions(+), 13 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ScheduleZnver2.td b/llvm/lib/Target/X86/X86ScheduleZnver2.td
index c0775847798d27d..c3a0f2684a29898 100644
--- a/llvm/lib/Target/X86/X86ScheduleZnver2.td
+++ b/llvm/lib/Target/X86/X86ScheduleZnver2.td
@@ -994,9 +994,18 @@ def : InstRW<[Zn2WritePCMPGTQYm], (instrs VPCMPGTQYrm)>;
 //-- Move instructions --//
 
 // VPERM2F128 / VPERM2I128.
-def : InstRW<[WriteMicrocoded], (instrs VPERM2F128rr,
+def Zn2WriteVPERM2r : SchedWriteRes<[Zn2FPU2]> {
+  let NumMicroOps = 1;
+  let Latency = 3;
+}
+def : InstRW<[Zn2WriteVPERM2r], (instrs VPERM2F128rr,
                                         VPERM2I128rr)>;
-def : InstRW<[WriteMicrocoded], (instrs VPERM2F128rm,
+
+def Zn2WriteVPERM2m : SchedWriteRes<[Zn2AGU, Zn2FPU2]> {
+  let NumMicroOps = 1;
+  let Latency = 8;
+}
+def : InstRW<[Zn2WriteVPERM2m], (instrs VPERM2F128rm,
                                         VPERM2I128rm)>;
 
 def Zn2WriteBROADCAST : SchedWriteRes<[Zn2AGU, Zn2FPU13]> {
@@ -1011,7 +1020,7 @@ def : InstRW<[Zn2WriteBROADCAST], (instrs VBROADCASTF128rm,
 // r32,x,i.
 def Zn2WriteEXTRACTPSr : SchedWriteRes<[Zn2FPU12, Zn2FPU2]> {
   let Latency = 2;
-  let NumMicroOps = 2; 
+  let NumMicroOps = 2;
   let ReleaseAtCycles = [1, 2];
 }
 def : InstRW<[Zn2WriteEXTRACTPSr], (instregex "(V?)EXTRACTPSrr")>;
diff --git a/llvm/test/tools/llvm-mca/X86/Znver2/resources-avx1.s b/llvm/test/tools/llvm-mca/X86/Znver2/resources-avx1.s
index 37e2cf4500c38a5..96ae0575969671d 100644
--- a/llvm/test/tools/llvm-mca/X86/Znver2/resources-avx1.s
+++ b/llvm/test/tools/llvm-mca/X86/Znver2/resources-avx1.s
@@ -1429,8 +1429,8 @@ vzeroupper
 # CHECK-NEXT:  1      100   0.25    *                   vpcmpistri	$1, (%rax), %xmm2
 # CHECK-NEXT:  1      100   0.25                        vpcmpistrm	$1, %xmm0, %xmm2
 # CHECK-NEXT:  1      100   0.25    *                   vpcmpistrm	$1, (%rax), %xmm2
-# CHECK-NEXT:  1      100   0.25                        vperm2f128	$1, %ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  1      100   0.25    *                   vperm2f128	$1, (%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      3     1.00                        vperm2f128	$1, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     1.00    *                   vperm2f128	$1, (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  1      1     0.50                        vpermilpd	$1, %xmm0, %xmm2
 # CHECK-NEXT:  1      8     0.50    *                   vpermilpd	$1, (%rax), %xmm2
 # CHECK-NEXT:  1      3     0.50                        vpermilpd	%xmm0, %xmm1, %xmm2
@@ -1739,7 +1739,7 @@ vzeroupper
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]
-# CHECK-NEXT: 117.00 117.00 117.00 0.25   0.25   0.25   0.25    -     126.58 192.58 196.75 305.08  -
+# CHECK-NEXT: 117.33 117.33 117.33 0.25   0.25   0.25   0.25    -     126.58 192.58 198.75 305.08  -
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   Instructions:
@@ -2142,8 +2142,8 @@ vzeroupper
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     vpcmpistri	$1, (%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     vpcmpistrm	$1, %xmm0, %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     vpcmpistrm	$1, (%rax), %xmm2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     vperm2f128	$1, %ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     vperm2f128	$1, (%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -     vperm2f128	$1, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -     1.00    -      -     vperm2f128	$1, (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpermilpd	$1, %xmm0, %xmm2
 # CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50    -      -     vpermilpd	$1, (%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpermilpd	%xmm0, %xmm1, %xmm2
diff --git a/llvm/test/tools/llvm-mca/X86/Znver2/resources-avx2.s b/llvm/test/tools/llvm-mca/X86/Znver2/resources-avx2.s
index 33d18b0fa0fb551..7be45be2c3578e7 100644
--- a/llvm/test/tools/llvm-mca/X86/Znver2/resources-avx2.s
+++ b/llvm/test/tools/llvm-mca/X86/Znver2/resources-avx2.s
@@ -558,8 +558,8 @@ vpxor           (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  1      8     0.50    *                   vpcmpgtq	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  1      1     0.33                        vpcmpgtw	%ymm0, %ymm1, %ymm2
 # CHECK-NEXT:  1      8     0.33    *                   vpcmpgtw	(%rax), %ymm1, %ymm2
-# CHECK-NEXT:  1      100   0.25                        vperm2i128	$1, %ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  1      100   0.25    *                   vperm2i128	$1, (%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      3     1.00                        vperm2i128	$1, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      8     1.00    *                   vperm2i128	$1, (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  1      2     0.50                        vpermd	%ymm0, %ymm1, %ymm2
 # CHECK-NEXT:  1      9     0.50    *                   vpermd	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  1      2     0.50                        vpermpd	$1, %ymm0, %ymm2
@@ -779,7 +779,7 @@ vpxor           (%rax), %ymm1, %ymm2
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]
-# CHECK-NEXT: 44.33  44.33  44.33   -      -      -      -      -     63.17  125.67 71.00  34.17   -
+# CHECK-NEXT: 44.67  44.67  44.67   -      -      -      -      -     63.17  125.67 73.00  34.17   -
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   Instructions:
@@ -881,8 +881,8 @@ vpxor           (%rax), %ymm1, %ymm2
 # CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -     0.50    -      -     0.50    -     vpcmpgtq	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.33   0.33    -     0.33    -     vpcmpgtw	%ymm0, %ymm1, %ymm2
 # CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -     0.33   0.33    -     0.33    -     vpcmpgtw	(%rax), %ymm1, %ymm2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     vperm2i128	$1, %ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     vperm2i128	$1, (%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -     vperm2i128	$1, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -     1.00    -      -     vperm2i128	$1, (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpermd	%ymm0, %ymm1, %ymm2
 # CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -     0.50   0.50    -      -     vpermd	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpermpd	$1, %ymm0, %ymm2

From 9e3e8b5715d01fc7ac6b0bdcd1870f3823f3ab30 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Tue, 13 Aug 2024 17:13:29 +0100
Subject: [PATCH 335/441] [X86] VPERM2*128 instructions aren't microcoded on
 znver1

AMD refer to them as microcoded, but not in the same way as LLVM - the uop count and pipe usage is high but predictable

Confirmed with Agner + uops.info.
---
 llvm/lib/Target/X86/X86ScheduleZnver1.td      | 19 +++++++++++++++----
 .../llvm-mca/X86/Znver1/resources-avx1.s      | 10 +++++-----
 .../llvm-mca/X86/Znver1/resources-avx2.s      | 10 +++++-----
 3 files changed, 25 insertions(+), 14 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ScheduleZnver1.td b/llvm/lib/Target/X86/X86ScheduleZnver1.td
index 7ee9eadf8439085..a044ddc30013295 100644
--- a/llvm/lib/Target/X86/X86ScheduleZnver1.td
+++ b/llvm/lib/Target/X86/X86ScheduleZnver1.td
@@ -986,10 +986,21 @@ def : InstRW<[ZnWritePCMPGTQYm], (instrs VPCMPGTQYrm)>;
 //-- Move instructions --//
 
 // VPERM2F128 / VPERM2I128.
-def : InstRW<[WriteMicrocoded], (instrs VPERM2F128rr,
-                                        VPERM2I128rr)>;
-def : InstRW<[WriteMicrocoded], (instrs VPERM2F128rm,
-                                        VPERM2I128rm)>;
+def ZnWriteVPERM2r : SchedWriteRes<[ZnFPU0, ZnFPU12]> {
+  let NumMicroOps = 8;
+  let Latency = 3;
+  let ReleaseAtCycles = [3,3];
+}
+def : InstRW<[ZnWriteVPERM2r], (instrs VPERM2F128rr,
+                                       VPERM2I128rr)>;
+
+def ZnWriteVPERM2m : SchedWriteRes<[ZnAGU, ZnFPU0, ZnFPU12]> {
+  let NumMicroOps = 12;
+  let Latency = 8;
+  let ReleaseAtCycles = [1,3,3];
+}
+def : InstRW<[ZnWriteVPERM2m], (instrs VPERM2F128rm,
+                                       VPERM2I128rm)>;
 
 def ZnWriteBROADCAST : SchedWriteRes<[ZnAGU, ZnFPU13]> {
   let NumMicroOps = 2;
diff --git a/llvm/test/tools/llvm-mca/X86/Znver1/resources-avx1.s b/llvm/test/tools/llvm-mca/X86/Znver1/resources-avx1.s
index 3914f865089247f..00e69d8960565fe 100644
--- a/llvm/test/tools/llvm-mca/X86/Znver1/resources-avx1.s
+++ b/llvm/test/tools/llvm-mca/X86/Znver1/resources-avx1.s
@@ -1429,8 +1429,8 @@ vzeroupper
 # CHECK-NEXT:  1      100   0.25    *                   vpcmpistri	$1, (%rax), %xmm2
 # CHECK-NEXT:  1      100   0.25                        vpcmpistrm	$1, %xmm0, %xmm2
 # CHECK-NEXT:  1      100   0.25    *                   vpcmpistrm	$1, (%rax), %xmm2
-# CHECK-NEXT:  1      100   0.25                        vperm2f128	$1, %ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  1      100   0.25    *                   vperm2f128	$1, (%rax), %ymm1, %ymm2
+# CHECK-NEXT:  8      3     3.00                        vperm2f128	$1, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  12     8     3.00    *                   vperm2f128	$1, (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  1      1     0.50                        vpermilpd	$1, %xmm0, %xmm2
 # CHECK-NEXT:  1      8     0.50    *                   vpermilpd	$1, (%rax), %xmm2
 # CHECK-NEXT:  1      1     0.50                        vpermilpd	%xmm0, %xmm1, %xmm2
@@ -1738,7 +1738,7 @@ vzeroupper
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]
-# CHECK-NEXT: 175.00 175.00  -      -      -      -      -     144.25 227.25 223.75 315.75  -
+# CHECK-NEXT: 175.50 175.50  -      -      -      -      -     150.25 230.25 226.75 315.75  -
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   Instructions:
@@ -2141,8 +2141,8 @@ vzeroupper
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     vpcmpistri	$1, (%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     vpcmpistrm	$1, %xmm0, %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     vpcmpistrm	$1, (%rax), %xmm2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     vperm2f128	$1, %ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     vperm2f128	$1, (%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -     3.00   1.50   1.50    -      -     vperm2f128	$1, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -     3.00   1.50   1.50    -      -     vperm2f128	$1, (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpermilpd	$1, %xmm0, %xmm2
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     vpermilpd	$1, (%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -     vpermilpd	%xmm0, %xmm1, %xmm2
diff --git a/llvm/test/tools/llvm-mca/X86/Znver1/resources-avx2.s b/llvm/test/tools/llvm-mca/X86/Znver1/resources-avx2.s
index 93b601e9285f162..a3ebe63e7189868 100644
--- a/llvm/test/tools/llvm-mca/X86/Znver1/resources-avx2.s
+++ b/llvm/test/tools/llvm-mca/X86/Znver1/resources-avx2.s
@@ -558,8 +558,8 @@ vpxor           (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  2      8     1.00    *                   vpcmpgtq	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  2      1     0.67                        vpcmpgtw	%ymm0, %ymm1, %ymm2
 # CHECK-NEXT:  2      8     0.67    *                   vpcmpgtw	(%rax), %ymm1, %ymm2
-# CHECK-NEXT:  1      100   0.25                        vperm2i128	$1, %ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  1      100   0.25    *                   vperm2i128	$1, (%rax), %ymm1, %ymm2
+# CHECK-NEXT:  8      3     3.00                        vperm2i128	$1, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  12     8     3.00    *                   vperm2i128	$1, (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  2      2     1.00                        vpermd	%ymm0, %ymm1, %ymm2
 # CHECK-NEXT:  2      9     1.00    *                   vpermd	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  2      2     1.00                        vpermpd	$1, %ymm0, %ymm2
@@ -778,7 +778,7 @@ vpxor           (%rax), %ymm1, %ymm2
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]
-# CHECK-NEXT: 66.50  66.50   -      -      -      -      -     120.17 239.17 158.00 66.67   -
+# CHECK-NEXT: 67.00  67.00   -      -      -      -      -     126.17 242.17 161.00 66.67   -
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   Instructions:
@@ -880,8 +880,8 @@ vpxor           (%rax), %ymm1, %ymm2
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -     1.00    -      -     1.00    -     vpcmpgtq	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.67   0.67    -     0.67    -     vpcmpgtw	%ymm0, %ymm1, %ymm2
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -     0.67   0.67    -     0.67    -     vpcmpgtw	(%rax), %ymm1, %ymm2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     vperm2i128	$1, %ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     vperm2i128	$1, (%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -     3.00   1.50   1.50    -      -     vperm2i128	$1, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -     3.00   1.50   1.50    -      -     vperm2i128	$1, (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -     vpermd	%ymm0, %ymm1, %ymm2
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -     1.00   1.00    -      -     vpermd	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -     vpermpd	$1, %ymm0, %ymm2

From 8b1916b7666ae29c3ca2a196f5d1062e2d51f10f Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Mon, 19 Aug 2024 10:05:04 +0100
Subject: [PATCH 336/441] [X86] Cleanup lowerShuffleWithUNPCK/PACK signatures
 to match (most) other lowerShuffle* methods. NFC.

---
 llvm/lib/Target/X86/X86ISelLowering.cpp | 89 +++++++++++++------------
 1 file changed, 45 insertions(+), 44 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index ca4e0a31e932bf0..20d190bd4aff2f9 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -10005,8 +10005,8 @@ static bool matchShuffleWithUNPCK(MVT VT, SDValue &V1, SDValue &V2,
 
 // X86 has dedicated unpack instructions that can handle specific blend
 // operations: UNPCKH and UNPCKL.
-static SDValue lowerShuffleWithUNPCK(const SDLoc &DL, MVT VT,
-                                     ArrayRef<int> Mask, SDValue V1, SDValue V2,
+static SDValue lowerShuffleWithUNPCK(const SDLoc &DL, MVT VT, SDValue V1,
+                                     SDValue V2, ArrayRef<int> Mask,
                                      SelectionDAG &DAG) {
   SmallVector<int, 8> Unpckl;
   createUnpackShuffleMask(VT, Unpckl, /* Lo = */ true, /* Unary = */ false);
@@ -10032,9 +10032,9 @@ static SDValue lowerShuffleWithUNPCK(const SDLoc &DL, MVT VT,
 
 /// Check if the mask can be mapped to a preliminary shuffle (vperm 64-bit)
 /// followed by unpack 256-bit.
-static SDValue lowerShuffleWithUNPCK256(const SDLoc &DL, MVT VT,
-                                        ArrayRef<int> Mask, SDValue V1,
-                                        SDValue V2, SelectionDAG &DAG) {
+static SDValue lowerShuffleWithUNPCK256(const SDLoc &DL, MVT VT, SDValue V1,
+                                        SDValue V2, ArrayRef<int> Mask,
+                                        SelectionDAG &DAG) {
   SmallVector<int, 32> Unpckl, Unpckh;
   createSplat2ShuffleMask(VT, Unpckl, /* Lo */ true);
   createSplat2ShuffleMask(VT, Unpckh, /* Lo */ false);
@@ -10432,9 +10432,10 @@ static bool matchShuffleWithPACK(MVT VT, MVT &SrcVT, SDValue &V1, SDValue &V2,
   return false;
 }
 
-static SDValue lowerShuffleWithPACK(const SDLoc &DL, MVT VT, ArrayRef<int> Mask,
-                                    SDValue V1, SDValue V2, SelectionDAG &DAG,
-                                    const X86Subtarget &Subtarget) {
+static SDValue lowerShuffleWithPACK(const SDLoc &DL, MVT VT, SDValue V1,
+                                    SDValue V2, ArrayRef<int> Mask,
+                                    const X86Subtarget &Subtarget,
+                                    SelectionDAG &DAG) {
   MVT PackVT;
   unsigned PackOpcode;
   unsigned SizeBits = VT.getSizeInBits();
@@ -12861,7 +12862,7 @@ static SDValue lowerV2F64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
       return Blend;
 
   // Use dedicated unpack instructions for masks that match their pattern.
-  if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v2f64, Mask, V1, V2, DAG))
+  if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v2f64, V1, V2, Mask, DAG))
     return V;
 
   unsigned SHUFPDMask = (Mask[0] == 1) | (((Mask[1] - 2) == 1) << 1);
@@ -12938,7 +12939,7 @@ static SDValue lowerV2I64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
       return Blend;
 
   // Use dedicated unpack instructions for masks that match their pattern.
-  if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v2i64, Mask, V1, V2, DAG))
+  if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v2i64, V1, V2, Mask, DAG))
     return V;
 
   // Try to use byte rotation instructions.
@@ -13160,7 +13161,7 @@ static SDValue lowerV4F32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
   }
 
   // Use dedicated unpack instructions for masks that match their pattern.
-  if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4f32, Mask, V1, V2, DAG))
+  if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4f32, V1, V2, Mask, DAG))
     return V;
 
   // Otherwise fall back to a SHUFPS lowering strategy.
@@ -13253,7 +13254,7 @@ static SDValue lowerV4I32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
     return Masked;
 
   // Use dedicated unpack instructions for masks that match their pattern.
-  if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4i32, Mask, V1, V2, DAG))
+  if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4i32, V1, V2, Mask, DAG))
     return V;
 
   // Try to use byte rotation instructions.
@@ -13908,12 +13909,12 @@ static SDValue lowerV8I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
       return Rotate;
 
     // Use dedicated unpack instructions for masks that match their pattern.
-    if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8i16, Mask, V1, V2, DAG))
+    if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8i16, V1, V2, Mask, DAG))
       return V;
 
     // Use dedicated pack instructions for masks that match their pattern.
-    if (SDValue V = lowerShuffleWithPACK(DL, MVT::v8i16, Mask, V1, V2, DAG,
-                                         Subtarget))
+    if (SDValue V =
+            lowerShuffleWithPACK(DL, MVT::v8i16, V1, V2, Mask, Subtarget, DAG))
       return V;
 
     // Try to use byte rotation instructions.
@@ -13962,12 +13963,12 @@ static SDValue lowerV8I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
     return Masked;
 
   // Use dedicated unpack instructions for masks that match their pattern.
-  if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8i16, Mask, V1, V2, DAG))
+  if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8i16, V1, V2, Mask, DAG))
     return V;
 
   // Use dedicated pack instructions for masks that match their pattern.
-  if (SDValue V = lowerShuffleWithPACK(DL, MVT::v8i16, Mask, V1, V2, DAG,
-                                       Subtarget))
+  if (SDValue V =
+          lowerShuffleWithPACK(DL, MVT::v8i16, V1, V2, Mask, Subtarget, DAG))
     return V;
 
   // Try to use lower using a truncation.
@@ -14168,8 +14169,8 @@ static SDValue lowerV16I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
     return Rotate;
 
   // Use dedicated pack instructions for masks that match their pattern.
-  if (SDValue V = lowerShuffleWithPACK(DL, MVT::v16i8, Mask, V1, V2, DAG,
-                                       Subtarget))
+  if (SDValue V =
+          lowerShuffleWithPACK(DL, MVT::v16i8, V1, V2, Mask, Subtarget, DAG))
     return V;
 
   // Try to use a zext lowering.
@@ -14206,7 +14207,7 @@ static SDValue lowerV16I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
                                                  Subtarget, DAG))
       return Rotate;
 
-    if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i8, Mask, V1, V2, DAG))
+    if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i8, V1, V2, Mask, DAG))
       return V;
 
     // Check whether we can widen this to an i16 shuffle by duplicating bytes.
@@ -14310,7 +14311,7 @@ static SDValue lowerV16I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
     return Masked;
 
   // Use dedicated unpack instructions for masks that match their pattern.
-  if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i8, Mask, V1, V2, DAG))
+  if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i8, V1, V2, Mask, DAG))
     return V;
 
   // Try to use byte shift instructions to mask.
@@ -15882,7 +15883,7 @@ static SDValue lowerV4F64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
   }
 
   // Use dedicated unpack instructions for masks that match their pattern.
-  if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4f64, Mask, V1, V2, DAG))
+  if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4f64, V1, V2, Mask, DAG))
     return V;
 
   if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v4f64, V1, V2, Mask,
@@ -16021,7 +16022,7 @@ static SDValue lowerV4I64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
     return Rotate;
 
   // Use dedicated unpack instructions for masks that match their pattern.
-  if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4i64, Mask, V1, V2, DAG))
+  if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4i64, V1, V2, Mask, DAG))
     return V;
 
   bool V1IsInPlace = isShuffleMaskInputInPlace(0, Mask);
@@ -16110,7 +16111,7 @@ static SDValue lowerV8F32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
                          getV4X86ShuffleImm8ForMask(RepeatedMask, DL, DAG));
 
     // Use dedicated unpack instructions for masks that match their pattern.
-    if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8f32, Mask, V1, V2, DAG))
+    if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8f32, V1, V2, Mask, DAG))
       return V;
 
     // Otherwise, fall back to a SHUFPS sequence. Here it is important that we
@@ -16251,7 +16252,7 @@ static SDValue lowerV8I32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
                          getV4X86ShuffleImm8ForMask(RepeatedMask, DL, DAG));
 
     // Use dedicated unpack instructions for masks that match their pattern.
-    if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8i32, Mask, V1, V2, DAG))
+    if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8i32, V1, V2, Mask, DAG))
       return V;
   }
 
@@ -16291,7 +16292,7 @@ static SDValue lowerV8I32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
   if (V2.isUndef()) {
     // Try to produce a fixed cross-128-bit lane permute followed by unpack
     // because that should be faster than the variable permute alternatives.
-    if (SDValue V = lowerShuffleWithUNPCK256(DL, MVT::v8i32, Mask, V1, V2, DAG))
+    if (SDValue V = lowerShuffleWithUNPCK256(DL, MVT::v8i32, V1, V2, Mask, DAG))
       return V;
 
     // If the shuffle patterns aren't repeated but it's a single input, directly
@@ -16352,12 +16353,12 @@ static SDValue lowerV16I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
     return Blend;
 
   // Use dedicated unpack instructions for masks that match their pattern.
-  if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i16, Mask, V1, V2, DAG))
+  if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i16, V1, V2, Mask, DAG))
     return V;
 
   // Use dedicated pack instructions for masks that match their pattern.
-  if (SDValue V = lowerShuffleWithPACK(DL, MVT::v16i16, Mask, V1, V2, DAG,
-                                       Subtarget))
+  if (SDValue V =
+          lowerShuffleWithPACK(DL, MVT::v16i16, V1, V2, Mask, Subtarget, DAG))
     return V;
 
   // Try to use lower using a truncation.
@@ -16390,7 +16391,7 @@ static SDValue lowerV16I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
 
     // Try to produce a fixed cross-128-bit lane permute followed by unpack
     // because that should be faster than the variable permute alternatives.
-    if (SDValue V = lowerShuffleWithUNPCK256(DL, MVT::v16i16, Mask, V1, V2, DAG))
+    if (SDValue V = lowerShuffleWithUNPCK256(DL, MVT::v16i16, V1, V2, Mask, DAG))
       return V;
 
     // There are no generalized cross-lane shuffle operations available on i16
@@ -16475,12 +16476,12 @@ static SDValue lowerV32I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
     return Blend;
 
   // Use dedicated unpack instructions for masks that match their pattern.
-  if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v32i8, Mask, V1, V2, DAG))
+  if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v32i8, V1, V2, Mask, DAG))
     return V;
 
   // Use dedicated pack instructions for masks that match their pattern.
-  if (SDValue V = lowerShuffleWithPACK(DL, MVT::v32i8, Mask, V1, V2, DAG,
-                                       Subtarget))
+  if (SDValue V =
+          lowerShuffleWithPACK(DL, MVT::v32i8, V1, V2, Mask, Subtarget, DAG))
     return V;
 
   // Try to use lower using a truncation.
@@ -16516,7 +16517,7 @@ static SDValue lowerV32I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
   if (V2.isUndef() && is128BitLaneCrossingShuffleMask(MVT::v32i8, Mask)) {
     // Try to produce a fixed cross-128-bit lane permute followed by unpack
     // because that should be faster than the variable permute alternatives.
-    if (SDValue V = lowerShuffleWithUNPCK256(DL, MVT::v32i8, Mask, V1, V2, DAG))
+    if (SDValue V = lowerShuffleWithUNPCK256(DL, MVT::v32i8, V1, V2, Mask, DAG))
       return V;
 
     if (SDValue V = lowerShuffleAsLanePermuteAndPermute(
@@ -16783,7 +16784,7 @@ static SDValue lowerV8F64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
                                            V2, Subtarget, DAG))
     return Shuf128;
 
-  if (SDValue Unpck = lowerShuffleWithUNPCK(DL, MVT::v8f64, Mask, V1, V2, DAG))
+  if (SDValue Unpck = lowerShuffleWithUNPCK(DL, MVT::v8f64, V1, V2, Mask, DAG))
     return Unpck;
 
   // Check if the blend happens to exactly fit that of SHUFPD.
@@ -16828,7 +16829,7 @@ static SDValue lowerV16F32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
                          getV4X86ShuffleImm8ForMask(RepeatedMask, DL, DAG));
 
     // Use dedicated unpack instructions for masks that match their pattern.
-    if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16f32, Mask, V1, V2, DAG))
+    if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16f32, V1, V2, Mask, DAG))
       return V;
 
     if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v16f32, V1, V2, Mask,
@@ -16927,7 +16928,7 @@ static SDValue lowerV8I64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
                                                   Subtarget, DAG))
       return Rotate;
 
-  if (SDValue Unpck = lowerShuffleWithUNPCK(DL, MVT::v8i64, Mask, V1, V2, DAG))
+  if (SDValue Unpck = lowerShuffleWithUNPCK(DL, MVT::v8i64, V1, V2, Mask, DAG))
     return Unpck;
 
   // If we have AVX512F support, we can use VEXPAND.
@@ -16985,7 +16986,7 @@ static SDValue lowerV16I32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
                          getV4X86ShuffleImm8ForMask(RepeatedMask, DL, DAG));
 
     // Use dedicated unpack instructions for masks that match their pattern.
-    if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i32, Mask, V1, V2, DAG))
+    if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i32, V1, V2, Mask, DAG))
       return V;
   }
 
@@ -17057,12 +17058,12 @@ static SDValue lowerV32I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
     return ZExt;
 
   // Use dedicated unpack instructions for masks that match their pattern.
-  if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v32i16, Mask, V1, V2, DAG))
+  if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v32i16, V1, V2, Mask, DAG))
     return V;
 
   // Use dedicated pack instructions for masks that match their pattern.
   if (SDValue V =
-          lowerShuffleWithPACK(DL, MVT::v32i16, Mask, V1, V2, DAG, Subtarget))
+          lowerShuffleWithPACK(DL, MVT::v32i16, V1, V2, Mask, Subtarget, DAG))
     return V;
 
   // Try to use shift instructions.
@@ -17121,12 +17122,12 @@ static SDValue lowerV64I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
     return ZExt;
 
   // Use dedicated unpack instructions for masks that match their pattern.
-  if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v64i8, Mask, V1, V2, DAG))
+  if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v64i8, V1, V2, Mask, DAG))
     return V;
 
   // Use dedicated pack instructions for masks that match their pattern.
-  if (SDValue V = lowerShuffleWithPACK(DL, MVT::v64i8, Mask, V1, V2, DAG,
-                                       Subtarget))
+  if (SDValue V =
+          lowerShuffleWithPACK(DL, MVT::v64i8, V1, V2, Mask, Subtarget, DAG))
     return V;
 
   // Try to use shift instructions.

From 9211977d134d81cbc7a24f13e244334484c31b87 Mon Sep 17 00:00:00 2001
From: Kerry McLaughlin <kerry.mclaughlin@arm.com>
Date: Mon, 19 Aug 2024 10:17:10 +0100
Subject: [PATCH 337/441] [AArch64][SME] Return false from
 produceCompactUnwindFrame if VG save required. (#104588)

The compact unwind format requires all registers are stored in pairs, so
return false from produceCompactUnwindFrame if we require saving VG.
---
 .../Target/AArch64/AArch64FrameLowering.cpp   |  3 +-
 .../test/CodeGen/AArch64/sme-darwin-sve-vg.ll | 55 +++++++++++++++++++
 2 files changed, 57 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/CodeGen/AArch64/sme-darwin-sve-vg.ll

diff --git a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
index 429affed63497d5..e0f8853b715354b 100644
--- a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
@@ -2860,7 +2860,8 @@ static bool produceCompactUnwindFrame(MachineFunction &MF) {
   return Subtarget.isTargetMachO() &&
          !(Subtarget.getTargetLowering()->supportSwiftError() &&
            Attrs.hasAttrSomewhere(Attribute::SwiftError)) &&
-         MF.getFunction().getCallingConv() != CallingConv::SwiftTail;
+         MF.getFunction().getCallingConv() != CallingConv::SwiftTail &&
+         !requiresSaveVG(MF);
 }
 
 static bool invalidateWindowsRegisterPairing(unsigned Reg1, unsigned Reg2,
diff --git a/llvm/test/CodeGen/AArch64/sme-darwin-sve-vg.ll b/llvm/test/CodeGen/AArch64/sme-darwin-sve-vg.ll
new file mode 100644
index 000000000000000..c32e9cbc053939c
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sme-darwin-sve-vg.ll
@@ -0,0 +1,55 @@
+; RUN: llc -mtriple=aarch64-darwin -mattr=+sve -mattr=+sme -verify-machineinstrs < %s | FileCheck %s
+
+declare void @normal_callee();
+
+define void @locally_streaming_fn() #0 {
+; CHECK-LABEL: locally_streaming_fn:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    stp d15, d14, [sp, #-96]! ; 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 96
+; CHECK-NEXT:    rdsvl x9, #1
+; CHECK-NEXT:    stp d13, d12, [sp, #16] ; 16-byte Folded Spill
+; CHECK-NEXT:    lsr x9, x9, #3
+; CHECK-NEXT:    stp d11, d10, [sp, #32] ; 16-byte Folded Spill
+; CHECK-NEXT:    stp d9, d8, [sp, #48] ; 16-byte Folded Spill
+; CHECK-NEXT:    stp x30, x9, [sp, #64] ; 16-byte Folded Spill
+; CHECK-NEXT:    cntd x9
+; CHECK-NEXT:    str x9, [sp, #80] ; 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_offset vg, -16
+; CHECK-NEXT:    .cfi_offset w30, -32
+; CHECK-NEXT:    .cfi_offset b8, -40
+; CHECK-NEXT:    .cfi_offset b9, -48
+; CHECK-NEXT:    .cfi_offset b10, -56
+; CHECK-NEXT:    .cfi_offset b11, -64
+; CHECK-NEXT:    .cfi_offset b12, -72
+; CHECK-NEXT:    .cfi_offset b13, -80
+; CHECK-NEXT:    .cfi_offset b14, -88
+; CHECK-NEXT:    .cfi_offset b15, -96
+; CHECK-NEXT:    smstart sm
+; CHECK-NEXT:    .cfi_offset vg, -24
+; CHECK-NEXT:    smstop sm
+; CHECK-NEXT:    bl _normal_callee
+; CHECK-NEXT:    smstart sm
+; CHECK-NEXT:    .cfi_restore vg
+; CHECK-NEXT:    smstop sm
+; CHECK-NEXT:    ldp d9, d8, [sp, #48] ; 16-byte Folded Reload
+; CHECK-NEXT:    ldr x30, [sp, #64] ; 8-byte Folded Reload
+; CHECK-NEXT:    ldp d11, d10, [sp, #32] ; 16-byte Folded Reload
+; CHECK-NEXT:    ldp d13, d12, [sp, #16] ; 16-byte Folded Reload
+; CHECK-NEXT:    ldp d15, d14, [sp], #96 ; 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore b8
+; CHECK-NEXT:    .cfi_restore b9
+; CHECK-NEXT:    .cfi_restore b10
+; CHECK-NEXT:    .cfi_restore b11
+; CHECK-NEXT:    .cfi_restore b12
+; CHECK-NEXT:    .cfi_restore b13
+; CHECK-NEXT:    .cfi_restore b14
+; CHECK-NEXT:    .cfi_restore b15
+; CHECK-NEXT:    ret
+  call void @normal_callee()
+  ret void
+}
+
+attributes #0 = { "aarch64_pstate_sm_body" uwtable(async) }

From 63267ca9016aa334b329aa408716456b4e3799c8 Mon Sep 17 00:00:00 2001
From: Weining Lu <luweining@loongson.cn>
Date: Mon, 19 Aug 2024 16:51:21 +0800
Subject: [PATCH 338/441] [LoongArch] Fix the assertion for atomic store with
 'ptr' type

---
 .../LoongArch/LoongArchISelLowering.cpp       |   5 +-
 .../ir-instruction/load-store-atomic.ll       | 119 ++++++++++++++++++
 2 files changed, 122 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
index 44d684331b3eb46..50c6c263e966b54 100644
--- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
+++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
@@ -5619,8 +5619,9 @@ bool LoongArchTargetLowering::shouldInsertFencesForAtomic(
 
   // On LA64, atomic store operations with IntegerBitWidth of 32 and 64 do not
   // require fences beacuse we can use amswap_db.[w/d].
-  if (isa<StoreInst>(I)) {
-    unsigned Size = I->getOperand(0)->getType()->getIntegerBitWidth();
+  Type *Ty = I->getOperand(0)->getType();
+  if (isa<StoreInst>(I) && Ty->isIntegerTy()) {
+    unsigned Size = Ty->getIntegerBitWidth();
     return (Size == 8 || Size == 16);
   }
 
diff --git a/llvm/test/CodeGen/LoongArch/ir-instruction/load-store-atomic.ll b/llvm/test/CodeGen/LoongArch/ir-instruction/load-store-atomic.ll
index c51fded410e83b0..1af2b38d7994363 100644
--- a/llvm/test/CodeGen/LoongArch/ir-instruction/load-store-atomic.ll
+++ b/llvm/test/CodeGen/LoongArch/ir-instruction/load-store-atomic.ll
@@ -72,6 +72,22 @@ define i64 @load_acquire_i64(ptr %ptr) {
   ret i64 %val
 }
 
+define ptr @load_acquire_ptr(ptr %ptr) {
+; LA32-LABEL: load_acquire_ptr:
+; LA32:       # %bb.0:
+; LA32-NEXT:    ld.w $a0, $a0, 0
+; LA32-NEXT:    dbar 20
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: load_acquire_ptr:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ld.d $a0, $a0, 0
+; LA64-NEXT:    dbar 20
+; LA64-NEXT:    ret
+  %val = load atomic ptr, ptr %ptr acquire, align 8
+  ret ptr %val
+}
+
 define i8 @load_unordered_i8(ptr %ptr) {
 ; LA32-LABEL: load_unordered_i8:
 ; LA32:       # %bb.0:
@@ -135,6 +151,20 @@ define i64 @load_unordered_i64(ptr %ptr) {
   ret i64 %val
 }
 
+define ptr @load_unordered_ptr(ptr %ptr) {
+; LA32-LABEL: load_unordered_ptr:
+; LA32:       # %bb.0:
+; LA32-NEXT:    ld.w $a0, $a0, 0
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: load_unordered_ptr:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ld.d $a0, $a0, 0
+; LA64-NEXT:    ret
+  %val = load atomic ptr, ptr %ptr unordered, align 8
+  ret ptr %val
+}
+
 define i8 @load_monotonic_i8(ptr %ptr) {
 ; LA32-LABEL: load_monotonic_i8:
 ; LA32:       # %bb.0:
@@ -198,6 +228,20 @@ define i64 @load_monotonic_i64(ptr %ptr) {
   ret i64 %val
 }
 
+define ptr @load_monotonic_ptr(ptr %ptr) {
+; LA32-LABEL: load_monotonic_ptr:
+; LA32:       # %bb.0:
+; LA32-NEXT:    ld.w $a0, $a0, 0
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: load_monotonic_ptr:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ld.d $a0, $a0, 0
+; LA64-NEXT:    ret
+  %val = load atomic ptr, ptr %ptr monotonic, align 8
+  ret ptr %val
+}
+
 define i8 @load_seq_cst_i8(ptr %ptr) {
 ; LA32-LABEL: load_seq_cst_i8:
 ; LA32:       # %bb.0:
@@ -268,6 +312,22 @@ define i64 @load_seq_cst_i64(ptr %ptr) {
   ret i64 %val
 }
 
+define ptr @load_seq_cst_ptr(ptr %ptr) {
+; LA32-LABEL: load_seq_cst_ptr:
+; LA32:       # %bb.0:
+; LA32-NEXT:    ld.w $a0, $a0, 0
+; LA32-NEXT:    dbar 16
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: load_seq_cst_ptr:
+; LA64:       # %bb.0:
+; LA64-NEXT:    ld.d $a0, $a0, 0
+; LA64-NEXT:    dbar 16
+; LA64-NEXT:    ret
+  %val = load atomic ptr, ptr %ptr seq_cst, align 8
+  ret ptr %val
+}
+
 define void @store_release_i8(ptr %ptr, i8 signext %v) {
 ; LA32-LABEL: store_release_i8:
 ; LA32:       # %bb.0:
@@ -336,6 +396,21 @@ define void @store_release_i64(ptr %ptr, i64 %v) {
   ret void
 }
 
+define void @store_release_ptr(ptr %ptr, ptr %v) {
+; LA32-LABEL: store_release_ptr:
+; LA32:       # %bb.0:
+; LA32-NEXT:    dbar 18
+; LA32-NEXT:    st.w $a1, $a0, 0
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: store_release_ptr:
+; LA64:       # %bb.0:
+; LA64-NEXT:    amswap_db.d $zero, $a1, $a0
+; LA64-NEXT:    ret
+  store atomic ptr %v, ptr %ptr release, align 8
+  ret void
+}
+
 define void @store_unordered_i8(ptr %ptr, i8 signext %v) {
 ; LA32-LABEL: store_unordered_i8:
 ; LA32:       # %bb.0:
@@ -399,6 +474,20 @@ define void @store_unordered_i64(ptr %ptr, i64 %v) {
   ret void
 }
 
+define void @store_unordered_ptr(ptr %ptr, ptr %v) {
+; LA32-LABEL: store_unordered_ptr:
+; LA32:       # %bb.0:
+; LA32-NEXT:    st.w $a1, $a0, 0
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: store_unordered_ptr:
+; LA64:       # %bb.0:
+; LA64-NEXT:    st.d $a1, $a0, 0
+; LA64-NEXT:    ret
+  store atomic ptr %v, ptr %ptr unordered, align 8
+  ret void
+}
+
 define void @store_monotonic_i8(ptr %ptr, i8 signext %v) {
 ; LA32-LABEL: store_monotonic_i8:
 ; LA32:       # %bb.0:
@@ -462,6 +551,20 @@ define void @store_monotonic_i64(ptr %ptr, i64 %v) {
   ret void
 }
 
+define void @store_monotonic_ptr(ptr %ptr, ptr %v) {
+; LA32-LABEL: store_monotonic_ptr:
+; LA32:       # %bb.0:
+; LA32-NEXT:    st.w $a1, $a0, 0
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: store_monotonic_ptr:
+; LA64:       # %bb.0:
+; LA64-NEXT:    st.d $a1, $a0, 0
+; LA64-NEXT:    ret
+  store atomic ptr %v, ptr %ptr monotonic, align 8
+  ret void
+}
+
 define void @store_seq_cst_i8(ptr %ptr, i8 signext %v) {
 ; LA32-LABEL: store_seq_cst_i8:
 ; LA32:       # %bb.0:
@@ -534,3 +637,19 @@ define void @store_seq_cst_i64(ptr %ptr, i64 %v) {
   store atomic i64 %v, ptr %ptr seq_cst, align 8
   ret void
 }
+
+define void @store_seq_cst_ptr(ptr %ptr, ptr %v) {
+; LA32-LABEL: store_seq_cst_ptr:
+; LA32:       # %bb.0:
+; LA32-NEXT:    dbar 16
+; LA32-NEXT:    st.w $a1, $a0, 0
+; LA32-NEXT:    dbar 16
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: store_seq_cst_ptr:
+; LA64:       # %bb.0:
+; LA64-NEXT:    amswap_db.d $zero, $a1, $a0
+; LA64-NEXT:    ret
+  store atomic ptr %v, ptr %ptr seq_cst, align 8
+  ret void
+}

From 065d2d9c60da0214c17a8300385b22be1c26dc70 Mon Sep 17 00:00:00 2001
From: Christian Ulmann <christianulmann@gmail.com>
Date: Mon, 19 Aug 2024 11:27:30 +0200
Subject: [PATCH 339/441] [MLIR][LLVM] Improve the noalias propagation during
 inlining (#104750)

This commit changes the LLVM dialect's inliner interface to properly
propagate noalias information to memory accesses that have different
underlying object. By always introducing an SSACopy intrinsic, it's
possible to understand that specific memory operations are using
unrelated pointers. Previously, the backwards slice walk did continue
beyond the boundary of the original function and failed to reason about
the "underlying objects".
---
 .../Transforms/InlinerInterfaceImpl.cpp       | 72 ++++++++++---------
 .../Dialect/LLVMIR/inlining-alias-scopes.mlir | 23 ++++++
 2 files changed, 60 insertions(+), 35 deletions(-)

diff --git a/mlir/lib/Dialect/LLVMIR/Transforms/InlinerInterfaceImpl.cpp b/mlir/lib/Dialect/LLVMIR/Transforms/InlinerInterfaceImpl.cpp
index 504f63b48c9433d..1399d419735db95 100644
--- a/mlir/lib/Dialect/LLVMIR/Transforms/InlinerInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/LLVMIR/Transforms/InlinerInterfaceImpl.cpp
@@ -271,37 +271,41 @@ getUnderlyingObjectSet(Value pointerValue) {
 static void createNewAliasScopesFromNoAliasParameter(
     Operation *call, iterator_range<Region::iterator> inlinedBlocks) {
 
-  // First collect all noalias parameters. These have been specially marked by
-  // the `handleArgument` implementation by using the `ssa.copy` intrinsic and
-  // attaching a `noalias` attribute to it.
-  // These are only meant to be temporary and should therefore be deleted after
-  // we're done using them here.
+  // First, collect all ssa copy operations, which correspond to function
+  // parameters, and additionally store the noalias parameters. All parameters
+  // have been marked by the `handleArgument` implementation by using the
+  // `ssa.copy` intrinsic. Additionally, noalias parameters have an attached
+  // `noalias` attribute to the intrinsics. These intrinsics are only meant to
+  // be temporary and should therefore be deleted after we're done using them
+  // here.
+  SetVector<LLVM::SSACopyOp> ssaCopies;
   SetVector<LLVM::SSACopyOp> noAliasParams;
   for (Value argument : cast<LLVM::CallOp>(call).getArgOperands()) {
     for (Operation *user : argument.getUsers()) {
       auto ssaCopy = llvm::dyn_cast<LLVM::SSACopyOp>(user);
       if (!ssaCopy)
         continue;
+      ssaCopies.insert(ssaCopy);
+
       if (!ssaCopy->hasAttr(LLVM::LLVMDialect::getNoAliasAttrName()))
         continue;
-
       noAliasParams.insert(ssaCopy);
     }
   }
 
-  // If there were none, we have nothing to do here.
-  if (noAliasParams.empty())
-    return;
-
   // Scope exit block to make it impossible to forget to get rid of the
   // intrinsics.
   auto exit = llvm::make_scope_exit([&] {
-    for (LLVM::SSACopyOp ssaCopyOp : noAliasParams) {
+    for (LLVM::SSACopyOp ssaCopyOp : ssaCopies) {
       ssaCopyOp.replaceAllUsesWith(ssaCopyOp.getOperand());
       ssaCopyOp->erase();
     }
   });
 
+  // If there were no noalias parameters, we have nothing to do here.
+  if (noAliasParams.empty())
+    return;
+
   // Create a new domain for this specific inlining and a new scope for every
   // noalias parameter.
   auto functionDomain = LLVM::AliasScopeDomainAttr::get(
@@ -335,7 +339,7 @@ static void createNewAliasScopesFromNoAliasParameter(
       bool aliasesOtherKnownObject = false;
       // Go through the based on pointers and check that they are either:
       // * Constants that can be ignored (undef, poison, null pointer).
-      // * Based on a noalias parameter.
+      // * Based on a pointer parameter.
       // * Other pointers that we know can't alias with our noalias parameter.
       //
       // Any other value might be a pointer based on any noalias parameter that
@@ -346,11 +350,13 @@ static void createNewAliasScopesFromNoAliasParameter(
             if (matchPattern(object, m_Constant()))
               return false;
 
-            if (noAliasParams.contains(object.getDefiningOp<LLVM::SSACopyOp>()))
+            if (auto ssaCopy = object.getDefiningOp<LLVM::SSACopyOp>()) {
+              // If that value is based on a noalias parameter, it is guaranteed
+              // to not alias with any other object.
+              aliasesOtherKnownObject |= !noAliasParams.contains(ssaCopy);
               return false;
+            }
 
-            // TODO: This should include other arguments from the inlined
-            //       callable.
             if (isa_and_nonnull<LLVM::AllocaOp, LLVM::AddressOfOp>(
                     object.getDefiningOp())) {
               aliasesOtherKnownObject = true;
@@ -773,29 +779,25 @@ struct LLVMInlinerInterface : public DialectInlinerInterface {
       return handleByValArgument(builder, callable, argument, elementType,
                                  requestedAlignment);
     }
-    if (argumentAttrs.contains(LLVM::LLVMDialect::getNoAliasAttrName())) {
-      if (argument.use_empty())
-        return argument;
-
-      // This code is essentially a workaround for deficiencies in the
-      // inliner interface: We need to transform operations *after* inlined
-      // based on the argument attributes of the parameters *before* inlining.
-      // This method runs prior to actual inlining and thus cannot transform the
-      // post-inlining code, while `processInlinedCallBlocks` does not have
-      // access to pre-inlining function arguments. Additionally, it is required
-      // to distinguish which parameter an SSA value originally came from.
-      // As a workaround until this is changed: Create an ssa.copy intrinsic
-      // with the noalias attribute that can easily be found, and is extremely
-      // unlikely to exist in the code prior to inlining, using this to
-      // communicate between this method and `processInlinedCallBlocks`.
-      // TODO: Fix this by refactoring the inliner interface.
-      auto copyOp = builder.create<LLVM::SSACopyOp>(call->getLoc(), argument);
+
+    // This code is essentially a workaround for deficiencies in the inliner
+    // interface: We need to transform operations *after* inlined based on the
+    // argument attributes of the parameters *before* inlining. This method runs
+    // prior to actual inlining and thus cannot transform the post-inlining
+    // code, while `processInlinedCallBlocks` does not have access to
+    // pre-inlining function arguments. Additionally, it is required to
+    // distinguish which parameter an SSA value originally came from. As a
+    // workaround until this is changed: Create an ssa.copy intrinsic with the
+    // noalias attribute (when it was present before) that can easily be found,
+    // and is extremely unlikely to exist in the code prior to inlining, using
+    // this to communicate between this method and `processInlinedCallBlocks`.
+    // TODO: Fix this by refactoring the inliner interface.
+    auto copyOp = builder.create<LLVM::SSACopyOp>(call->getLoc(), argument);
+    if (argumentAttrs.contains(LLVM::LLVMDialect::getNoAliasAttrName()))
       copyOp->setDiscardableAttr(
           builder.getStringAttr(LLVM::LLVMDialect::getNoAliasAttrName()),
           builder.getUnitAttr());
-      return copyOp;
-    }
-    return argument;
+    return copyOp;
   }
 
   void processInlinedCallBlocks(
diff --git a/mlir/test/Dialect/LLVMIR/inlining-alias-scopes.mlir b/mlir/test/Dialect/LLVMIR/inlining-alias-scopes.mlir
index a91b991c5ed2b92..bd5e7aa996ada7e 100644
--- a/mlir/test/Dialect/LLVMIR/inlining-alias-scopes.mlir
+++ b/mlir/test/Dialect/LLVMIR/inlining-alias-scopes.mlir
@@ -234,6 +234,29 @@ llvm.func @bar(%arg0: !llvm.ptr, %arg1: !llvm.ptr, %arg2: !llvm.ptr) {
 
 // -----
 
+// CHECK-DAG: #[[DOMAIN:.*]] = #llvm.alias_scope_domain<{{.*}}>
+// CHECK-DAG: #[[$ARG_SCOPE:.*]] = #llvm.alias_scope<id = {{.*}}, domain = #[[DOMAIN]]{{(,.*)?}}>
+
+llvm.func @foo(%arg0: !llvm.ptr {llvm.noalias}, %arg1: !llvm.ptr) {
+  %0 = llvm.mlir.constant(5 : i64) : i64
+  %1 = llvm.load %arg0 {alignment = 4 : i64} : !llvm.ptr -> f32
+  %2 = llvm.getelementptr inbounds %arg1[%0] : (!llvm.ptr, i64) -> !llvm.ptr, f32
+  llvm.store %1, %2 {alignment = 4 : i64} : f32, !llvm.ptr
+  llvm.return
+}
+
+// CHECK-LABEL: llvm.func @missing_noalias_on_one_ptr
+// CHECK: llvm.load
+// CHECK-SAME: alias_scopes = [#[[$ARG_SCOPE]]]
+// CHECK: llvm.store
+// CHECK-SAME: noalias_scopes = [#[[$ARG_SCOPE]]]
+llvm.func @missing_noalias_on_one_ptr(%arg0: !llvm.ptr, %arg1: !llvm.ptr, %arg2: !llvm.ptr) {
+  llvm.call @foo(%arg0, %arg2) : (!llvm.ptr, !llvm.ptr) -> ()
+  llvm.return
+}
+
+// -----
+
 // CHECK-DAG: #[[DOMAIN:.*]] = #llvm.alias_scope_domain<{{.*}}>
 // CHECK-DAG: #[[$ARG0_SCOPE:.*]] = #llvm.alias_scope<id = {{.*}}, domain = #[[DOMAIN]]{{(,.*)?}}>
 // CHECK-DAG: #[[$ARG1_SCOPE:.*]] = #llvm.alias_scope<id = {{.*}}, domain = #[[DOMAIN]]{{(,.*)?}}>

From 740f05545150fb01cfca8c16da50c956b4f30959 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Mon, 19 Aug 2024 10:44:49 +0100
Subject: [PATCH 340/441] [VPlan] Rename getBestVF -> computeBestVF (NFC).

As suggested in https://github.com/llvm/llvm-project/pull/103033, more
accurately rename to computeBestVF, as it now does not simply return the
best VF, but directly computes it.
---
 llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h | 6 +++---
 llvm/lib/Transforms/Vectorize/LoopVectorize.cpp          | 6 +++---
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index f62713005393206..cd2680b0d43acf1 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -365,9 +365,9 @@ class LoopVectorizationPlanner {
   /// Return the best VPlan for \p VF.
   VPlan &getBestPlanFor(ElementCount VF) const;
 
-  /// Return the most profitable vectorization factor. Also collect all
-  /// profitable VFs in ProfitableVFs.
-  ElementCount getBestVF();
+  /// Compute and return the most profitable vectorization factor. Also collect
+  /// all profitable VFs in ProfitableVFs.
+  ElementCount computeBestVF();
 
   /// Generate the IR code for the vectorized loop captured in VPlan \p BestPlan
   /// according to the best selected \p VF and  \p UF.
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index e4307bf32804f7b..05a6c5f36d26ebc 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7250,7 +7250,7 @@ InstructionCost LoopVectorizationPlanner::cost(VPlan &Plan,
   return Cost;
 }
 
-ElementCount LoopVectorizationPlanner::getBestVF() {
+ElementCount LoopVectorizationPlanner::computeBestVF() {
   // If there is a single VPlan with a single VF, return it directly.
   VPlan &FirstPlan = *VPlans[0];
   if (VPlans.size() == 1 && size(FirstPlan.vectorFactors()) == 1)
@@ -9959,7 +9959,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
       InnerLoopUnroller Unroller(L, PSE, LI, DT, TLI, TTI, AC, ORE, IC, &LVL,
                                  &CM, BFI, PSI, Checks);
 
-      ElementCount BestVF = LVP.getBestVF();
+      ElementCount BestVF = LVP.computeBestVF();
       assert(BestVF.isScalar() &&
              "VPlan cost model and legacy cost model disagreed");
       VPlan &BestPlan = LVP.getBestPlanFor(BestVF);
@@ -9974,7 +9974,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
     } else {
       // If we decided that it is *legal* to vectorize the loop, then do it.
 
-      ElementCount BestVF = LVP.getBestVF();
+      ElementCount BestVF = LVP.computeBestVF();
       LLVM_DEBUG(dbgs() << "VF picked by VPlan cost model: " << BestVF << "\n");
       assert(VF.Width == BestVF &&
              "VPlan cost model and legacy cost model disagreed");

From de5deb7b07db39810c4b07870c75ccc8dccc5fe9 Mon Sep 17 00:00:00 2001
From: Daniil Kovalev <dkovalev@accesssoftek.com>
Date: Mon, 19 Aug 2024 12:53:43 +0300
Subject: [PATCH 341/441] [PAC][ELF][AArch64] Encode several ptrauth features
 in PAuth core info (#102508)

For llvm_linux platform, define the following meaning for bits 9, 10,
11:

- bit 9: set if indirect gotos signing is enabled;
- bit 10: set if type info vtable pointer discrimination is enabled;
- bit 11: set if function pointer type discrimination is enabled.
---
 clang/lib/CodeGen/CodeGenModule.cpp           | 10 +++-
 clang/test/CodeGen/aarch64-elf-pauthabi.c     | 31 +++++++++-
 llvm/include/llvm/BinaryFormat/ELF.h          |  5 +-
 .../AArch64/note-gnu-property-elf-pauthabi.ll |  6 +-
 .../ELF/AArch64/aarch64-feature-pauth.s       | 60 +++++++++----------
 llvm/tools/llvm-readobj/ELFDumper.cpp         |  7 ++-
 6 files changed, 79 insertions(+), 40 deletions(-)

diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp
index 6d11bd17d0a5f5d..0b61ef0f89989ce 100644
--- a/clang/lib/CodeGen/CodeGenModule.cpp
+++ b/clang/lib/CodeGen/CodeGenModule.cpp
@@ -1220,8 +1220,14 @@ void CodeGenModule::Release() {
           (LangOpts.PointerAuthInitFiniAddressDiscrimination
            << AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_INITFINIADDRDISC) |
           (LangOpts.PointerAuthELFGOT
-           << AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_GOT);
-      static_assert(AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_GOT ==
+           << AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_GOT) |
+          (LangOpts.PointerAuthIndirectGotos
+           << AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_GOTOS) |
+          (LangOpts.PointerAuthTypeInfoVTPtrDiscrimination
+           << AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_TYPEINFOVPTRDISCR) |
+          (LangOpts.PointerAuthFunctionTypeDiscrimination
+           << AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_FPTRTYPEDISCR);
+      static_assert(AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_FPTRTYPEDISCR ==
                         AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_LAST,
                     "Update when new enum items are defined");
       if (PAuthABIVersion != 0) {
diff --git a/clang/test/CodeGen/aarch64-elf-pauthabi.c b/clang/test/CodeGen/aarch64-elf-pauthabi.c
index d60dcee3618887e..023fa8c18e1300b 100644
--- a/clang/test/CodeGen/aarch64-elf-pauthabi.c
+++ b/clang/test/CodeGen/aarch64-elf-pauthabi.c
@@ -1,3 +1,5 @@
+//// TODO: also test with -fptrauth-elf-got when the driver flag is supported
+
 // RUN: %clang_cc1 -triple aarch64-linux -emit-llvm -o - \
 // RUN:   -fptrauth-intrinsics \
 // RUN:   -fptrauth-calls \
@@ -5,8 +7,11 @@
 // RUN:   -fptrauth-auth-traps \
 // RUN:   -fptrauth-vtable-pointer-address-discrimination \
 // RUN:   -fptrauth-vtable-pointer-type-discrimination \
-// RUN:   -fptrauth-init-fini %s \
-// RUN:   -fptrauth-init-fini-address-discrimination %s | \
+// RUN:   -fptrauth-init-fini \
+// RUN:   -fptrauth-init-fini-address-discrimination \
+// RUN:   -fptrauth-indirect-gotos \
+// RUN:   -fptrauth-type-info-vtable-pointer-discrimination \
+// RUN:   -fptrauth-function-pointer-type-discrimination %s | \
 // RUN:   FileCheck %s --check-prefix=ALL
 
 // RUN: %clang_cc1 -triple aarch64-linux -emit-llvm -o - \
@@ -37,8 +42,19 @@
 // RUN:   -fptrauth-calls -fptrauth-init-fini -fptrauth-init-fini-address-discrimination %s | \
 // RUN:   FileCheck %s --check-prefix=INITFINIADDR
 
+// RUN: %clang_cc1 -triple aarch64-linux -emit-llvm -o - \
+// RUN:   -fptrauth-indirect-gotos %s | FileCheck %s --check-prefix=GOTOS
+
+// RUN: %clang_cc1 -triple aarch64-linux -emit-llvm -o - \
+// RUN:   -fptrauth-calls -fptrauth-type-info-vtable-pointer-discrimination %s | \
+// RUN:   FileCheck %s --check-prefix=TYPEINFO
+
+// RUN: %clang_cc1 -triple aarch64-linux -emit-llvm -o - \
+// RUN:   -fptrauth-calls -fptrauth-function-pointer-type-discrimination %s | \
+// RUN:   FileCheck %s --check-prefix=FPTRTYPE
+
 // ALL: !{i32 1, !"aarch64-elf-pauthabi-platform", i32 268435458}
-// ALL: !{i32 1, !"aarch64-elf-pauthabi-version", i32 255}
+// ALL: !{i32 1, !"aarch64-elf-pauthabi-version", i32 3839}
 
 // INTRIN: !{i32 1, !"aarch64-elf-pauthabi-platform", i32 268435458}
 // INTRIN: !{i32 1, !"aarch64-elf-pauthabi-version", i32 1}
@@ -64,4 +80,13 @@
 // INITFINIADDR: !{i32 1, !"aarch64-elf-pauthabi-platform", i32 268435458}
 // INITFINIADDR: !{i32 1, !"aarch64-elf-pauthabi-version", i32 194}
 
+// GOTOS: !{i32 1, !"aarch64-elf-pauthabi-platform", i32 268435458}
+// GOTOS: !{i32 1, !"aarch64-elf-pauthabi-version", i32 512}
+
+// TYPEINFO: !{i32 1, !"aarch64-elf-pauthabi-platform", i32 268435458}
+// TYPEINFO: !{i32 1, !"aarch64-elf-pauthabi-version", i32 1026}
+
+// FPTRTYPE: !{i32 1, !"aarch64-elf-pauthabi-platform", i32 268435458}
+// FPTRTYPE: !{i32 1, !"aarch64-elf-pauthabi-version", i32 2050}
+
 void foo() {}
diff --git a/llvm/include/llvm/BinaryFormat/ELF.h b/llvm/include/llvm/BinaryFormat/ELF.h
index 2c9f70872962732..d61d0961d5987d0 100644
--- a/llvm/include/llvm/BinaryFormat/ELF.h
+++ b/llvm/include/llvm/BinaryFormat/ELF.h
@@ -1810,8 +1810,11 @@ enum : unsigned {
   AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_INITFINI = 6,
   AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_INITFINIADDRDISC = 7,
   AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_GOT = 8,
+  AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_GOTOS = 9,
+  AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_TYPEINFOVPTRDISCR = 10,
+  AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_FPTRTYPEDISCR = 11,
   AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_LAST =
-      AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_GOT,
+      AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_FPTRTYPEDISCR,
 };
 
 // x86 processor feature bits.
diff --git a/llvm/test/CodeGen/AArch64/note-gnu-property-elf-pauthabi.ll b/llvm/test/CodeGen/AArch64/note-gnu-property-elf-pauthabi.ll
index b2ebf1fc041137b..e5f675e62577d0c 100644
--- a/llvm/test/CodeGen/AArch64/note-gnu-property-elf-pauthabi.ll
+++ b/llvm/test/CodeGen/AArch64/note-gnu-property-elf-pauthabi.ll
@@ -10,7 +10,7 @@
 !llvm.module.flags = !{!0, !1}
 
 !0 = !{i32 1, !"aarch64-elf-pauthabi-platform", i32 268435458}
-!1 = !{i32 1, !"aarch64-elf-pauthabi-version", i32 341}
+!1 = !{i32 1, !"aarch64-elf-pauthabi-version", i32 1365}
 
 ; ASM: .section .note.gnu.property,"a",@note
 ; ASM-NEXT: .p2align 3, 0x0
@@ -22,12 +22,12 @@
 ; ASM-NEXT: .word 3221225473
 ; ASM-NEXT: .word 16
 ; ASM-NEXT: .xword 268435458
-; ASM-NEXT: .xword 341
+; ASM-NEXT: .xword 1365
 
 ; OBJ: Displaying notes found in: .note.gnu.property
 ; OBJ-NEXT:   Owner                 Data size	Description
 ; OBJ-NEXT:   GNU                   0x00000018	NT_GNU_PROPERTY_TYPE_0 (property note)
-; OBJ-NEXT:   AArch64 PAuth ABI core info: platform 0x10000002 (llvm_linux), version 0x155 (PointerAuthIntrinsics, !PointerAuthCalls, PointerAuthReturns, !PointerAuthAuthTraps, PointerAuthVTPtrAddressDiscrimination, !PointerAuthVTPtrTypeDiscrimination, PointerAuthInitFini, !PointerAuthInitFiniAddressDiscrimination, PointerAuthELFGOT)
+; OBJ-NEXT:   AArch64 PAuth ABI core info: platform 0x10000002 (llvm_linux), version 0x555 (PointerAuthIntrinsics, !PointerAuthCalls, PointerAuthReturns, !PointerAuthAuthTraps, PointerAuthVTPtrAddressDiscrimination, !PointerAuthVTPtrTypeDiscrimination, PointerAuthInitFini, !PointerAuthInitFiniAddressDiscrimination, PointerAuthELFGOT, !PointerAuthIndirectGotos, PointerAuthTypeInfoVTPtrDiscrimination, !PointerAuthFPtrTypeDiscrimination)
 
 ; ERR: either both or no 'aarch64-elf-pauthabi-platform' and 'aarch64-elf-pauthabi-version' module flags must be present
 
diff --git a/llvm/test/tools/llvm-readobj/ELF/AArch64/aarch64-feature-pauth.s b/llvm/test/tools/llvm-readobj/ELF/AArch64/aarch64-feature-pauth.s
index a71f8a231545f2c..aaf28ed4a1e79a1 100644
--- a/llvm/test/tools/llvm-readobj/ELF/AArch64/aarch64-feature-pauth.s
+++ b/llvm/test/tools/llvm-readobj/ELF/AArch64/aarch64-feature-pauth.s
@@ -108,12 +108,12 @@ end:
 # RUN: llvm-mc -filetype=obj -triple aarch64-linux-gnu gnu-0x10000002-0.s -o gnu-0x10000002-0.o
 # RUN: llvm-readelf --notes gnu-0x10000002-0.o | \
 # RUN:   FileCheck --check-prefix=ELF -DPLATFORM="0x10000002 (llvm_linux)" \
-# RUN:   -DVERSION="0x0 (!PointerAuthIntrinsics, !PointerAuthCalls, !PointerAuthReturns, !PointerAuthAuthTraps, !PointerAuthVTPtrAddressDiscrimination, !PointerAuthVTPtrTypeDiscrimination, !PointerAuthInitFini, !PointerAuthInitFiniAddressDiscrimination, !PointerAuthELFGOT)" %s
+# RUN:   -DVERSION="0x0 (!PointerAuthIntrinsics, !PointerAuthCalls, !PointerAuthReturns, !PointerAuthAuthTraps, !PointerAuthVTPtrAddressDiscrimination, !PointerAuthVTPtrTypeDiscrimination, !PointerAuthInitFini, !PointerAuthInitFiniAddressDiscrimination, !PointerAuthELFGOT, !PointerAuthIndirectGotos, !PointerAuthTypeInfoVTPtrDiscrimination, !PointerAuthFPtrTypeDiscrimination)" %s
 # RUN: llvm-readobj --notes gnu-0x10000002-0.o | \
 # RUN:   FileCheck --check-prefix=OBJ -DPLATFORM="0x10000002 (llvm_linux)" \
-# RUN:   -DVERSION="0x0 (!PointerAuthIntrinsics, !PointerAuthCalls, !PointerAuthReturns, !PointerAuthAuthTraps, !PointerAuthVTPtrAddressDiscrimination, !PointerAuthVTPtrTypeDiscrimination, !PointerAuthInitFini, !PointerAuthInitFiniAddressDiscrimination, !PointerAuthELFGOT)" %s
+# RUN:   -DVERSION="0x0 (!PointerAuthIntrinsics, !PointerAuthCalls, !PointerAuthReturns, !PointerAuthAuthTraps, !PointerAuthVTPtrAddressDiscrimination, !PointerAuthVTPtrTypeDiscrimination, !PointerAuthInitFini, !PointerAuthInitFiniAddressDiscrimination, !PointerAuthELFGOT, !PointerAuthIndirectGotos, !PointerAuthTypeInfoVTPtrDiscrimination, !PointerAuthFPtrTypeDiscrimination)" %s
 
-#--- gnu-0x10000002-341.s
+#--- gnu-0x10000002-1365.s
 .section ".note.gnu.property", "a"
   .long 4           // Name length is always 4 ("GNU")
   .long end - begin // Data length
@@ -125,19 +125,19 @@ begin:
   .long 0xc0000001  // Type: GNU_PROPERTY_AARCH64_FEATURE_PAUTH
   .long 16          // Data size
   .quad 0x10000002  // PAuth ABI platform
-  .quad 341          // PAuth ABI version
+  .quad 1365        // PAuth ABI version
   .p2align 3        // Align to 8 byte for 64 bit
 end:
 
-# RUN: llvm-mc -filetype=obj -triple aarch64-linux-gnu gnu-0x10000002-341.s -o gnu-0x10000002-341.o
-# RUN: llvm-readelf --notes gnu-0x10000002-341.o | \
+# RUN: llvm-mc -filetype=obj -triple aarch64-linux-gnu gnu-0x10000002-1365.s -o gnu-0x10000002-1365.o
+# RUN: llvm-readelf --notes gnu-0x10000002-1365.o | \
 # RUN:   FileCheck --check-prefix=ELF -DPLATFORM="0x10000002 (llvm_linux)" \
-# RUN:   -DVERSION="0x155 (PointerAuthIntrinsics, !PointerAuthCalls, PointerAuthReturns, !PointerAuthAuthTraps, PointerAuthVTPtrAddressDiscrimination, !PointerAuthVTPtrTypeDiscrimination, PointerAuthInitFini, !PointerAuthInitFiniAddressDiscrimination, PointerAuthELFGOT)" %s
-# RUN: llvm-readobj --notes gnu-0x10000002-341.o | \
+# RUN:   -DVERSION="0x555 (PointerAuthIntrinsics, !PointerAuthCalls, PointerAuthReturns, !PointerAuthAuthTraps, PointerAuthVTPtrAddressDiscrimination, !PointerAuthVTPtrTypeDiscrimination, PointerAuthInitFini, !PointerAuthInitFiniAddressDiscrimination, PointerAuthELFGOT, !PointerAuthIndirectGotos, PointerAuthTypeInfoVTPtrDiscrimination, !PointerAuthFPtrTypeDiscrimination)" %s
+# RUN: llvm-readobj --notes gnu-0x10000002-1365.o | \
 # RUN:   FileCheck --check-prefix=OBJ -DPLATFORM="0x10000002 (llvm_linux)" \
-# RUN:   -DVERSION="0x155 (PointerAuthIntrinsics, !PointerAuthCalls, PointerAuthReturns, !PointerAuthAuthTraps, PointerAuthVTPtrAddressDiscrimination, !PointerAuthVTPtrTypeDiscrimination, PointerAuthInitFini, !PointerAuthInitFiniAddressDiscrimination, PointerAuthELFGOT)" %s
+# RUN:   -DVERSION="0x555 (PointerAuthIntrinsics, !PointerAuthCalls, PointerAuthReturns, !PointerAuthAuthTraps, PointerAuthVTPtrAddressDiscrimination, !PointerAuthVTPtrTypeDiscrimination, PointerAuthInitFini, !PointerAuthInitFiniAddressDiscrimination, PointerAuthELFGOT, !PointerAuthIndirectGotos, PointerAuthTypeInfoVTPtrDiscrimination, !PointerAuthFPtrTypeDiscrimination)" %s
 
-#--- gnu-0x10000002-170.s
+#--- gnu-0x10000002-2730.s
 .section ".note.gnu.property", "a"
   .long 4           // Name length is always 4 ("GNU")
   .long end - begin // Data length
@@ -149,19 +149,19 @@ begin:
   .long 0xc0000001  // Type: GNU_PROPERTY_AARCH64_FEATURE_PAUTH
   .long 16          // Data size
   .quad 0x10000002  // PAuth ABI platform
-  .quad 170         // PAuth ABI version
+  .quad 2730        // PAuth ABI version
   .p2align 3        // Align to 8 byte for 64 bit
 end:
 
-# RUN: llvm-mc -filetype=obj -triple aarch64-linux-gnu gnu-0x10000002-170.s -o gnu-0x10000002-170.o
-# RUN: llvm-readelf --notes gnu-0x10000002-170.o | \
+# RUN: llvm-mc -filetype=obj -triple aarch64-linux-gnu gnu-0x10000002-2730.s -o gnu-0x10000002-2730.o
+# RUN: llvm-readelf --notes gnu-0x10000002-2730.o | \
 # RUN:   FileCheck --check-prefix=ELF -DPLATFORM="0x10000002 (llvm_linux)" \
-# RUN:   -DVERSION="0xaa (!PointerAuthIntrinsics, PointerAuthCalls, !PointerAuthReturns, PointerAuthAuthTraps, !PointerAuthVTPtrAddressDiscrimination, PointerAuthVTPtrTypeDiscrimination, !PointerAuthInitFini, PointerAuthInitFiniAddressDiscrimination, !PointerAuthELFGOT)" %s
-# RUN: llvm-readobj --notes gnu-0x10000002-170.o | \
+# RUN:   -DVERSION="0xaaa (!PointerAuthIntrinsics, PointerAuthCalls, !PointerAuthReturns, PointerAuthAuthTraps, !PointerAuthVTPtrAddressDiscrimination, PointerAuthVTPtrTypeDiscrimination, !PointerAuthInitFini, PointerAuthInitFiniAddressDiscrimination, !PointerAuthELFGOT, PointerAuthIndirectGotos, !PointerAuthTypeInfoVTPtrDiscrimination, PointerAuthFPtrTypeDiscrimination)" %s
+# RUN: llvm-readobj --notes gnu-0x10000002-2730.o | \
 # RUN:   FileCheck --check-prefix=OBJ -DPLATFORM="0x10000002 (llvm_linux)" \
-# RUN:   -DVERSION="0xaa (!PointerAuthIntrinsics, PointerAuthCalls, !PointerAuthReturns, PointerAuthAuthTraps, !PointerAuthVTPtrAddressDiscrimination, PointerAuthVTPtrTypeDiscrimination, !PointerAuthInitFini, PointerAuthInitFiniAddressDiscrimination, !PointerAuthELFGOT)" %s
+# RUN:   -DVERSION="0xaaa (!PointerAuthIntrinsics, PointerAuthCalls, !PointerAuthReturns, PointerAuthAuthTraps, !PointerAuthVTPtrAddressDiscrimination, PointerAuthVTPtrTypeDiscrimination, !PointerAuthInitFini, PointerAuthInitFiniAddressDiscrimination, !PointerAuthELFGOT, PointerAuthIndirectGotos, !PointerAuthTypeInfoVTPtrDiscrimination, PointerAuthFPtrTypeDiscrimination)" %s
 
-#--- gnu-0x10000002-511.s
+#--- gnu-0x10000002-4095.s
 .section ".note.gnu.property", "a"
   .long 4           // Name length is always 4 ("GNU")
   .long end - begin // Data length
@@ -173,19 +173,19 @@ begin:
   .long 0xc0000001  // Type: GNU_PROPERTY_AARCH64_FEATURE_PAUTH
   .long 16          // Data size
   .quad 0x10000002  // PAuth ABI platform
-  .quad 511         // PAuth ABI version
+  .quad 4095        // PAuth ABI version
   .p2align 3        // Align to 8 byte for 64 bit
 end:
 
-# RUN: llvm-mc -filetype=obj -triple aarch64-linux-gnu gnu-0x10000002-511.s -o gnu-0x10000002-511.o
-# RUN: llvm-readelf --notes gnu-0x10000002-511.o | \
+# RUN: llvm-mc -filetype=obj -triple aarch64-linux-gnu gnu-0x10000002-4095.s -o gnu-0x10000002-4095.o
+# RUN: llvm-readelf --notes gnu-0x10000002-4095.o | \
 # RUN:   FileCheck --check-prefix=ELF -DPLATFORM="0x10000002 (llvm_linux)" \
-# RUN:   -DVERSION="0x1ff (PointerAuthIntrinsics, PointerAuthCalls, PointerAuthReturns, PointerAuthAuthTraps, PointerAuthVTPtrAddressDiscrimination, PointerAuthVTPtrTypeDiscrimination, PointerAuthInitFini, PointerAuthInitFiniAddressDiscrimination, PointerAuthELFGOT)" %s
-# RUN: llvm-readobj --notes gnu-0x10000002-511.o | \
+# RUN:   -DVERSION="0xfff (PointerAuthIntrinsics, PointerAuthCalls, PointerAuthReturns, PointerAuthAuthTraps, PointerAuthVTPtrAddressDiscrimination, PointerAuthVTPtrTypeDiscrimination, PointerAuthInitFini, PointerAuthInitFiniAddressDiscrimination, PointerAuthELFGOT, PointerAuthIndirectGotos, PointerAuthTypeInfoVTPtrDiscrimination, PointerAuthFPtrTypeDiscrimination)" %s
+# RUN: llvm-readobj --notes gnu-0x10000002-4095.o | \
 # RUN:   FileCheck --check-prefix=OBJ -DPLATFORM="0x10000002 (llvm_linux)" \
-# RUN:   -DVERSION="0x1ff (PointerAuthIntrinsics, PointerAuthCalls, PointerAuthReturns, PointerAuthAuthTraps, PointerAuthVTPtrAddressDiscrimination, PointerAuthVTPtrTypeDiscrimination, PointerAuthInitFini, PointerAuthInitFiniAddressDiscrimination, PointerAuthELFGOT)" %s
+# RUN:   -DVERSION="0xfff (PointerAuthIntrinsics, PointerAuthCalls, PointerAuthReturns, PointerAuthAuthTraps, PointerAuthVTPtrAddressDiscrimination, PointerAuthVTPtrTypeDiscrimination, PointerAuthInitFini, PointerAuthInitFiniAddressDiscrimination, PointerAuthELFGOT, PointerAuthIndirectGotos, PointerAuthTypeInfoVTPtrDiscrimination, PointerAuthFPtrTypeDiscrimination)" %s
 
-#--- gnu-0x10000002-512.s
+#--- gnu-0x10000002-4096.s
 .section ".note.gnu.property", "a"
   .long 4           // Name length is always 4 ("GNU")
   .long end - begin // Data length
@@ -197,15 +197,15 @@ begin:
   .long 0xc0000001  // Type: GNU_PROPERTY_AARCH64_FEATURE_PAUTH
   .long 16          // Data size
   .quad 0x10000002  // PAuth ABI platform
-  .quad 512         // PAuth ABI version
+  .quad 4096        // PAuth ABI version
   .p2align 3        // Align to 8 byte for 64 bit
 end:
 
-# RUN: llvm-mc -filetype=obj -triple aarch64-linux-gnu gnu-0x10000002-512.s -o gnu-0x10000002-512.o
-# RUN: llvm-readelf --notes gnu-0x10000002-512.o | \
-# RUN:   FileCheck --check-prefix=ELF -DPLATFORM="0x10000002 (llvm_linux)" -DVERSION="0x200 (unknown)" %s
-# RUN: llvm-readobj --notes gnu-0x10000002-512.o | \
-# RUN:   FileCheck --check-prefix=OBJ -DPLATFORM="0x10000002 (llvm_linux)" -DVERSION="0x200 (unknown)" %s
+# RUN: llvm-mc -filetype=obj -triple aarch64-linux-gnu gnu-0x10000002-4096.s -o gnu-0x10000002-4096.o
+# RUN: llvm-readelf --notes gnu-0x10000002-4096.o | \
+# RUN:   FileCheck --check-prefix=ELF -DPLATFORM="0x10000002 (llvm_linux)" -DVERSION="0x1000 (unknown)" %s
+# RUN: llvm-readobj --notes gnu-0x10000002-4096.o | \
+# RUN:   FileCheck --check-prefix=OBJ -DPLATFORM="0x10000002 (llvm_linux)" -DVERSION="0x1000 (unknown)" %s
 
 #--- gnu-short.s
 .section ".note.gnu.property", "a"
diff --git a/llvm/tools/llvm-readobj/ELFDumper.cpp b/llvm/tools/llvm-readobj/ELFDumper.cpp
index b4804246875c606..97633c0fe5a37ea 100644
--- a/llvm/tools/llvm-readobj/ELFDumper.cpp
+++ b/llvm/tools/llvm-readobj/ELFDumper.cpp
@@ -5300,8 +5300,13 @@ static bool printAArch64PAuthABICoreInfo(raw_ostream &OS, uint32_t DataSize,
     Flags[AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_INITFINIADDRDISC] =
         "InitFiniAddressDiscrimination";
     Flags[AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_GOT] = "ELFGOT";
+    Flags[AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_GOTOS] = "IndirectGotos";
+    Flags[AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_TYPEINFOVPTRDISCR] =
+        "TypeInfoVTPtrDiscrimination";
+    Flags[AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_FPTRTYPEDISCR] =
+        "FPtrTypeDiscrimination";
 
-    static_assert(AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_GOT ==
+    static_assert(AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_FPTRTYPEDISCR ==
                       AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_LAST,
                   "Update when new enum items are defined");
 

From b6d1df2afd0a464f3f37055f16fb9321a3b2eaaf Mon Sep 17 00:00:00 2001
From: Qiu Chaofan <qiucofan@cn.ibm.com>
Date: Mon, 19 Aug 2024 17:58:08 +0800
Subject: [PATCH 342/441] [PowerPC] Support -mno-red-zone option (#94581)

---
 clang/lib/Driver/ToolChains/Clang.cpp  |   3 +
 clang/test/Driver/flags.c              |   3 +
 llvm/test/CodeGen/PowerPC/noredzone.ll | 519 +++++++++++++++++++++++++
 3 files changed, 525 insertions(+)
 create mode 100644 llvm/test/CodeGen/PowerPC/noredzone.ll

diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index daffa7cf96978a3..a5a87db97e96b48 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -2104,6 +2104,9 @@ void Clang::AddPPCTargetArgs(const ArgList &Args,
     CmdArgs.push_back("-mabi=vec-extabi");
   }
 
+  if (!Args.hasFlag(options::OPT_mred_zone, options::OPT_mno_red_zone, true))
+    CmdArgs.push_back("-disable-red-zone");
+
   ppc::FloatABI FloatABI = ppc::getPPCFloatABI(D, Args);
   if (FloatABI == ppc::FloatABI::Soft) {
     // Floating point operations and argument passing are soft.
diff --git a/clang/test/Driver/flags.c b/clang/test/Driver/flags.c
index 16b760609c36dd3..7d2bef63fca3e17 100644
--- a/clang/test/Driver/flags.c
+++ b/clang/test/Driver/flags.c
@@ -33,3 +33,6 @@
 //
 // RUN: %clang --target=riscv64 -### -S -mno-implicit-float %s 2>&1 | FileCheck -check-prefix=TEST11 %s
 // TEST11: "-no-implicit-float"
+//
+// RUN: %clang --target=ppc64le -### -S -mno-red-zone %s 2>&1 | FileCheck -check-prefix=TEST12 %s
+// TEST12: "-disable-red-zone"
diff --git a/llvm/test/CodeGen/PowerPC/noredzone.ll b/llvm/test/CodeGen/PowerPC/noredzone.ll
new file mode 100644
index 000000000000000..797a15569d615dd
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/noredzone.ll
@@ -0,0 +1,519 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64-ibm-aix < %s | FileCheck -check-prefix=AIX64 %s
+; RUN: llc -verify-machineinstrs -mtriple=powerpc-ibm-aix < %s | FileCheck -check-prefix=AIX32 %s
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu < %s | FileCheck -check-prefix=LE64 %s
+
+define signext i32 @leaf1_noredzone(i32 signext %a, i32 signext %b) #0 {
+; AIX64-LABEL: leaf1_noredzone:
+; AIX64:       # %bb.0: # %entry
+; AIX64-NEXT:    stdu 1, -64(1)
+; AIX64-NEXT:    stw 3, 60(1)
+; AIX64-NEXT:    add 3, 3, 4
+; AIX64-NEXT:    extsw 3, 3
+; AIX64-NEXT:    stw 4, 56(1)
+; AIX64-NEXT:    addi 1, 1, 64
+; AIX64-NEXT:    blr
+;
+; AIX32-LABEL: leaf1_noredzone:
+; AIX32:       # %bb.0: # %entry
+; AIX32-NEXT:    stwu 1, -32(1)
+; AIX32-NEXT:    stw 3, 28(1)
+; AIX32-NEXT:    add 3, 3, 4
+; AIX32-NEXT:    stw 4, 24(1)
+; AIX32-NEXT:    addi 1, 1, 32
+; AIX32-NEXT:    blr
+;
+; LE64-LABEL: leaf1_noredzone:
+; LE64:       # %bb.0: # %entry
+; LE64-NEXT:    stdu 1, -48(1)
+; LE64-NEXT:    stw 3, 44(1)
+; LE64-NEXT:    add 3, 3, 4
+; LE64-NEXT:    stw 4, 40(1)
+; LE64-NEXT:    extsw 3, 3
+; LE64-NEXT:    addi 1, 1, 48
+; LE64-NEXT:    blr
+entry:
+  %a.addr = alloca i32, align 4
+  %b.addr = alloca i32, align 4
+  store i32 %a, ptr %a.addr, align 4
+  store i32 %b, ptr %b.addr, align 4
+  %0 = load i32, ptr %a.addr, align 4
+  %1 = load i32, ptr %b.addr, align 4
+  %add = add nsw i32 %0, %1
+  ret i32 %add
+}
+
+define void @nonleaf1_noredzone(i32 signext %a, i32 signext %b) #0 {
+; AIX64-LABEL: nonleaf1_noredzone:
+; AIX64:       # %bb.0: # %entry
+; AIX64-NEXT:    mflr 0
+; AIX64-NEXT:    stdu 1, -128(1)
+; AIX64-NEXT:    std 0, 144(1)
+; AIX64-NEXT:    stw 3, 124(1)
+; AIX64-NEXT:    add 3, 3, 4
+; AIX64-NEXT:    extsw 3, 3
+; AIX64-NEXT:    stw 4, 120(1)
+; AIX64-NEXT:    bl .leaf2[PR]
+; AIX64-NEXT:    nop
+; AIX64-NEXT:    lwz 3, 124(1)
+; AIX64-NEXT:    lwz 4, 120(1)
+; AIX64-NEXT:    sub 3, 3, 4
+; AIX64-NEXT:    extsw 3, 3
+; AIX64-NEXT:    bl .leaf2[PR]
+; AIX64-NEXT:    nop
+; AIX64-NEXT:    addi 1, 1, 128
+; AIX64-NEXT:    ld 0, 16(1)
+; AIX64-NEXT:    mtlr 0
+; AIX64-NEXT:    blr
+;
+; AIX32-LABEL: nonleaf1_noredzone:
+; AIX32:       # %bb.0: # %entry
+; AIX32-NEXT:    mflr 0
+; AIX32-NEXT:    stwu 1, -64(1)
+; AIX32-NEXT:    stw 0, 72(1)
+; AIX32-NEXT:    stw 3, 60(1)
+; AIX32-NEXT:    add 3, 3, 4
+; AIX32-NEXT:    stw 4, 56(1)
+; AIX32-NEXT:    bl .leaf2[PR]
+; AIX32-NEXT:    nop
+; AIX32-NEXT:    lwz 3, 60(1)
+; AIX32-NEXT:    lwz 4, 56(1)
+; AIX32-NEXT:    sub 3, 3, 4
+; AIX32-NEXT:    bl .leaf2[PR]
+; AIX32-NEXT:    nop
+; AIX32-NEXT:    addi 1, 1, 64
+; AIX32-NEXT:    lwz 0, 8(1)
+; AIX32-NEXT:    mtlr 0
+; AIX32-NEXT:    blr
+;
+; LE64-LABEL: nonleaf1_noredzone:
+; LE64:       # %bb.0: # %entry
+; LE64-NEXT:    mflr 0
+; LE64-NEXT:    stdu 1, -48(1)
+; LE64-NEXT:    std 0, 64(1)
+; LE64-NEXT:    stw 3, 44(1)
+; LE64-NEXT:    add 3, 3, 4
+; LE64-NEXT:    extsw 3, 3
+; LE64-NEXT:    stw 4, 40(1)
+; LE64-NEXT:    bl leaf2
+; LE64-NEXT:    nop
+; LE64-NEXT:    lwz 3, 44(1)
+; LE64-NEXT:    lwz 4, 40(1)
+; LE64-NEXT:    sub 3, 3, 4
+; LE64-NEXT:    extsw 3, 3
+; LE64-NEXT:    bl leaf2
+; LE64-NEXT:    nop
+; LE64-NEXT:    addi 1, 1, 48
+; LE64-NEXT:    ld 0, 16(1)
+; LE64-NEXT:    mtlr 0
+; LE64-NEXT:    blr
+entry:
+  %a.addr = alloca i32, align 4
+  %b.addr = alloca i32, align 4
+  store i32 %a, ptr %a.addr, align 4
+  store i32 %b, ptr %b.addr, align 4
+  %0 = load i32, ptr %a.addr, align 4
+  %1 = load i32, ptr %b.addr, align 4
+  %add = add nsw i32 %0, %1
+  call void @leaf2(i32 signext %add)
+  %2 = load i32, ptr %a.addr, align 4
+  %3 = load i32, ptr %b.addr, align 4
+  %sub = sub nsw i32 %2, %3
+  call void @leaf2(i32 signext %sub)
+  ret void
+}
+
+declare void @leaf2(i32 signext)
+
+define signext i32 @leaf3_noredzone(i32 signext %a, i32 signext %b) #0 {
+; AIX64-LABEL: leaf3_noredzone:
+; AIX64:       # %bb.0: # %entry
+; AIX64-NEXT:    stdu 1, -48(1)
+; AIX64-NEXT:    ld 6, 0(1)
+; AIX64-NEXT:    mr 5, 3
+; AIX64-NEXT:    add 3, 5, 4
+; AIX64-NEXT:    extsw 3, 3
+; AIX64-NEXT:    stw 5, 48(6)
+; AIX64-NEXT:    stw 4, 52(6)
+; AIX64-NEXT:    addi 1, 1, 48
+; AIX64-NEXT:    blr
+;
+; AIX32-LABEL: leaf3_noredzone:
+; AIX32:       # %bb.0: # %entry
+; AIX32-NEXT:    stwu 1, -32(1)
+; AIX32-NEXT:    lwz 6, 0(1)
+; AIX32-NEXT:    mr 5, 3
+; AIX32-NEXT:    add 3, 3, 4
+; AIX32-NEXT:    stw 5, 24(6)
+; AIX32-NEXT:    stw 4, 28(6)
+; AIX32-NEXT:    addi 1, 1, 32
+; AIX32-NEXT:    blr
+;
+; LE64-LABEL: leaf3_noredzone:
+; LE64:       # %bb.0: # %entry
+; LE64-NEXT:    stdu 1, -32(1)
+; LE64-NEXT:    ld 5, 0(1)
+; LE64-NEXT:    stw 3, 48(5)
+; LE64-NEXT:    add 3, 3, 4
+; LE64-NEXT:    stw 4, 52(5)
+; LE64-NEXT:    extsw 3, 3
+; LE64-NEXT:    addi 1, 1, 32
+; LE64-NEXT:    blr
+entry:
+  %f.addr = call ptr @llvm.frameaddress(i32 1)
+  %a.addr = getelementptr ptr, ptr %f.addr, i32 6
+  %b.addr = getelementptr i32, ptr %a.addr, i32 1
+  store i32 %a, ptr %a.addr, align 4
+  store i32 %b, ptr %b.addr, align 4
+  %0 = load i32, ptr %a.addr, align 4
+  %1 = load i32, ptr %b.addr, align 4
+  %add = add nsw i32 %0, %1
+  ret i32 %add
+}
+
+define void @nonleaf2_noredzone(i32 signext %a, i32 signext %b) #0 {
+; AIX64-LABEL: nonleaf2_noredzone:
+; AIX64:       # %bb.0: # %entry
+; AIX64-NEXT:    mflr 0
+; AIX64-NEXT:    stdu 1, -128(1)
+; AIX64-NEXT:    std 0, 144(1)
+; AIX64-NEXT:    std 31, 120(1) # 8-byte Folded Spill
+; AIX64-NEXT:    add 5, 3, 4
+; AIX64-NEXT:    ld 31, 0(1)
+; AIX64-NEXT:    extsw 5, 5
+; AIX64-NEXT:    stw 3, 48(31)
+; AIX64-NEXT:    mr 3, 5
+; AIX64-NEXT:    stw 4, 52(31)
+; AIX64-NEXT:    bl .leaf2[PR]
+; AIX64-NEXT:    nop
+; AIX64-NEXT:    lwz 3, 48(31)
+; AIX64-NEXT:    lwz 4, 52(31)
+; AIX64-NEXT:    sub 3, 3, 4
+; AIX64-NEXT:    extsw 3, 3
+; AIX64-NEXT:    bl .leaf2[PR]
+; AIX64-NEXT:    nop
+; AIX64-NEXT:    ld 31, 120(1) # 8-byte Folded Reload
+; AIX64-NEXT:    addi 1, 1, 128
+; AIX64-NEXT:    ld 0, 16(1)
+; AIX64-NEXT:    mtlr 0
+; AIX64-NEXT:    blr
+;
+; AIX32-LABEL: nonleaf2_noredzone:
+; AIX32:       # %bb.0: # %entry
+; AIX32-NEXT:    mflr 0
+; AIX32-NEXT:    stwu 1, -64(1)
+; AIX32-NEXT:    stw 0, 72(1)
+; AIX32-NEXT:    add 5, 3, 4
+; AIX32-NEXT:    stw 31, 60(1) # 4-byte Folded Spill
+; AIX32-NEXT:    lwz 31, 0(1)
+; AIX32-NEXT:    stw 3, 24(31)
+; AIX32-NEXT:    mr 3, 5
+; AIX32-NEXT:    stw 4, 28(31)
+; AIX32-NEXT:    bl .leaf2[PR]
+; AIX32-NEXT:    nop
+; AIX32-NEXT:    lwz 3, 24(31)
+; AIX32-NEXT:    lwz 4, 28(31)
+; AIX32-NEXT:    sub 3, 3, 4
+; AIX32-NEXT:    bl .leaf2[PR]
+; AIX32-NEXT:    nop
+; AIX32-NEXT:    lwz 31, 60(1) # 4-byte Folded Reload
+; AIX32-NEXT:    addi 1, 1, 64
+; AIX32-NEXT:    lwz 0, 8(1)
+; AIX32-NEXT:    mtlr 0
+; AIX32-NEXT:    blr
+;
+; LE64-LABEL: nonleaf2_noredzone:
+; LE64:       # %bb.0: # %entry
+; LE64-NEXT:    mflr 0
+; LE64-NEXT:    std 30, -16(1) # 8-byte Folded Spill
+; LE64-NEXT:    stdu 1, -48(1)
+; LE64-NEXT:    std 0, 64(1)
+; LE64-NEXT:    ld 30, 0(1)
+; LE64-NEXT:    stw 3, 48(30)
+; LE64-NEXT:    add 3, 3, 4
+; LE64-NEXT:    stw 4, 52(30)
+; LE64-NEXT:    extsw 3, 3
+; LE64-NEXT:    bl leaf2
+; LE64-NEXT:    nop
+; LE64-NEXT:    lwz 3, 48(30)
+; LE64-NEXT:    lwz 4, 52(30)
+; LE64-NEXT:    sub 3, 3, 4
+; LE64-NEXT:    extsw 3, 3
+; LE64-NEXT:    bl leaf2
+; LE64-NEXT:    nop
+; LE64-NEXT:    addi 1, 1, 48
+; LE64-NEXT:    ld 0, 16(1)
+; LE64-NEXT:    ld 30, -16(1) # 8-byte Folded Reload
+; LE64-NEXT:    mtlr 0
+; LE64-NEXT:    blr
+entry:
+  %f.addr = call ptr @llvm.frameaddress(i32 1)
+  %a.addr = getelementptr ptr, ptr %f.addr, i32 6
+  %b.addr = getelementptr i32, ptr %a.addr, i32 1
+  store i32 %a, ptr %a.addr, align 4
+  store i32 %b, ptr %b.addr, align 4
+  %0 = load i32, ptr %a.addr, align 4
+  %1 = load i32, ptr %b.addr, align 4
+  %add = add nsw i32 %0, %1
+  call void @leaf2(i32 signext %add)
+  %2 = load i32, ptr %a.addr, align 4
+  %3 = load i32, ptr %b.addr, align 4
+  %sub = sub nsw i32 %2, %3
+  call void @leaf2(i32 signext %sub)
+  ret void
+}
+
+define signext i32 @leaf1_redzone(i32 signext %a, i32 signext %b) #1 {
+; AIX64-LABEL: leaf1_redzone:
+; AIX64:       # %bb.0: # %entry
+; AIX64-NEXT:    stw 3, -4(1)
+; AIX64-NEXT:    add 3, 3, 4
+; AIX64-NEXT:    extsw 3, 3
+; AIX64-NEXT:    stw 4, -8(1)
+; AIX64-NEXT:    blr
+;
+; AIX32-LABEL: leaf1_redzone:
+; AIX32:       # %bb.0: # %entry
+; AIX32-NEXT:    stw 3, -4(1)
+; AIX32-NEXT:    add 3, 3, 4
+; AIX32-NEXT:    stw 4, -8(1)
+; AIX32-NEXT:    blr
+;
+; LE64-LABEL: leaf1_redzone:
+; LE64:       # %bb.0: # %entry
+; LE64-NEXT:    stw 3, -4(1)
+; LE64-NEXT:    add 3, 3, 4
+; LE64-NEXT:    stw 4, -8(1)
+; LE64-NEXT:    extsw 3, 3
+; LE64-NEXT:    blr
+entry:
+  %a.addr = alloca i32, align 4
+  %b.addr = alloca i32, align 4
+  store i32 %a, ptr %a.addr, align 4
+  store i32 %b, ptr %b.addr, align 4
+  %0 = load i32, ptr %a.addr, align 4
+  %1 = load i32, ptr %b.addr, align 4
+  %add = add nsw i32 %0, %1
+  ret i32 %add
+}
+
+define void @nonleaf1_redzone(i32 signext %a, i32 signext %b) #1 {
+; AIX64-LABEL: nonleaf1_redzone:
+; AIX64:       # %bb.0: # %entry
+; AIX64-NEXT:    mflr 0
+; AIX64-NEXT:    stdu 1, -128(1)
+; AIX64-NEXT:    std 0, 144(1)
+; AIX64-NEXT:    stw 3, 124(1)
+; AIX64-NEXT:    add 3, 3, 4
+; AIX64-NEXT:    extsw 3, 3
+; AIX64-NEXT:    stw 4, 120(1)
+; AIX64-NEXT:    bl .leaf2[PR]
+; AIX64-NEXT:    nop
+; AIX64-NEXT:    lwz 3, 124(1)
+; AIX64-NEXT:    lwz 4, 120(1)
+; AIX64-NEXT:    sub 3, 3, 4
+; AIX64-NEXT:    extsw 3, 3
+; AIX64-NEXT:    bl .leaf2[PR]
+; AIX64-NEXT:    nop
+; AIX64-NEXT:    addi 1, 1, 128
+; AIX64-NEXT:    ld 0, 16(1)
+; AIX64-NEXT:    mtlr 0
+; AIX64-NEXT:    blr
+;
+; AIX32-LABEL: nonleaf1_redzone:
+; AIX32:       # %bb.0: # %entry
+; AIX32-NEXT:    mflr 0
+; AIX32-NEXT:    stwu 1, -64(1)
+; AIX32-NEXT:    stw 0, 72(1)
+; AIX32-NEXT:    stw 3, 60(1)
+; AIX32-NEXT:    add 3, 3, 4
+; AIX32-NEXT:    stw 4, 56(1)
+; AIX32-NEXT:    bl .leaf2[PR]
+; AIX32-NEXT:    nop
+; AIX32-NEXT:    lwz 3, 60(1)
+; AIX32-NEXT:    lwz 4, 56(1)
+; AIX32-NEXT:    sub 3, 3, 4
+; AIX32-NEXT:    bl .leaf2[PR]
+; AIX32-NEXT:    nop
+; AIX32-NEXT:    addi 1, 1, 64
+; AIX32-NEXT:    lwz 0, 8(1)
+; AIX32-NEXT:    mtlr 0
+; AIX32-NEXT:    blr
+;
+; LE64-LABEL: nonleaf1_redzone:
+; LE64:       # %bb.0: # %entry
+; LE64-NEXT:    mflr 0
+; LE64-NEXT:    stdu 1, -48(1)
+; LE64-NEXT:    std 0, 64(1)
+; LE64-NEXT:    stw 3, 44(1)
+; LE64-NEXT:    add 3, 3, 4
+; LE64-NEXT:    extsw 3, 3
+; LE64-NEXT:    stw 4, 40(1)
+; LE64-NEXT:    bl leaf2
+; LE64-NEXT:    nop
+; LE64-NEXT:    lwz 3, 44(1)
+; LE64-NEXT:    lwz 4, 40(1)
+; LE64-NEXT:    sub 3, 3, 4
+; LE64-NEXT:    extsw 3, 3
+; LE64-NEXT:    bl leaf2
+; LE64-NEXT:    nop
+; LE64-NEXT:    addi 1, 1, 48
+; LE64-NEXT:    ld 0, 16(1)
+; LE64-NEXT:    mtlr 0
+; LE64-NEXT:    blr
+entry:
+  %a.addr = alloca i32, align 4
+  %b.addr = alloca i32, align 4
+  store i32 %a, ptr %a.addr, align 4
+  store i32 %b, ptr %b.addr, align 4
+  %0 = load i32, ptr %a.addr, align 4
+  %1 = load i32, ptr %b.addr, align 4
+  %add = add nsw i32 %0, %1
+  call void @leaf2(i32 signext %add)
+  %2 = load i32, ptr %a.addr, align 4
+  %3 = load i32, ptr %b.addr, align 4
+  %sub = sub nsw i32 %2, %3
+  call void @leaf2(i32 signext %sub)
+  ret void
+}
+
+define signext i32 @leaf3_redzone(i32 signext %a, i32 signext %b) #1 {
+; AIX64-LABEL: leaf3_redzone:
+; AIX64:       # %bb.0: # %entry
+; AIX64-NEXT:    stdu 1, -48(1)
+; AIX64-NEXT:    ld 6, 0(1)
+; AIX64-NEXT:    mr 5, 3
+; AIX64-NEXT:    add 3, 5, 4
+; AIX64-NEXT:    extsw 3, 3
+; AIX64-NEXT:    stw 5, 48(6)
+; AIX64-NEXT:    stw 4, 52(6)
+; AIX64-NEXT:    addi 1, 1, 48
+; AIX64-NEXT:    blr
+;
+; AIX32-LABEL: leaf3_redzone:
+; AIX32:       # %bb.0: # %entry
+; AIX32-NEXT:    stwu 1, -32(1)
+; AIX32-NEXT:    lwz 6, 0(1)
+; AIX32-NEXT:    mr 5, 3
+; AIX32-NEXT:    add 3, 3, 4
+; AIX32-NEXT:    stw 5, 24(6)
+; AIX32-NEXT:    stw 4, 28(6)
+; AIX32-NEXT:    addi 1, 1, 32
+; AIX32-NEXT:    blr
+;
+; LE64-LABEL: leaf3_redzone:
+; LE64:       # %bb.0: # %entry
+; LE64-NEXT:    stdu 1, -32(1)
+; LE64-NEXT:    ld 5, 0(1)
+; LE64-NEXT:    stw 3, 48(5)
+; LE64-NEXT:    add 3, 3, 4
+; LE64-NEXT:    stw 4, 52(5)
+; LE64-NEXT:    extsw 3, 3
+; LE64-NEXT:    addi 1, 1, 32
+; LE64-NEXT:    blr
+entry:
+  %f.addr = call ptr @llvm.frameaddress(i32 1)
+  %a.addr = getelementptr ptr, ptr %f.addr, i32 6
+  %b.addr = getelementptr i32, ptr %a.addr, i32 1
+  store i32 %a, ptr %a.addr, align 4
+  store i32 %b, ptr %b.addr, align 4
+  %0 = load i32, ptr %a.addr, align 4
+  %1 = load i32, ptr %b.addr, align 4
+  %add = add nsw i32 %0, %1
+  ret i32 %add
+}
+
+define void @nonleaf2_redzone(i32 signext %a, i32 signext %b) #1 {
+; AIX64-LABEL: nonleaf2_redzone:
+; AIX64:       # %bb.0: # %entry
+; AIX64-NEXT:    mflr 0
+; AIX64-NEXT:    stdu 1, -128(1)
+; AIX64-NEXT:    std 0, 144(1)
+; AIX64-NEXT:    std 31, 120(1) # 8-byte Folded Spill
+; AIX64-NEXT:    add 5, 3, 4
+; AIX64-NEXT:    ld 31, 0(1)
+; AIX64-NEXT:    extsw 5, 5
+; AIX64-NEXT:    stw 3, 48(31)
+; AIX64-NEXT:    mr 3, 5
+; AIX64-NEXT:    stw 4, 52(31)
+; AIX64-NEXT:    bl .leaf2[PR]
+; AIX64-NEXT:    nop
+; AIX64-NEXT:    lwz 3, 48(31)
+; AIX64-NEXT:    lwz 4, 52(31)
+; AIX64-NEXT:    sub 3, 3, 4
+; AIX64-NEXT:    extsw 3, 3
+; AIX64-NEXT:    bl .leaf2[PR]
+; AIX64-NEXT:    nop
+; AIX64-NEXT:    ld 31, 120(1) # 8-byte Folded Reload
+; AIX64-NEXT:    addi 1, 1, 128
+; AIX64-NEXT:    ld 0, 16(1)
+; AIX64-NEXT:    mtlr 0
+; AIX64-NEXT:    blr
+;
+; AIX32-LABEL: nonleaf2_redzone:
+; AIX32:       # %bb.0: # %entry
+; AIX32-NEXT:    mflr 0
+; AIX32-NEXT:    stwu 1, -64(1)
+; AIX32-NEXT:    stw 0, 72(1)
+; AIX32-NEXT:    add 5, 3, 4
+; AIX32-NEXT:    stw 31, 60(1) # 4-byte Folded Spill
+; AIX32-NEXT:    lwz 31, 0(1)
+; AIX32-NEXT:    stw 3, 24(31)
+; AIX32-NEXT:    mr 3, 5
+; AIX32-NEXT:    stw 4, 28(31)
+; AIX32-NEXT:    bl .leaf2[PR]
+; AIX32-NEXT:    nop
+; AIX32-NEXT:    lwz 3, 24(31)
+; AIX32-NEXT:    lwz 4, 28(31)
+; AIX32-NEXT:    sub 3, 3, 4
+; AIX32-NEXT:    bl .leaf2[PR]
+; AIX32-NEXT:    nop
+; AIX32-NEXT:    lwz 31, 60(1) # 4-byte Folded Reload
+; AIX32-NEXT:    addi 1, 1, 64
+; AIX32-NEXT:    lwz 0, 8(1)
+; AIX32-NEXT:    mtlr 0
+; AIX32-NEXT:    blr
+;
+; LE64-LABEL: nonleaf2_redzone:
+; LE64:       # %bb.0: # %entry
+; LE64-NEXT:    mflr 0
+; LE64-NEXT:    std 30, -16(1) # 8-byte Folded Spill
+; LE64-NEXT:    stdu 1, -48(1)
+; LE64-NEXT:    std 0, 64(1)
+; LE64-NEXT:    ld 30, 0(1)
+; LE64-NEXT:    stw 3, 48(30)
+; LE64-NEXT:    add 3, 3, 4
+; LE64-NEXT:    stw 4, 52(30)
+; LE64-NEXT:    extsw 3, 3
+; LE64-NEXT:    bl leaf2
+; LE64-NEXT:    nop
+; LE64-NEXT:    lwz 3, 48(30)
+; LE64-NEXT:    lwz 4, 52(30)
+; LE64-NEXT:    sub 3, 3, 4
+; LE64-NEXT:    extsw 3, 3
+; LE64-NEXT:    bl leaf2
+; LE64-NEXT:    nop
+; LE64-NEXT:    addi 1, 1, 48
+; LE64-NEXT:    ld 0, 16(1)
+; LE64-NEXT:    ld 30, -16(1) # 8-byte Folded Reload
+; LE64-NEXT:    mtlr 0
+; LE64-NEXT:    blr
+entry:
+  %f.addr = call ptr @llvm.frameaddress(i32 1)
+  %a.addr = getelementptr ptr, ptr %f.addr, i32 6
+  %b.addr = getelementptr i32, ptr %a.addr, i32 1
+  store i32 %a, ptr %a.addr, align 4
+  store i32 %b, ptr %b.addr, align 4
+  %0 = load i32, ptr %a.addr, align 4
+  %1 = load i32, ptr %b.addr, align 4
+  %add = add nsw i32 %0, %1
+  call void @leaf2(i32 signext %add)
+  %2 = load i32, ptr %a.addr, align 4
+  %3 = load i32, ptr %b.addr, align 4
+  %sub = sub nsw i32 %2, %3
+  call void @leaf2(i32 signext %sub)
+  ret void
+}
+
+attributes #0 = { noredzone nounwind noinline }
+attributes #1 = { nounwind noinline }

From cde806b0e63fddcca013536b4bd55e45607205d1 Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett@linaro.org>
Date: Mon, 19 Aug 2024 10:04:29 +0000
Subject: [PATCH 343/441] [lld][MachO] Fix a suspicous assert in
 SyntheticSections.cpp

This was comparing some .size() (uint64_t) against the sizeof a size_t
which changes with system bitness. This produced a warning that
brought this to my attention.

These tests were failing too on 32 bit Arm only:
  lld :: MachO/objc-category-merging-complete-test.s
  lld :: MachO/objc-category-merging-minimal.s

The assert I think meant to check the value of target->wordSize,
not the size of its type. Which is a type that changes size between
systems.
---
 lld/MachO/SyntheticSections.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lld/MachO/SyntheticSections.cpp b/lld/MachO/SyntheticSections.cpp
index 6b4ec4989ca4a11..939e9b286d77f59 100644
--- a/lld/MachO/SyntheticSections.cpp
+++ b/lld/MachO/SyntheticSections.cpp
@@ -2119,7 +2119,7 @@ void ObjCMethListSection::writeRelativeOffsetForIsec(
     assert(selRef && "Expected all selector names to already be already be "
                      "present in __objc_selrefs");
     symVA = selRef->getVA();
-    assert(selRef->data.size() == sizeof(target->wordSize) &&
+    assert(selRef->data.size() == target->wordSize &&
            "Expected one selref per ConcatInputSection");
   } else if (reloc->referent.is<Symbol *>()) {
     auto *def = dyn_cast_or_null<Defined>(reloc->referent.get<Symbol *>());

From 74a512df2bea0f0d1f314c6b8dc33c97f9086336 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell@arm.com>
Date: Mon, 19 Aug 2024 11:20:18 +0100
Subject: [PATCH 344/441] [mlir][vector] Populate sink patterns in
 apply_patterns.vector.reduction_to_contract (#104754)

This restores the functionality to before:
https://github.com/llvm/llvm-project/commit/42944da5ba7617bbc02f341e9ef401c325310a73
This fixes a buildbot failure:
https://lab.llvm.org/buildbot/#/builders/143/builds/1487
---
 .../mlir/Dialect/Vector/TransformOps/VectorTransformOps.td   | 5 +++--
 mlir/lib/Dialect/Vector/TransformOps/VectorTransformOps.cpp  | 1 +
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/mlir/include/mlir/Dialect/Vector/TransformOps/VectorTransformOps.td b/mlir/include/mlir/Dialect/Vector/TransformOps/VectorTransformOps.td
index 820a18731ffdb03..aad2eab83dbd383 100644
--- a/mlir/include/mlir/Dialect/Vector/TransformOps/VectorTransformOps.td
+++ b/mlir/include/mlir/Dialect/Vector/TransformOps/VectorTransformOps.td
@@ -410,7 +410,7 @@ def ApplyFoldElementwiseToVectorPatternsOp : Op<Transform_Dialect,
     "apply_patterns.vector.elementwise_to_vector",
     [DeclareOpInterfaceMethods<PatternDescriptorOpInterface>]> {
   let description = [{
-    Collect a set of patterns that fold elementwise op on vectors to the vector 
+    Collect a set of patterns that fold elementwise op on vectors to the vector
     dialect.
   }];
 
@@ -426,8 +426,9 @@ def ApplyVectorReductionToContractPatternsOp : Op<Transform_Dialect,
       - CombineContractBroadcast
       - CombineContractABTranspose
       - CombineContractResultTranspose
-      - ReorderCastOpsOnBroadcast
       - ReorderElementwiseOpsOnTranspose
+      - ReorderElementwiseOpsOnBroadcast
+      - ReorderCastOpsOnBroadcast
 
     These patterns have the effect of rewriting a vector.multi_reduce into a
     vector.contract.
diff --git a/mlir/lib/Dialect/Vector/TransformOps/VectorTransformOps.cpp b/mlir/lib/Dialect/Vector/TransformOps/VectorTransformOps.cpp
index bc423a3781bf045..3ae70ace3934cdd 100644
--- a/mlir/lib/Dialect/Vector/TransformOps/VectorTransformOps.cpp
+++ b/mlir/lib/Dialect/Vector/TransformOps/VectorTransformOps.cpp
@@ -67,6 +67,7 @@ void transform::ApplyFoldElementwiseToVectorPatternsOp::populatePatterns(
 void transform::ApplyVectorReductionToContractPatternsOp::populatePatterns(
     RewritePatternSet &patterns) {
   vector::populateVectorReductionToContractPatterns(patterns);
+  vector::populateSinkVectorOpsPatterns(patterns);
 }
 
 void transform::ApplyLowerCreateMaskPatternsOp::populatePatterns(

From 00def06c6e9a0582e85bef5f3a186839c48d1ae2 Mon Sep 17 00:00:00 2001
From: Tim Besard <tim.besard@gmail.com>
Date: Mon, 19 Aug 2024 12:34:23 +0200
Subject: [PATCH 345/441] [LLVM][NewPM] Add C API for running the pipeline on a
 single function. (#103773)

By adding a new entrypoint, `LLVMRunPassesOnFunction`, as suggested in
https://discourse.llvm.org/t/newpm-c-api-questions/80598.

Also removes erroneous `LLVMConsumeError`s from the pass builder unit
tests as the string conversion already consumes the error, causing an
abort when the test would fail.
---
 llvm/docs/ReleaseNotes.rst                    |  6 +++
 llvm/include/llvm-c/Transforms/PassBuilder.h  | 10 ++++
 llvm/lib/Passes/PassBuilderBindings.cpp       | 50 ++++++++++++++-----
 .../PassBuilderBindingsTest.cpp               | 16 +++++-
 4 files changed, 68 insertions(+), 14 deletions(-)

diff --git a/llvm/docs/ReleaseNotes.rst b/llvm/docs/ReleaseNotes.rst
index 057bd5345fba289..d40bb2682f9ad82 100644
--- a/llvm/docs/ReleaseNotes.rst
+++ b/llvm/docs/ReleaseNotes.rst
@@ -157,6 +157,12 @@ Changes to the C API
   * ``LLVMGetNamedFunctionWithLength``
   * ``LLVMGetNamedGlobalWithLength``
 
+* The new pass manager can now be invoked with a custom alias analysis pipeline, using
+  the ``LLVMPassBuilderOptionsSetAAPipeline`` function.
+
+* It is now also possible to run the new pass manager on a single function, by calling
+  ``LLVMRunPassesOnFunction`` instead of ``LLVMRunPasses``.
+
 Changes to the CodeGen infrastructure
 -------------------------------------
 
diff --git a/llvm/include/llvm-c/Transforms/PassBuilder.h b/llvm/include/llvm-c/Transforms/PassBuilder.h
index 03a5abaa753312a..d297b57cadd07d3 100644
--- a/llvm/include/llvm-c/Transforms/PassBuilder.h
+++ b/llvm/include/llvm-c/Transforms/PassBuilder.h
@@ -50,6 +50,16 @@ LLVMErrorRef LLVMRunPasses(LLVMModuleRef M, const char *Passes,
                            LLVMTargetMachineRef TM,
                            LLVMPassBuilderOptionsRef Options);
 
+/**
+ * Construct and run a set of passes over a function.
+ *
+ * This function behaves the same as LLVMRunPasses, but operates on a single
+ * function instead of an entire module.
+ */
+LLVMErrorRef LLVMRunPassesOnFunction(LLVMValueRef F, const char *Passes,
+                                     LLVMTargetMachineRef TM,
+                                     LLVMPassBuilderOptionsRef Options);
+
 /**
  * Create a new set of options for a PassBuilder
  *
diff --git a/llvm/lib/Passes/PassBuilderBindings.cpp b/llvm/lib/Passes/PassBuilderBindings.cpp
index 4e12dd2226c4d22..933fe89e53a94cf 100644
--- a/llvm/lib/Passes/PassBuilderBindings.cpp
+++ b/llvm/lib/Passes/PassBuilderBindings.cpp
@@ -48,15 +48,12 @@ static TargetMachine *unwrap(LLVMTargetMachineRef P) {
 DEFINE_SIMPLE_CONVERSION_FUNCTIONS(LLVMPassBuilderOptions,
                                    LLVMPassBuilderOptionsRef)
 
-LLVMErrorRef LLVMRunPasses(LLVMModuleRef M, const char *Passes,
-                           LLVMTargetMachineRef TM,
-                           LLVMPassBuilderOptionsRef Options) {
-  TargetMachine *Machine = unwrap(TM);
-  LLVMPassBuilderOptions *PassOpts = unwrap(Options);
+static LLVMErrorRef runPasses(Module *Mod, Function *Fun, const char *Passes,
+                              TargetMachine *Machine,
+                              LLVMPassBuilderOptions *PassOpts) {
   bool Debug = PassOpts->DebugLogging;
   bool VerifyEach = PassOpts->VerifyEach;
 
-  Module *Mod = unwrap(M);
   PassInstrumentationCallbacks PIC;
   PassBuilder PB(Machine, PassOpts->PTO, std::nullopt, &PIC);
 
@@ -80,18 +77,45 @@ LLVMErrorRef LLVMRunPasses(LLVMModuleRef M, const char *Passes,
 
   StandardInstrumentations SI(Mod->getContext(), Debug, VerifyEach);
   SI.registerCallbacks(PIC, &MAM);
-  ModulePassManager MPM;
-  if (VerifyEach) {
-    MPM.addPass(VerifierPass());
-  }
-  if (auto Err = PB.parsePassPipeline(MPM, Passes)) {
-    return wrap(std::move(Err));
+
+  // Run the pipeline.
+  if (Fun) {
+    FunctionPassManager FPM;
+    if (VerifyEach)
+      FPM.addPass(VerifierPass());
+    if (auto Err = PB.parsePassPipeline(FPM, Passes))
+      return wrap(std::move(Err));
+    FPM.run(*Fun, FAM);
+  } else {
+    ModulePassManager MPM;
+    if (VerifyEach)
+      MPM.addPass(VerifierPass());
+    if (auto Err = PB.parsePassPipeline(MPM, Passes))
+      return wrap(std::move(Err));
+    MPM.run(*Mod, MAM);
   }
 
-  MPM.run(*Mod, MAM);
   return LLVMErrorSuccess;
 }
 
+LLVMErrorRef LLVMRunPasses(LLVMModuleRef M, const char *Passes,
+                           LLVMTargetMachineRef TM,
+                           LLVMPassBuilderOptionsRef Options) {
+  TargetMachine *Machine = unwrap(TM);
+  LLVMPassBuilderOptions *PassOpts = unwrap(Options);
+  Module *Mod = unwrap(M);
+  return runPasses(Mod, nullptr, Passes, Machine, PassOpts);
+}
+
+LLVMErrorRef LLVMRunPassesOnFunction(LLVMValueRef F, const char *Passes,
+                                     LLVMTargetMachineRef TM,
+                                     LLVMPassBuilderOptionsRef Options) {
+  TargetMachine *Machine = unwrap(TM);
+  LLVMPassBuilderOptions *PassOpts = unwrap(Options);
+  Function *Fun = unwrap<Function>(F);
+  return runPasses(Fun->getParent(), Fun, Passes, Machine, PassOpts);
+}
+
 LLVMPassBuilderOptionsRef LLVMCreatePassBuilderOptions() {
   return wrap(new LLVMPassBuilderOptions());
 }
diff --git a/llvm/unittests/Passes/PassBuilderBindings/PassBuilderBindingsTest.cpp b/llvm/unittests/Passes/PassBuilderBindings/PassBuilderBindingsTest.cpp
index 2b06033f0c3fa26..4e17b1ad09e2b35 100644
--- a/llvm/unittests/Passes/PassBuilderBindings/PassBuilderBindingsTest.cpp
+++ b/llvm/unittests/Passes/PassBuilderBindings/PassBuilderBindingsTest.cpp
@@ -35,6 +35,9 @@ class PassBuilderCTest : public testing::Test {
     LLVMDisposeMessage(Triple);
     Context = LLVMContextCreate();
     Module = LLVMModuleCreateWithNameInContext("test", Context);
+    LLVMTypeRef FT =
+        LLVMFunctionType(LLVMVoidTypeInContext(Context), nullptr, 0, 0);
+    Function = LLVMAddFunction(Module, "test", FT);
   }
 
   void TearDown() override {
@@ -52,6 +55,7 @@ class PassBuilderCTest : public testing::Test {
 public:
   LLVMTargetMachineRef TM;
   LLVMModuleRef Module;
+  LLVMValueRef Function;
   LLVMContextRef Context;
 };
 
@@ -63,7 +67,6 @@ TEST_F(PassBuilderCTest, Basic) {
   LLVMPassBuilderOptionsSetAAPipeline(Options, "basic-aa");
   if (LLVMErrorRef E = LLVMRunPasses(Module, "default<O2>", TM, Options)) {
     char *Msg = LLVMGetErrorMessage(E);
-    LLVMConsumeError(E);
     LLVMDisposePassBuilderOptions(Options);
     FAIL() << "Failed to run passes: " << Msg;
   }
@@ -80,3 +83,14 @@ TEST_F(PassBuilderCTest, InvalidPassIsError) {
   LLVMConsumeError(E2);
   LLVMDisposePassBuilderOptions(Options);
 }
+
+TEST_F(PassBuilderCTest, Function) {
+  LLVMPassBuilderOptionsRef Options = LLVMCreatePassBuilderOptions();
+  if (LLVMErrorRef E =
+          LLVMRunPassesOnFunction(Function, "no-op-function", TM, Options)) {
+    char *Msg = LLVMGetErrorMessage(E);
+    LLVMDisposePassBuilderOptions(Options);
+    FAIL() << "Failed to run passes on function: " << Msg;
+  }
+  LLVMDisposePassBuilderOptions(Options);
+}

From b8dccb7d56c7e32b549c00a094d1b6ac32a01328 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Mon, 19 Aug 2024 12:40:19 +0100
Subject: [PATCH 346/441] [VPlan] Emit note when UserVF > MaxUserVF (NFCI).

As suggested in https://github.com/llvm/llvm-project/pull/103033, add a
remark when the UserVF is ignored due to it being larger than MaxUserVF.

Only changes behavior of diagnostic/debug output.
---
 .../Transforms/Vectorize/LoopVectorize.cpp    | 45 ++++++++++---------
 .../LoopVectorize/unsafe-vf-hint-remark.ll    |  1 +
 2 files changed, 26 insertions(+), 20 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 05a6c5f36d26ebc..0bae9ad1844ec48 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7014,27 +7014,32 @@ LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
 
   ElementCount MaxUserVF =
       UserVF.isScalable() ? MaxFactors.ScalableVF : MaxFactors.FixedVF;
-  bool UserVFIsLegal = ElementCount::isKnownLE(UserVF, MaxUserVF);
-  if (!UserVF.isZero() && UserVFIsLegal) {
-    assert(isPowerOf2_32(UserVF.getKnownMinValue()) &&
-           "VF needs to be a power of two");
-    // Collect the instructions (and their associated costs) that will be more
-    // profitable to scalarize.
-    CM.collectInLoopReductions();
-    if (CM.selectUserVectorizationFactor(UserVF)) {
-      LLVM_DEBUG(dbgs() << "LV: Using user VF " << UserVF << ".\n");
-      buildVPlansWithVPRecipes(UserVF, UserVF);
-      if (!hasPlanWithVF(UserVF)) {
-        LLVM_DEBUG(dbgs() << "LV: No VPlan could be built for " << UserVF
-                          << ".\n");
-        return std::nullopt;
-      }
+  if (!UserVF.isZero()) {
+    if (!ElementCount::isKnownLE(UserVF, MaxUserVF)) {
+      reportVectorizationInfo(
+          "UserVF ignored because it may be larger than the maximal safe VF",
+          "InvalidUserVF", ORE, OrigLoop);
+    } else {
+      assert(isPowerOf2_32(UserVF.getKnownMinValue()) &&
+             "VF needs to be a power of two");
+      // Collect the instructions (and their associated costs) that will be more
+      // profitable to scalarize.
+      CM.collectInLoopReductions();
+      if (CM.selectUserVectorizationFactor(UserVF)) {
+        LLVM_DEBUG(dbgs() << "LV: Using user VF " << UserVF << ".\n");
+        buildVPlansWithVPRecipes(UserVF, UserVF);
+        if (!hasPlanWithVF(UserVF)) {
+          LLVM_DEBUG(dbgs()
+                     << "LV: No VPlan could be built for " << UserVF << ".\n");
+          return std::nullopt;
+        }
 
-      LLVM_DEBUG(printPlans(dbgs()));
-      return {{UserVF, 0, 0}};
-    } else
-      reportVectorizationInfo("UserVF ignored because of invalid costs.",
-                              "InvalidCost", ORE, OrigLoop);
+        LLVM_DEBUG(printPlans(dbgs()));
+        return {{UserVF, 0, 0}};
+      } else
+        reportVectorizationInfo("UserVF ignored because of invalid costs.",
+                                "InvalidCost", ORE, OrigLoop);
+    }
   }
 
   // Collect the Vectorization Factor Candidates.
diff --git a/llvm/test/Transforms/LoopVectorize/unsafe-vf-hint-remark.ll b/llvm/test/Transforms/LoopVectorize/unsafe-vf-hint-remark.ll
index 0c9fa742765d311..8104441ad19934b 100644
--- a/llvm/test/Transforms/LoopVectorize/unsafe-vf-hint-remark.ll
+++ b/llvm/test/Transforms/LoopVectorize/unsafe-vf-hint-remark.ll
@@ -14,6 +14,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3
 
 ; CHECK: LV: User VF=4 is unsafe, clamping to max safe VF=2.
 ; CHECK: remark: <unknown>:0:0: User-specified vectorization factor 4 is unsafe, clamping to maximum safe vectorization factor 2
+; CHECK: remark: <unknown>:0:0: UserVF ignored because it may be larger than the maximal safe VF
 ; CHECK-LABEL: @foo
 ; CHECK: <2 x i32>
 define void @foo(ptr %a, ptr %b) {

From abf69a167bbc99054871e3f9cc8810bbebcb6747 Mon Sep 17 00:00:00 2001
From: Volodymyr Vasylkun <vvmposeydon@gmail.com>
Date: Mon, 19 Aug 2024 13:02:29 +0100
Subject: [PATCH 347/441] [InstCombine] Fold `(x < y) ? -1 : zext(x != y)` into
 `u/scmp(x,y)` (#101049)

This patch adds the aforementioned fold to InstCombine. This pattern is
produced after naive implementations of 3-way comparison in high-level
languages are transformed into LLVM IR and then optimized.

Proofs: https://alive2.llvm.org/ce/z/w4QLq_
---
 .../InstCombine/InstCombineInternal.h         |   1 +
 .../InstCombine/InstCombineSelect.cpp         |  52 +++
 llvm/test/Transforms/InstCombine/scmp.ll      |  56 ++++
 .../Transforms/InstCombine/select-select.ll   |  44 +--
 llvm/test/Transforms/InstCombine/ucmp.ll      | 306 ++++++++++++++++++
 .../PhaseOrdering/partialord-ule.ll           |   4 +-
 6 files changed, 426 insertions(+), 37 deletions(-)

diff --git a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
index a0f2399972b52f2..a0e846c3b5a566b 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
+++ b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
@@ -729,6 +729,7 @@ class LLVM_LIBRARY_VISIBILITY InstCombinerImpl final
 
   // Helpers of visitSelectInst().
   Instruction *foldSelectOfBools(SelectInst &SI);
+  Instruction *foldSelectToCmp(SelectInst &SI);
   Instruction *foldSelectExtConst(SelectInst &Sel);
   Instruction *foldSelectOpOp(SelectInst &SI, Instruction *TI, Instruction *FI);
   Instruction *foldSelectIntoOp(SelectInst &SI, Value *, Value *);
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
index b8b77bec1b900d6..1f6d5759883fd0c 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
@@ -3558,6 +3558,55 @@ static Instruction *foldBitCeil(SelectInst &SI, IRBuilderBase &Builder) {
                                 Masked);
 }
 
+// This function tries to fold the following operations:
+//   (x < y) ? -1 : zext(x != y)
+//   (x > y) ? 1 : sext(x != y)
+// Into ucmp/scmp(x, y), where signedness is determined by the signedness
+// of the comparison in the original sequence.
+Instruction *InstCombinerImpl::foldSelectToCmp(SelectInst &SI) {
+  Value *TV = SI.getTrueValue();
+  Value *FV = SI.getFalseValue();
+
+  ICmpInst::Predicate Pred;
+  Value *LHS, *RHS;
+  if (!match(SI.getCondition(), m_ICmp(Pred, m_Value(LHS), m_Value(RHS))))
+    return nullptr;
+
+  if (!LHS->getType()->isIntOrIntVectorTy())
+    return nullptr;
+
+  // Try to swap operands and the predicate. We need to be careful when doing
+  // so because two of the patterns have opposite predicates, so use the
+  // constant inside select to determine if swapping operands would be
+  // beneficial to us.
+  if ((ICmpInst::isGT(Pred) && match(TV, m_AllOnes())) ||
+      (ICmpInst::isLT(Pred) && match(TV, m_One()))) {
+    Pred = ICmpInst::getSwappedPredicate(Pred);
+    std::swap(LHS, RHS);
+  }
+
+  Intrinsic::ID IID =
+      ICmpInst::isSigned(Pred) ? Intrinsic::scmp : Intrinsic::ucmp;
+
+  bool Replace = false;
+  // (x < y) ? -1 : zext(x != y)
+  if (ICmpInst::isLT(Pred) && match(TV, m_AllOnes()) &&
+      match(FV, m_ZExt(m_c_SpecificICmp(ICmpInst::ICMP_NE, m_Specific(LHS),
+                                        m_Specific(RHS)))))
+    Replace = true;
+
+  // (x > y) ? 1 : sext(x != y)
+  if (ICmpInst::isGT(Pred) && match(TV, m_One()) &&
+      match(FV, m_SExt(m_c_SpecificICmp(ICmpInst::ICMP_NE, m_Specific(LHS),
+                                        m_Specific(RHS)))))
+    Replace = true;
+
+  if (Replace)
+    return replaceInstUsesWith(
+        SI, Builder.CreateIntrinsic(SI.getType(), IID, {LHS, RHS}));
+  return nullptr;
+}
+
 bool InstCombinerImpl::fmulByZeroIsZero(Value *MulVal, FastMathFlags FMF,
                                         const Instruction *CtxI) const {
   KnownFPClass Known = computeKnownFPClass(MulVal, FMF, fcNegative, CtxI);
@@ -4061,6 +4110,9 @@ Instruction *InstCombinerImpl::visitSelectInst(SelectInst &SI) {
   if (Instruction *I = foldBitCeil(SI, Builder))
     return I;
 
+  if (Instruction *I = foldSelectToCmp(SI))
+    return I;
+
   // Fold:
   // (select A && B, T, F) -> (select A, (select B, T, F), F)
   // (select A || B, T, F) -> (select A, T, (select B, T, F))
diff --git a/llvm/test/Transforms/InstCombine/scmp.ll b/llvm/test/Transforms/InstCombine/scmp.ll
index 460ea96b16fcfa9..7f374c5f9a1d645 100644
--- a/llvm/test/Transforms/InstCombine/scmp.ll
+++ b/llvm/test/Transforms/InstCombine/scmp.ll
@@ -208,3 +208,59 @@ define i8 @scmp_negated_multiuse(i32 %x, i32 %y) {
   %2 = sub i8 0, %1
   ret i8 %2
 }
+
+; Fold ((x s< y) ? -1 : (x != y)) into scmp(x, y)
+define i8 @scmp_from_select_lt(i32 %x, i32 %y) {
+; CHECK-LABEL: define i8 @scmp_from_select_lt(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.scmp.i8.i32(i32 [[X]], i32 [[Y]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ne_bool = icmp ne i32 %x, %y
+  %ne = zext i1 %ne_bool to i8
+  %lt = icmp slt i32 %x, %y
+  %r = select i1 %lt, i8 -1, i8 %ne
+  ret i8 %r
+}
+
+; Vector version
+define <4 x i8> @scmp_from_select_vec_lt(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: define <4 x i8> @scmp_from_select_vec_lt(
+; CHECK-SAME: <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = call <4 x i8> @llvm.scmp.v4i8.v4i32(<4 x i32> [[X]], <4 x i32> [[Y]])
+; CHECK-NEXT:    ret <4 x i8> [[R]]
+;
+  %ne_bool = icmp ne <4 x i32> %x, %y
+  %ne = zext <4 x i1> %ne_bool to <4 x i8>
+  %lt = icmp slt <4 x i32> %x, %y
+  %r = select <4 x i1> %lt, <4 x i8> splat(i8 -1), <4 x i8> %ne
+  ret <4 x i8> %r
+}
+
+; Fold (x s<= y) ? sext(x != y) : 1 into scmp(x, y)
+define i8 @scmp_from_select_le(i32 %x, i32 %y) {
+; CHECK-LABEL: define i8 @scmp_from_select_le(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.scmp.i8.i32(i32 [[X]], i32 [[Y]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ne_bool = icmp ne i32 %x, %y
+  %ne = sext i1 %ne_bool to i8
+  %le = icmp sle i32 %x, %y
+  %r = select i1 %le, i8 %ne, i8 1
+  ret i8 %r
+}
+
+; Fold (x s>= y) ? zext(x != y) : -1 into scmp(x, y)
+define i8 @scmp_from_select_ge(i32 %x, i32 %y) {
+; CHECK-LABEL: define i8 @scmp_from_select_ge(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.scmp.i8.i32(i32 [[X]], i32 [[Y]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ne_bool = icmp ne i32 %x, %y
+  %ne = zext i1 %ne_bool to i8
+  %ge = icmp sge i32 %x, %y
+  %r = select i1 %ge, i8 %ne, i8 -1
+  ret i8 %r
+}
diff --git a/llvm/test/Transforms/InstCombine/select-select.ll b/llvm/test/Transforms/InstCombine/select-select.ll
index 84fe973093e327f..5460ba1bc558384 100644
--- a/llvm/test/Transforms/InstCombine/select-select.ll
+++ b/llvm/test/Transforms/InstCombine/select-select.ll
@@ -282,10 +282,7 @@ define i8 @strong_order_cmp_ugt_eq(i32 %a, i32 %b) {
 
 define i8 @strong_order_cmp_eq_slt(i32 %a, i32 %b) {
 ; CHECK-LABEL: @strong_order_cmp_eq_slt(
-; CHECK-NEXT:    [[CMP_EQ:%.*]] = icmp ne i32 [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[SEL_EQ:%.*]] = zext i1 [[CMP_EQ]] to i8
-; CHECK-NEXT:    [[CMP_LT:%.*]] = icmp slt i32 [[A]], [[B]]
-; CHECK-NEXT:    [[SEL_LT:%.*]] = select i1 [[CMP_LT]], i8 -1, i8 [[SEL_EQ]]
+; CHECK-NEXT:    [[SEL_LT:%.*]] = call i8 @llvm.scmp.i8.i32(i32 [[A:%.*]], i32 [[B:%.*]])
 ; CHECK-NEXT:    ret i8 [[SEL_LT]]
 ;
   %cmp.eq = icmp eq i32 %a, %b
@@ -297,10 +294,7 @@ define i8 @strong_order_cmp_eq_slt(i32 %a, i32 %b) {
 
 define i8 @strong_order_cmp_eq_sgt(i32 %a, i32 %b) {
 ; CHECK-LABEL: @strong_order_cmp_eq_sgt(
-; CHECK-NEXT:    [[CMP_EQ:%.*]] = icmp ne i32 [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[SEL_EQ:%.*]] = sext i1 [[CMP_EQ]] to i8
-; CHECK-NEXT:    [[CMP_GT:%.*]] = icmp sgt i32 [[A]], [[B]]
-; CHECK-NEXT:    [[SEL_GT:%.*]] = select i1 [[CMP_GT]], i8 1, i8 [[SEL_EQ]]
+; CHECK-NEXT:    [[SEL_GT:%.*]] = call i8 @llvm.scmp.i8.i32(i32 [[A:%.*]], i32 [[B:%.*]])
 ; CHECK-NEXT:    ret i8 [[SEL_GT]]
 ;
   %cmp.eq = icmp eq i32 %a, %b
@@ -312,10 +306,7 @@ define i8 @strong_order_cmp_eq_sgt(i32 %a, i32 %b) {
 
 define i8 @strong_order_cmp_eq_ult(i32 %a, i32 %b) {
 ; CHECK-LABEL: @strong_order_cmp_eq_ult(
-; CHECK-NEXT:    [[CMP_EQ:%.*]] = icmp ne i32 [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[SEL_EQ:%.*]] = zext i1 [[CMP_EQ]] to i8
-; CHECK-NEXT:    [[CMP_LT:%.*]] = icmp ult i32 [[A]], [[B]]
-; CHECK-NEXT:    [[SEL_LT:%.*]] = select i1 [[CMP_LT]], i8 -1, i8 [[SEL_EQ]]
+; CHECK-NEXT:    [[SEL_LT:%.*]] = call i8 @llvm.ucmp.i8.i32(i32 [[A:%.*]], i32 [[B:%.*]])
 ; CHECK-NEXT:    ret i8 [[SEL_LT]]
 ;
   %cmp.eq = icmp eq i32 %a, %b
@@ -327,10 +318,7 @@ define i8 @strong_order_cmp_eq_ult(i32 %a, i32 %b) {
 
 define i8 @strong_order_cmp_eq_ugt(i32 %a, i32 %b) {
 ; CHECK-LABEL: @strong_order_cmp_eq_ugt(
-; CHECK-NEXT:    [[CMP_EQ:%.*]] = icmp ne i32 [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[SEL_EQ:%.*]] = sext i1 [[CMP_EQ]] to i8
-; CHECK-NEXT:    [[CMP_GT:%.*]] = icmp ugt i32 [[A]], [[B]]
-; CHECK-NEXT:    [[SEL_GT:%.*]] = select i1 [[CMP_GT]], i8 1, i8 [[SEL_EQ]]
+; CHECK-NEXT:    [[SEL_GT:%.*]] = call i8 @llvm.ucmp.i8.i32(i32 [[A:%.*]], i32 [[B:%.*]])
 ; CHECK-NEXT:    ret i8 [[SEL_GT]]
 ;
   %cmp.eq = icmp eq i32 %a, %b
@@ -404,9 +392,7 @@ define i8 @strong_order_cmp_ne_ugt_ne_not_one_use(i32 %a, i32 %b) {
 ; CHECK-LABEL: @strong_order_cmp_ne_ugt_ne_not_one_use(
 ; CHECK-NEXT:    [[CMP_NE:%.*]] = icmp ne i32 [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    call void @use1(i1 [[CMP_NE]])
-; CHECK-NEXT:    [[SEL_EQ:%.*]] = sext i1 [[CMP_NE]] to i8
-; CHECK-NEXT:    [[CMP_GT:%.*]] = icmp ugt i32 [[A]], [[B]]
-; CHECK-NEXT:    [[SEL_GT:%.*]] = select i1 [[CMP_GT]], i8 1, i8 [[SEL_EQ]]
+; CHECK-NEXT:    [[SEL_GT:%.*]] = call i8 @llvm.ucmp.i8.i32(i32 [[A]], i32 [[B]])
 ; CHECK-NEXT:    ret i8 [[SEL_GT]]
 ;
   %cmp.ne = icmp ne i32 %a, %b
@@ -535,10 +521,7 @@ define <2 x i8> @strong_order_cmp_ugt_ult_vector_poison(<2 x i32> %a, <2 x i32>
 
 define <2 x i8> @strong_order_cmp_eq_ugt_vector(<2 x i32> %a, <2 x i32> %b) {
 ; CHECK-LABEL: @strong_order_cmp_eq_ugt_vector(
-; CHECK-NEXT:    [[CMP_EQ:%.*]] = icmp ne <2 x i32> [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[SEL_EQ:%.*]] = sext <2 x i1> [[CMP_EQ]] to <2 x i8>
-; CHECK-NEXT:    [[CMP_GT:%.*]] = icmp ugt <2 x i32> [[A]], [[B]]
-; CHECK-NEXT:    [[SEL_GT:%.*]] = select <2 x i1> [[CMP_GT]], <2 x i8> <i8 1, i8 1>, <2 x i8> [[SEL_EQ]]
+; CHECK-NEXT:    [[SEL_GT:%.*]] = call <2 x i8> @llvm.ucmp.v2i8.v2i32(<2 x i32> [[A:%.*]], <2 x i32> [[B:%.*]])
 ; CHECK-NEXT:    ret <2 x i8> [[SEL_GT]]
 ;
   %cmp.eq = icmp eq <2 x i32> %a, %b
@@ -550,10 +533,7 @@ define <2 x i8> @strong_order_cmp_eq_ugt_vector(<2 x i32> %a, <2 x i32> %b) {
 
 define <2 x i8> @strong_order_cmp_eq_ugt_vector_poison1(<2 x i32> %a, <2 x i32> %b) {
 ; CHECK-LABEL: @strong_order_cmp_eq_ugt_vector_poison1(
-; CHECK-NEXT:    [[CMP_EQ:%.*]] = icmp ne <2 x i32> [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[SEL_EQ:%.*]] = sext <2 x i1> [[CMP_EQ]] to <2 x i8>
-; CHECK-NEXT:    [[CMP_GT:%.*]] = icmp ugt <2 x i32> [[A]], [[B]]
-; CHECK-NEXT:    [[SEL_GT:%.*]] = select <2 x i1> [[CMP_GT]], <2 x i8> <i8 1, i8 1>, <2 x i8> [[SEL_EQ]]
+; CHECK-NEXT:    [[SEL_GT:%.*]] = call <2 x i8> @llvm.ucmp.v2i8.v2i32(<2 x i32> [[A:%.*]], <2 x i32> [[B:%.*]])
 ; CHECK-NEXT:    ret <2 x i8> [[SEL_GT]]
 ;
   %cmp.eq = icmp eq <2 x i32> %a, %b
@@ -565,10 +545,7 @@ define <2 x i8> @strong_order_cmp_eq_ugt_vector_poison1(<2 x i32> %a, <2 x i32>
 
 define <2 x i8> @strong_order_cmp_eq_ugt_vector_poison2(<2 x i32> %a, <2 x i32> %b) {
 ; CHECK-LABEL: @strong_order_cmp_eq_ugt_vector_poison2(
-; CHECK-NEXT:    [[CMP_EQ:%.*]] = icmp ne <2 x i32> [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[SEL_EQ:%.*]] = sext <2 x i1> [[CMP_EQ]] to <2 x i8>
-; CHECK-NEXT:    [[CMP_GT:%.*]] = icmp ugt <2 x i32> [[A]], [[B]]
-; CHECK-NEXT:    [[SEL_GT:%.*]] = select <2 x i1> [[CMP_GT]], <2 x i8> <i8 1, i8 1>, <2 x i8> [[SEL_EQ]]
+; CHECK-NEXT:    [[SEL_GT:%.*]] = call <2 x i8> @llvm.ucmp.v2i8.v2i32(<2 x i32> [[A:%.*]], <2 x i32> [[B:%.*]])
 ; CHECK-NEXT:    ret <2 x i8> [[SEL_GT]]
 ;
   %cmp.eq = icmp eq <2 x i32> %a, %b
@@ -580,10 +557,7 @@ define <2 x i8> @strong_order_cmp_eq_ugt_vector_poison2(<2 x i32> %a, <2 x i32>
 
 define <2 x i8> @strong_order_cmp_eq_ugt_vector_poison3(<2 x i32> %a, <2 x i32> %b) {
 ; CHECK-LABEL: @strong_order_cmp_eq_ugt_vector_poison3(
-; CHECK-NEXT:    [[CMP_EQ:%.*]] = icmp ne <2 x i32> [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    [[SEL_EQ:%.*]] = sext <2 x i1> [[CMP_EQ]] to <2 x i8>
-; CHECK-NEXT:    [[CMP_GT:%.*]] = icmp ugt <2 x i32> [[A]], [[B]]
-; CHECK-NEXT:    [[SEL_GT:%.*]] = select <2 x i1> [[CMP_GT]], <2 x i8> <i8 1, i8 poison>, <2 x i8> [[SEL_EQ]]
+; CHECK-NEXT:    [[SEL_GT:%.*]] = call <2 x i8> @llvm.ucmp.v2i8.v2i32(<2 x i32> [[A:%.*]], <2 x i32> [[B:%.*]])
 ; CHECK-NEXT:    ret <2 x i8> [[SEL_GT]]
 ;
   %cmp.eq = icmp eq <2 x i32> %a, %b
diff --git a/llvm/test/Transforms/InstCombine/ucmp.ll b/llvm/test/Transforms/InstCombine/ucmp.ll
index 8690e23ea2e0e53..ad8a57825253b0f 100644
--- a/llvm/test/Transforms/InstCombine/ucmp.ll
+++ b/llvm/test/Transforms/InstCombine/ucmp.ll
@@ -207,3 +207,309 @@ define i8 @ucmp_negated_multiuse(i32 %x, i32 %y) {
   %2 = sub i8 0, %1
   ret i8 %2
 }
+
+; Fold ((x u< y) ? -1 : (x != y)) into ucmp(x, y)
+define i8 @ucmp_from_select_lt(i32 %x, i32 %y) {
+; CHECK-LABEL: define i8 @ucmp_from_select_lt(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.ucmp.i8.i32(i32 [[X]], i32 [[Y]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ne_bool = icmp ne i32 %x, %y
+  %ne = zext i1 %ne_bool to i8
+  %lt = icmp ult i32 %x, %y
+  %r = select i1 %lt, i8 -1, i8 %ne
+  ret i8 %r
+}
+
+; Vector version
+define <4 x i8> @ucmp_from_select_vec_lt(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: define <4 x i8> @ucmp_from_select_vec_lt(
+; CHECK-SAME: <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = call <4 x i8> @llvm.ucmp.v4i8.v4i32(<4 x i32> [[X]], <4 x i32> [[Y]])
+; CHECK-NEXT:    ret <4 x i8> [[R]]
+;
+  %ne_bool = icmp ne <4 x i32> %x, %y
+  %ne = zext <4 x i1> %ne_bool to <4 x i8>
+  %lt = icmp ult <4 x i32> %x, %y
+  %r = select <4 x i1> %lt, <4 x i8> splat(i8 -1), <4 x i8> %ne
+  ret <4 x i8> %r
+}
+
+; Commuted operands
+define i8 @ucmp_from_select_lt_commuted_ops1(i32 %x, i32 %y) {
+; CHECK-LABEL: define i8 @ucmp_from_select_lt_commuted_ops1(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.ucmp.i8.i32(i32 [[X]], i32 [[Y]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ne_bool = icmp ne i32 %y, %x
+  %ne = zext i1 %ne_bool to i8
+  %lt = icmp ult i32 %x, %y
+  %r = select i1 %lt, i8 -1, i8 %ne
+  ret i8 %r
+}
+
+define i8 @ucmp_from_select_lt_commuted_ops2(i32 %x, i32 %y) {
+; CHECK-LABEL: define i8 @ucmp_from_select_lt_commuted_ops2(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.ucmp.i8.i32(i32 [[X]], i32 [[Y]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ne_bool = icmp ne i32 %x, %y
+  %ne = zext i1 %ne_bool to i8
+  %lt = icmp ugt i32 %y, %x
+  %r = select i1 %lt, i8 -1, i8 %ne
+  ret i8 %r
+}
+
+; Negative test: false value of the select is not `icmp ne x, y`
+define i8 @ucmp_from_select_lt_neg1(i32 %x, i32 %y) {
+; CHECK-LABEL: define i8 @ucmp_from_select_lt_neg1(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[NE_BOOL:%.*]] = icmp eq i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[NE:%.*]] = zext i1 [[NE_BOOL]] to i8
+; CHECK-NEXT:    [[LT:%.*]] = icmp ult i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[LT]], i8 -1, i8 [[NE]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ne_bool = icmp eq i32 %x, %y
+  %ne = zext i1 %ne_bool to i8
+  %lt = icmp ult i32 %x, %y
+  %r = select i1 %lt, i8 -1, i8 %ne
+  ret i8 %r
+}
+
+; Negative test: true value of select is not -1
+define i8 @ucmp_from_select_lt_neg2(i32 %x, i32 %y) {
+; CHECK-LABEL: define i8 @ucmp_from_select_lt_neg2(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[NE_BOOL:%.*]] = icmp ne i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[NE:%.*]] = zext i1 [[NE_BOOL]] to i8
+; CHECK-NEXT:    [[LT:%.*]] = icmp ult i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[LT]], i8 2, i8 [[NE]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ne_bool = icmp ne i32 %x, %y
+  %ne = zext i1 %ne_bool to i8
+  %lt = icmp ult i32 %x, %y
+  %r = select i1 %lt, i8 2, i8 %ne
+  ret i8 %r
+}
+
+; Negative test: false value of select is sign-extended instead of zero-extended
+define i8 @ucmp_from_select_lt_neg3(i32 %x, i32 %y) {
+; CHECK-LABEL: define i8 @ucmp_from_select_lt_neg3(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[NE_BOOL:%.*]] = icmp ne i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[R:%.*]] = sext i1 [[NE_BOOL]] to i8
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ne_bool = icmp ne i32 %x, %y
+  %ne = sext i1 %ne_bool to i8
+  %lt = icmp ult i32 %x, %y
+  %r = select i1 %lt, i8 -1, i8 %ne
+  ret i8 %r
+}
+
+; Negative test: condition of select is not (x u< y)
+define i8 @ucmp_from_select_lt_neg4(i32 %x, i32 %y) {
+; CHECK-LABEL: define i8 @ucmp_from_select_lt_neg4(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[NE_BOOL:%.*]] = icmp ne i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[NE:%.*]] = zext i1 [[NE_BOOL]] to i8
+; CHECK-NEXT:    [[LT_NOT:%.*]] = icmp ugt i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[LT_NOT]], i8 [[NE]], i8 -1
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ne_bool = icmp ne i32 %x, %y
+  %ne = zext i1 %ne_bool to i8
+  %lt = icmp ule i32 %x, %y
+  %r = select i1 %lt, i8 -1, i8 %ne
+  ret i8 %r
+}
+
+; Fold (x u<= y) ? sext(x != y) : 1 into ucmp(x, y)
+define i8 @ucmp_from_select_le(i32 %x, i32 %y) {
+; CHECK-LABEL: define i8 @ucmp_from_select_le(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.ucmp.i8.i32(i32 [[X]], i32 [[Y]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ne_bool = icmp ne i32 %x, %y
+  %ne = sext i1 %ne_bool to i8
+  %le = icmp ule i32 %x, %y
+  %r = select i1 %le, i8 %ne, i8 1
+  ret i8 %r
+}
+
+; Negative test: condition of select is not (x u<= y)
+define i8 @ucmp_from_select_le_neg1(i32 %x, i32 %y) {
+; CHECK-LABEL: define i8 @ucmp_from_select_le_neg1(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[NE_BOOL:%.*]] = icmp ult i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[NE:%.*]] = sext i1 [[NE_BOOL]] to i8
+; CHECK-NEXT:    [[LE_NOT:%.*]] = icmp ugt i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[LE_NOT]], i8 1, i8 [[NE]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ne_bool = icmp ult i32 %x, %y
+  %ne = sext i1 %ne_bool to i8
+  %le = icmp ule i32 %x, %y
+  %r = select i1 %le, i8 %ne, i8 1
+  ret i8 %r
+}
+
+; Negative test: true value of select is zero-extended instead of sign-extended
+define i8 @ucmp_from_select_le_neg2(i32 %x, i32 %y) {
+; CHECK-LABEL: define i8 @ucmp_from_select_le_neg2(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[NE_BOOL:%.*]] = icmp ne i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[R:%.*]] = zext i1 [[NE_BOOL]] to i8
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ne_bool = icmp ne i32 %x, %y
+  %ne = zext i1 %ne_bool to i8
+  %le = icmp ule i32 %x, %y
+  %r = select i1 %le, i8 %ne, i8 1
+  ret i8 %r
+}
+
+; Negative test: true value is not x != y
+define i8 @ucmp_from_select_le_neg3(i32 %x, i32 %y) {
+; CHECK-LABEL: define i8 @ucmp_from_select_le_neg3(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[NE_BOOL:%.*]] = icmp sgt i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[NE:%.*]] = sext i1 [[NE_BOOL]] to i8
+; CHECK-NEXT:    [[LE_NOT:%.*]] = icmp ugt i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[LE_NOT]], i8 1, i8 [[NE]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ne_bool = icmp sgt i32 %x, %y
+  %ne = sext i1 %ne_bool to i8
+  %le = icmp ule i32 %x, %y
+  %r = select i1 %le, i8 %ne, i8 1
+  ret i8 %r
+}
+
+; Negative test: false value is not 1
+define i8 @ucmp_from_select_le_neg4(i32 %x, i32 %y) {
+; CHECK-LABEL: define i8 @ucmp_from_select_le_neg4(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[NE_BOOL:%.*]] = icmp ne i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[R:%.*]] = sext i1 [[NE_BOOL]] to i8
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ne_bool = icmp ne i32 %x, %y
+  %ne = sext i1 %ne_bool to i8
+  %le = icmp ule i32 %x, %y
+  %r = select i1 %le, i8 %ne, i8 -1
+  ret i8 %r
+}
+
+; Fold (x u>= y) ? zext(x != y) : -1 into ucmp(x, y)
+define i8 @ucmp_from_select_ge(i32 %x, i32 %y) {
+; CHECK-LABEL: define i8 @ucmp_from_select_ge(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.ucmp.i8.i32(i32 [[X]], i32 [[Y]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ne_bool = icmp ne i32 %x, %y
+  %ne = zext i1 %ne_bool to i8
+  %ge = icmp uge i32 %x, %y
+  %r = select i1 %ge, i8 %ne, i8 -1
+  ret i8 %r
+}
+
+; Commuted operands
+define i8 @ucmp_from_select_ge_commuted_ops1(i32 %x, i32 %y) {
+; CHECK-LABEL: define i8 @ucmp_from_select_ge_commuted_ops1(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.ucmp.i8.i32(i32 [[X]], i32 [[Y]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ne_bool = icmp ne i32 %y, %x
+  %ne = zext i1 %ne_bool to i8
+  %ge = icmp uge i32 %x, %y
+  %r = select i1 %ge, i8 %ne, i8 -1
+  ret i8 %r
+}
+
+define i8 @ucmp_from_select_ge_commuted_ops2(i32 %x, i32 %y) {
+; CHECK-LABEL: define i8 @ucmp_from_select_ge_commuted_ops2(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.ucmp.i8.i32(i32 [[X]], i32 [[Y]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ne_bool = icmp ne i32 %x, %y
+  %ne = zext i1 %ne_bool to i8
+  %ge = icmp ule i32 %y, %x
+  %r = select i1 %ge, i8 %ne, i8 -1
+  ret i8 %r
+}
+
+; Negative test: condition is not x u>= y
+define i8 @ucmp_from_select_ge_neg1(i32 %x, i32 %y) {
+; CHECK-LABEL: define i8 @ucmp_from_select_ge_neg1(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[NE_BOOL:%.*]] = icmp ne i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[NE:%.*]] = zext i1 [[NE_BOOL]] to i8
+; CHECK-NEXT:    [[GE:%.*]] = icmp ult i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[GE]], i8 [[NE]], i8 -1
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ne_bool = icmp ne i32 %x, %y
+  %ne = zext i1 %ne_bool to i8
+  %ge = icmp ult i32 %x, %y
+  %r = select i1 %ge, i8 %ne, i8 -1
+  ret i8 %r
+}
+
+; Negative test: true value is sign-extended instead of zero-extended
+define i8 @ucmp_from_select_ge_neg2(i32 %x, i32 %y) {
+; CHECK-LABEL: define i8 @ucmp_from_select_ge_neg2(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[NE_BOOL:%.*]] = icmp ne i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[R:%.*]] = sext i1 [[NE_BOOL]] to i8
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ne_bool = icmp ne i32 %x, %y
+  %ne = sext i1 %ne_bool to i8
+  %ge = icmp uge i32 %x, %y
+  %r = select i1 %ge, i8 %ne, i8 -1
+  ret i8 %r
+}
+
+; Negative test: true value is not x != y
+define i8 @ucmp_from_select_ge_neg3(i32 %x, i32 %y) {
+; CHECK-LABEL: define i8 @ucmp_from_select_ge_neg3(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[NE_BOOL:%.*]] = icmp sgt i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[NE:%.*]] = zext i1 [[NE_BOOL]] to i8
+; CHECK-NEXT:    [[GE_NOT:%.*]] = icmp ult i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[GE_NOT]], i8 -1, i8 [[NE]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ne_bool = icmp sgt i32 %x, %y
+  %ne = zext i1 %ne_bool to i8
+  %ge = icmp uge i32 %x, %y
+  %r = select i1 %ge, i8 %ne, i8 -1
+  ret i8 %r
+}
+
+; Negative test: false value is not -1
+define i8 @ucmp_from_select_ge_neg4(i32 %x, i32 %y) {
+; CHECK-LABEL: define i8 @ucmp_from_select_ge_neg4(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[NE_BOOL:%.*]] = icmp ne i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[NE:%.*]] = zext i1 [[NE_BOOL]] to i8
+; CHECK-NEXT:    [[GE_NOT:%.*]] = icmp ult i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[GE_NOT]], i8 3, i8 [[NE]]
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %ne_bool = icmp ne i32 %x, %y
+  %ne = zext i1 %ne_bool to i8
+  %ge = icmp uge i32 %x, %y
+  %r = select i1 %ge, i8 %ne, i8 3
+  ret i8 %r
+}
diff --git a/llvm/test/Transforms/PhaseOrdering/partialord-ule.ll b/llvm/test/Transforms/PhaseOrdering/partialord-ule.ll
index 23a82c29012d967..6d4a8e31edbec05 100644
--- a/llvm/test/Transforms/PhaseOrdering/partialord-ule.ll
+++ b/llvm/test/Transforms/PhaseOrdering/partialord-ule.ll
@@ -6,8 +6,8 @@
 define i1 @ule(i32 %a, i32 %b) {
 ; CHECK-LABEL: @ule(
 ; CHECK-NEXT:  start:
-; CHECK-NEXT:    [[DOTNOT:%.*]] = icmp ule i32 [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    ret i1 [[DOTNOT]]
+; CHECK-NEXT:    [[SWITCH_SELECTCMP1:%.*]] = icmp ule i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret i1 [[SWITCH_SELECTCMP1]]
 ;
 start:
   %cmp1 = icmp eq i32 %a, %b

From f2fcd9cb97f76f34196233fc5c9170fb0905d844 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Mon, 19 Aug 2024 13:05:19 +0100
Subject: [PATCH 348/441] [VPlan] Rename getBestPlanFor -> getPlanFor (NFC).

As suggested in https://github.com/llvm/llvm-project/pull/103033, more
accurately rename to getPlanFor , as it simplify returns the VPlan for
VF, relying on the fact that there is a single VPlan for each VF at the
moment.
---
 llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h | 5 +++--
 llvm/lib/Transforms/Vectorize/LoopVectorize.cpp          | 8 ++++----
 llvm/lib/Transforms/Vectorize/VPlan.cpp                  | 4 ++--
 3 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index cd2680b0d43acf1..7d242082172c62b 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -362,8 +362,9 @@ class LoopVectorizationPlanner {
   /// VF and its cost.
   VectorizationFactor planInVPlanNativePath(ElementCount UserVF);
 
-  /// Return the best VPlan for \p VF.
-  VPlan &getBestPlanFor(ElementCount VF) const;
+  /// Return the VPlan for \p VF. At the moment, there is always a single VPlan
+  /// for each VF.
+  VPlan &getPlanFor(ElementCount VF) const;
 
   /// Compute and return the most profitable vectorization factor. Also collect
   /// all profitable VFs in ProfitableVFs.
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 0bae9ad1844ec48..d097ae116a0d22d 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -9487,7 +9487,7 @@ static bool processLoopInVPlanNativePath(
   if (VPlanBuildStressTest || VectorizationFactor::Disabled() == VF)
     return false;
 
-  VPlan &BestPlan = LVP.getBestPlanFor(VF.Width);
+  VPlan &BestPlan = LVP.getPlanFor(VF.Width);
 
   {
     bool AddBranchWeights =
@@ -9967,7 +9967,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
       ElementCount BestVF = LVP.computeBestVF();
       assert(BestVF.isScalar() &&
              "VPlan cost model and legacy cost model disagreed");
-      VPlan &BestPlan = LVP.getBestPlanFor(BestVF);
+      VPlan &BestPlan = LVP.getPlanFor(BestVF);
       LVP.executePlan(BestVF, IC, BestPlan, Unroller, DT, false);
 
       ORE->emit([&]() {
@@ -9983,7 +9983,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
       LLVM_DEBUG(dbgs() << "VF picked by VPlan cost model: " << BestVF << "\n");
       assert(VF.Width == BestVF &&
              "VPlan cost model and legacy cost model disagreed");
-      VPlan &BestPlan = LVP.getBestPlanFor(BestVF);
+      VPlan &BestPlan = LVP.getPlanFor(BestVF);
       // Consider vectorizing the epilogue too if it's profitable.
       VectorizationFactor EpilogueVF =
           LVP.selectEpilogueVectorizationFactor(BestVF, IC);
@@ -10009,7 +10009,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
                                                  ORE, EPI, &LVL, &CM, BFI, PSI,
                                                  Checks);
 
-        VPlan &BestEpiPlan = LVP.getBestPlanFor(EPI.EpilogueVF);
+        VPlan &BestEpiPlan = LVP.getPlanFor(EPI.EpilogueVF);
         VPRegionBlock *VectorLoop = BestEpiPlan.getVectorLoopRegion();
         VPBasicBlock *Header = VectorLoop->getEntryBasicBlock();
         Header->setName("vec.epilog.vector.body");
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index fe5edcedff54abd..15acfc00251e291 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -1680,11 +1680,11 @@ void LoopVectorizationPlanner::buildVPlans(ElementCount MinVF,
   }
 }
 
-VPlan &LoopVectorizationPlanner::getBestPlanFor(ElementCount VF) const {
+VPlan &LoopVectorizationPlanner::getPlanFor(ElementCount VF) const {
   assert(count_if(VPlans,
                   [VF](const VPlanPtr &Plan) { return Plan->hasVF(VF); }) ==
              1 &&
-         "Best VF has not a single VPlan.");
+         "Multiple VPlans for VF.");
 
   for (const VPlanPtr &Plan : VPlans) {
     if (Plan->hasVF(VF))

From 816068e462888f31f4bcf71f2f728eb77ff2ea15 Mon Sep 17 00:00:00 2001
From: tcwzxx <tcwzxx@gmail.com>
Date: Mon, 19 Aug 2024 20:16:51 +0800
Subject: [PATCH 349/441] [NFC][SLP] Remove useless code of the schedule
 (#104697)

Currently, the SLP schedule has two containers of `ScheduleData`:
`ExtraScheduleDataMap` and `ScheduleDataMap`. However, the
`ScheduleData` in `ExtraScheduleDataMap` is only used to indicate
whether the instruction is processed or not and does not participate in
the schedule, which is useless. `ScheduleDataMap` is sufficient for this
purpose. The `OpValue` member is used only in `ExtraScheduleDataMap`,
which is also useless.
---
 .../Transforms/Vectorize/SLPVectorizer.cpp    | 155 +++++-------------
 1 file changed, 38 insertions(+), 117 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 9ecd8160a978916..dee6d688b1b9032 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -795,16 +795,6 @@ struct InstructionsState {
 
 } // end anonymous namespace
 
-/// Chooses the correct key for scheduling data. If \p Op has the same (or
-/// alternate) opcode as \p OpValue, the key is \p Op. Otherwise the key is \p
-/// OpValue.
-static Value *isOneOf(const InstructionsState &S, Value *Op) {
-  auto *I = dyn_cast<Instruction>(Op);
-  if (I && S.isOpcodeOrAlt(I))
-    return Op;
-  return S.OpValue;
-}
-
 /// \returns true if \p Opcode is allowed as part of the main/alternate
 /// instruction for SLP vectorization.
 ///
@@ -3583,14 +3573,14 @@ class BoUpSLP {
 
     ScheduleData() = default;
 
-    void init(int BlockSchedulingRegionID, Value *OpVal) {
+    void init(int BlockSchedulingRegionID, Instruction *I) {
       FirstInBundle = this;
       NextInBundle = nullptr;
       NextLoadStore = nullptr;
       IsScheduled = false;
       SchedulingRegionID = BlockSchedulingRegionID;
       clearDependencies();
-      OpValue = OpVal;
+      Inst = I;
       TE = nullptr;
     }
 
@@ -3696,9 +3686,6 @@ class BoUpSLP {
 
     Instruction *Inst = nullptr;
 
-    /// Opcode of the current instruction in the schedule data.
-    Value *OpValue = nullptr;
-
     /// The TreeEntry that this instruction corresponds to.
     TreeEntry *TE = nullptr;
 
@@ -3815,18 +3802,6 @@ class BoUpSLP {
       return nullptr;
     }
 
-    ScheduleData *getScheduleData(Value *V, Value *Key) {
-      if (V == Key)
-        return getScheduleData(V);
-      auto I = ExtraScheduleDataMap.find(V);
-      if (I != ExtraScheduleDataMap.end()) {
-        ScheduleData *SD = I->second.lookup(Key);
-        if (SD && isInSchedulingRegion(SD))
-          return SD;
-      }
-      return nullptr;
-    }
-
     bool isInSchedulingRegion(ScheduleData *SD) const {
       return SD->SchedulingRegionID == SchedulingRegionID;
     }
@@ -3840,27 +3815,24 @@ class BoUpSLP {
 
       for (ScheduleData *BundleMember = SD; BundleMember;
            BundleMember = BundleMember->NextInBundle) {
-        if (BundleMember->Inst != BundleMember->OpValue)
-          continue;
 
         // Handle the def-use chain dependencies.
 
         // Decrement the unscheduled counter and insert to ready list if ready.
         auto &&DecrUnsched = [this, &ReadyList](Instruction *I) {
-          doForAllOpcodes(I, [&ReadyList](ScheduleData *OpDef) {
-            if (OpDef && OpDef->hasValidDependencies() &&
-                OpDef->incrementUnscheduledDeps(-1) == 0) {
-              // There are no more unscheduled dependencies after
-              // decrementing, so we can put the dependent instruction
-              // into the ready list.
-              ScheduleData *DepBundle = OpDef->FirstInBundle;
-              assert(!DepBundle->IsScheduled &&
-                     "already scheduled bundle gets ready");
-              ReadyList.insert(DepBundle);
-              LLVM_DEBUG(dbgs()
-                         << "SLP:    gets ready (def): " << *DepBundle << "\n");
-            }
-          });
+          ScheduleData *OpDef = getScheduleData(I);
+          if (OpDef && OpDef->hasValidDependencies() &&
+              OpDef->incrementUnscheduledDeps(-1) == 0) {
+            // There are no more unscheduled dependencies after
+            // decrementing, so we can put the dependent instruction
+            // into the ready list.
+            ScheduleData *DepBundle = OpDef->FirstInBundle;
+            assert(!DepBundle->IsScheduled &&
+                   "already scheduled bundle gets ready");
+            ReadyList.insert(DepBundle);
+            LLVM_DEBUG(dbgs()
+                       << "SLP:    gets ready (def): " << *DepBundle << "\n");
+          }
         };
 
         // If BundleMember is a vector bundle, its operands may have been
@@ -3944,8 +3916,7 @@ class BoUpSLP {
                "primary schedule data not in window?");
         assert(isInSchedulingRegion(SD->FirstInBundle) &&
                "entire bundle in window!");
-        (void)SD;
-        doForAllOpcodes(I, [](ScheduleData *SD) { SD->verify(); });
+        SD->verify();
       }
 
       for (auto *SD : ReadyInsts) {
@@ -3955,29 +3926,17 @@ class BoUpSLP {
       }
     }
 
-    void doForAllOpcodes(Value *V,
-                         function_ref<void(ScheduleData *SD)> Action) {
-      if (ScheduleData *SD = getScheduleData(V))
-        Action(SD);
-      auto I = ExtraScheduleDataMap.find(V);
-      if (I != ExtraScheduleDataMap.end())
-        for (auto &P : I->second)
-          if (isInSchedulingRegion(P.second))
-            Action(P.second);
-    }
-
     /// Put all instructions into the ReadyList which are ready for scheduling.
     template <typename ReadyListType>
     void initialFillReadyList(ReadyListType &ReadyList) {
       for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
-        doForAllOpcodes(I, [&](ScheduleData *SD) {
-          if (SD->isSchedulingEntity() && SD->hasValidDependencies() &&
-              SD->isReady()) {
-            ReadyList.insert(SD);
-            LLVM_DEBUG(dbgs()
-                       << "SLP:    initially in ready list: " << *SD << "\n");
-          }
-        });
+        ScheduleData *SD = getScheduleData(I);
+        if (SD && SD->isSchedulingEntity() && SD->hasValidDependencies() &&
+            SD->isReady()) {
+          ReadyList.insert(SD);
+          LLVM_DEBUG(dbgs()
+                     << "SLP:    initially in ready list: " << *SD << "\n");
+        }
       }
     }
 
@@ -4035,10 +3994,6 @@ class BoUpSLP {
     /// ScheduleData structures are recycled.
     DenseMap<Instruction *, ScheduleData *> ScheduleDataMap;
 
-    /// Attaches ScheduleData to Instruction with the leading key.
-    DenseMap<Value *, SmallDenseMap<Value *, ScheduleData *>>
-        ExtraScheduleDataMap;
-
     /// The ready-list for scheduling (only used for the dry-run).
     SetVector<ScheduleData *> ReadyInsts;
 
@@ -11898,8 +11853,7 @@ Instruction &BoUpSLP::getLastInstructionInBundle(const TreeEntry *E) {
     auto *Bundle = BlocksSchedules[BB]->getScheduleData(V);
     if (Bundle && Bundle->isPartOfBundle())
       for (; Bundle; Bundle = Bundle->NextInBundle)
-        if (Bundle->OpValue == Bundle->Inst)
-          Res.second = Bundle->Inst;
+        Res.second = Bundle->Inst;
   }
 
   // LastInst can still be null at this point if there's either not an entry
@@ -14966,7 +14920,8 @@ BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
     // initial bundle to the region.
     if (ScheduleEnd != OldScheduleEnd) {
       for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode())
-        doForAllOpcodes(I, [](ScheduleData *SD) { SD->clearDependencies(); });
+        if (ScheduleData *SD = getScheduleData(I))
+          SD->clearDependencies();
       ReSchedule = true;
     }
     if (Bundle) {
@@ -15085,37 +15040,21 @@ BoUpSLP::ScheduleData *BoUpSLP::BlockScheduling::allocateScheduleDataChunks() {
   return &(ScheduleDataChunks.back()[ChunkPos++]);
 }
 
-bool BoUpSLP::BlockScheduling::extendSchedulingRegion(Value *V,
-                                                      const InstructionsState &S) {
-  if (getScheduleData(V, isOneOf(S, V)))
-    return true;
+bool BoUpSLP::BlockScheduling::extendSchedulingRegion(
+    Value *V, const InstructionsState &S) {
   Instruction *I = dyn_cast<Instruction>(V);
   assert(I && "bundle member must be an instruction");
   assert(!isa<PHINode>(I) && !isVectorLikeInstWithConstOps(I) &&
          !doesNotNeedToBeScheduled(I) &&
          "phi nodes/insertelements/extractelements/extractvalues don't need to "
          "be scheduled");
-  auto &&CheckScheduleForI = [this, &S](Instruction *I) -> bool {
-    ScheduleData *ISD = getScheduleData(I);
-    if (!ISD)
-      return false;
-    assert(isInSchedulingRegion(ISD) &&
-           "ScheduleData not in scheduling region");
-    ScheduleData *SD = allocateScheduleDataChunks();
-    SD->Inst = I;
-    SD->init(SchedulingRegionID, S.OpValue);
-    ExtraScheduleDataMap[I][S.OpValue] = SD;
-    return true;
-  };
-  if (CheckScheduleForI(I))
+  if (getScheduleData(I))
     return true;
   if (!ScheduleStart) {
     // It's the first instruction in the new region.
     initScheduleData(I, I->getNextNode(), nullptr, nullptr);
     ScheduleStart = I;
     ScheduleEnd = I->getNextNode();
-    if (isOneOf(S, I) != I)
-      CheckScheduleForI(I);
     assert(ScheduleEnd && "tried to vectorize a terminator?");
     LLVM_DEBUG(dbgs() << "SLP:  initialize schedule region to " << *I << "\n");
     return true;
@@ -15154,8 +15093,6 @@ bool BoUpSLP::BlockScheduling::extendSchedulingRegion(Value *V,
            "Instruction is in wrong basic block.");
     initScheduleData(I, ScheduleStart, nullptr, FirstLoadStoreInRegion);
     ScheduleStart = I;
-    if (isOneOf(S, I) != I)
-      CheckScheduleForI(I);
     LLVM_DEBUG(dbgs() << "SLP:  extend schedule region start to " << *I
                       << "\n");
     return true;
@@ -15168,8 +15105,6 @@ bool BoUpSLP::BlockScheduling::extendSchedulingRegion(Value *V,
   initScheduleData(ScheduleEnd, I->getNextNode(), LastLoadStoreInRegion,
                    nullptr);
   ScheduleEnd = I->getNextNode();
-  if (isOneOf(S, I) != I)
-    CheckScheduleForI(I);
   assert(ScheduleEnd && "tried to vectorize a terminator?");
   LLVM_DEBUG(dbgs() << "SLP:  extend schedule region end to " << *I << "\n");
   return true;
@@ -15188,7 +15123,6 @@ void BoUpSLP::BlockScheduling::initScheduleData(Instruction *FromI,
     if (!SD) {
       SD = allocateScheduleDataChunks();
       ScheduleDataMap[I] = SD;
-      SD->Inst = I;
     }
     assert(!isInSchedulingRegion(SD) &&
            "new ScheduleData already in scheduling region");
@@ -15242,8 +15176,8 @@ void BoUpSLP::BlockScheduling::calculateDependencies(ScheduleData *SD,
       BundleMember->resetUnscheduledDeps();
 
       // Handle def-use chain dependencies.
-      if (BundleMember->OpValue != BundleMember->Inst) {
-        if (ScheduleData *UseSD = getScheduleData(BundleMember->Inst)) {
+      for (User *U : BundleMember->Inst->users()) {
+        if (ScheduleData *UseSD = getScheduleData(cast<Instruction>(U))) {
           BundleMember->Dependencies++;
           ScheduleData *DestBundle = UseSD->FirstInBundle;
           if (!DestBundle->IsScheduled)
@@ -15251,17 +15185,6 @@ void BoUpSLP::BlockScheduling::calculateDependencies(ScheduleData *SD,
           if (!DestBundle->hasValidDependencies())
             WorkList.push_back(DestBundle);
         }
-      } else {
-        for (User *U : BundleMember->Inst->users()) {
-          if (ScheduleData *UseSD = getScheduleData(cast<Instruction>(U))) {
-            BundleMember->Dependencies++;
-            ScheduleData *DestBundle = UseSD->FirstInBundle;
-            if (!DestBundle->IsScheduled)
-              BundleMember->incrementUnscheduledDeps(1);
-            if (!DestBundle->hasValidDependencies())
-              WorkList.push_back(DestBundle);
-          }
-        }
       }
 
       auto MakeControlDependent = [&](Instruction *I) {
@@ -15409,12 +15332,12 @@ void BoUpSLP::BlockScheduling::resetSchedule() {
   assert(ScheduleStart &&
          "tried to reset schedule on block which has not been scheduled");
   for (Instruction *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
-    doForAllOpcodes(I, [&](ScheduleData *SD) {
+    if (ScheduleData *SD = getScheduleData(I)) {
       assert(isInSchedulingRegion(SD) &&
              "ScheduleData not in scheduling region");
       SD->IsScheduled = false;
       SD->resetUnscheduledDeps();
-    });
+    }
   }
   ReadyInsts.clear();
 }
@@ -15449,7 +15372,7 @@ void BoUpSLP::scheduleBlock(BlockScheduling *BS) {
   int Idx = 0;
   for (auto *I = BS->ScheduleStart; I != BS->ScheduleEnd;
        I = I->getNextNode()) {
-    BS->doForAllOpcodes(I, [this, &Idx, BS](ScheduleData *SD) {
+    if (ScheduleData *SD = BS->getScheduleData(I)) {
       TreeEntry *SDTE = getTreeEntry(SD->Inst);
       (void)SDTE;
       assert((isVectorLikeInstWithConstOps(SD->Inst) ||
@@ -15460,7 +15383,7 @@ void BoUpSLP::scheduleBlock(BlockScheduling *BS) {
 
       if (SD->isSchedulingEntity() && SD->isPartOfBundle())
         BS->calculateDependencies(SD, false, this);
-    });
+    }
   }
   BS->initialFillReadyList(ReadyInsts);
 
@@ -15492,11 +15415,9 @@ void BoUpSLP::scheduleBlock(BlockScheduling *BS) {
 #if !defined(NDEBUG) || defined(EXPENSIVE_CHECKS)
   // Check that all schedulable entities got scheduled
   for (auto *I = BS->ScheduleStart; I != BS->ScheduleEnd; I = I->getNextNode()) {
-    BS->doForAllOpcodes(I, [&](ScheduleData *SD) {
-      if (SD->isSchedulingEntity() && SD->hasValidDependencies()) {
-        assert(SD->IsScheduled && "must be scheduled at this point");
-      }
-    });
+    ScheduleData *SD = BS->getScheduleData(I);
+    if (SD && SD->isSchedulingEntity() && SD->hasValidDependencies())
+      assert(SD->IsScheduled && "must be scheduled at this point");
   }
 #endif
 

From c6e9493a00d22f2591c5a9c51d9fd34fe5eee5b6 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Mon, 19 Aug 2024 13:35:02 +0100
Subject: [PATCH 350/441] [CostModel][X86] Add cost tests for scmp/ucmp
 intrinsics

---
 .../Analysis/CostModel/X86/icmp-codesize.ll   | 332 ++++++++++
 .../Analysis/CostModel/X86/icmp-latency.ll    | 424 ++++++++++++
 .../CostModel/X86/icmp-sizelatency.ll         | 332 ++++++++++
 llvm/test/Analysis/CostModel/X86/icmp.ll      | 608 ++++++++++++++++++
 4 files changed, 1696 insertions(+)

diff --git a/llvm/test/Analysis/CostModel/X86/icmp-codesize.ll b/llvm/test/Analysis/CostModel/X86/icmp-codesize.ll
index d952e9896d21592..2dc6737a3d8a07d 100644
--- a/llvm/test/Analysis/CostModel/X86/icmp-codesize.ll
+++ b/llvm/test/Analysis/CostModel/X86/icmp-codesize.ll
@@ -2089,3 +2089,335 @@ define i32 @cmp_int_ult(i8 %arg8, <16 x i8> %argv16i8, <32 x i8> %argv32i8, <64
   ret i32 undef
 }
 
+define i32 @scmp_int(i8 %arg8, <16 x i8> %argv16i8, <32 x i8> %argv32i8, <64 x i8> %argv64i8, <128 x i8> %argv128i8, i16 %arg16, <8 x i16> %argv8i16, <16 x i16> %argv16i16, <32 x i16> %argv32i16, <64 x i16> %argv64i16, i32 %arg32, <4 x i32> %argv4i32, <8 x i32> %argv8i32, <16 x i32> %argv16i32, <32 x i32> %argv32i32, i64 %arg64, <2 x i64> %argv2i64, <4 x i64> %argv4i64, <8 x i64> %argv8i64, <16 x i64> %argv16i64) {
+; AVX1-LABEL: 'scmp_int'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; AVX2-LABEL: 'scmp_int'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; AVX512F-LABEL: 'scmp_int'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 198 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 396 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 204 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; AVX512BW-LABEL: 'scmp_int'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 198 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 396 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 204 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; XOPAVX1-LABEL: 'scmp_int'
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; XOPAVX2-LABEL: 'scmp_int'
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+  %I8     = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+  %V16I8  = call <16 x i8> @llvm.scmp.v16i8.16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+  %V32I8  = call <32 x i8> @llvm.scmp.v32i8.32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+  %V64I8  = call <64 x i8> @llvm.scmp.v64i8.64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+  %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+
+  %I16    = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+  %V8I16  = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+  %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+  %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+  %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+
+  %I32    = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+  %V4I32  = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+  %V8I32  = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+  %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+  %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+
+  %I64    = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+  %V2I64  = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+  %V4I64  = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+  %V8I64  = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+  %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+
+  ret i32 undef
+}
+
+define i32 @ucmp_int(i8 %arg8, <16 x i8> %argv16i8, <32 x i8> %argv32i8, <64 x i8> %argv64i8, <128 x i8> %argv128i8, i16 %arg16, <8 x i16> %argv8i16, <16 x i16> %argv16i16, <32 x i16> %argv32i16, <64 x i16> %argv64i16, i32 %arg32, <4 x i32> %argv4i32, <8 x i32> %argv8i32, <16 x i32> %argv16i32, <32 x i32> %argv32i32, i64 %arg64, <2 x i64> %argv2i64, <4 x i64> %argv4i64, <8 x i64> %argv8i64, <16 x i64> %argv16i64) {
+; AVX1-LABEL: 'ucmp_int'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; AVX2-LABEL: 'ucmp_int'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; AVX512F-LABEL: 'ucmp_int'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 198 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 396 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 204 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; AVX512BW-LABEL: 'ucmp_int'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 198 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 396 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 204 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; XOPAVX1-LABEL: 'ucmp_int'
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; XOPAVX2-LABEL: 'ucmp_int'
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+  %I8     = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+  %V16I8  = call <16 x i8> @llvm.ucmp.v16i8.16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+  %V32I8  = call <32 x i8> @llvm.ucmp.v32i8.32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+  %V64I8  = call <64 x i8> @llvm.ucmp.v64i8.64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+  %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+
+  %I16    = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+  %V8I16  = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+  %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+  %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+  %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+
+  %I32    = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+  %V4I32  = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+  %V8I32  = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+  %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+  %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+
+  %I64    = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+  %V2I64  = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+  %V4I64  = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+  %V8I64  = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+  %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+
+  ret i32 undef
+}
+
diff --git a/llvm/test/Analysis/CostModel/X86/icmp-latency.ll b/llvm/test/Analysis/CostModel/X86/icmp-latency.ll
index 89cc6e893b31814..726a6dd782a4f7c 100644
--- a/llvm/test/Analysis/CostModel/X86/icmp-latency.ll
+++ b/llvm/test/Analysis/CostModel/X86/icmp-latency.ll
@@ -2319,3 +2319,427 @@ define i32 @cmp_int_ult(i8 %arg8, <16 x i8> %argv16i8, <32 x i8> %argv32i8, <64
   ret i32 undef
 }
 
+define i32 @scmp_int(i8 %arg8, <16 x i8> %argv16i8, <32 x i8> %argv32i8, <64 x i8> %argv64i8, <128 x i8> %argv128i8, i16 %arg16, <8 x i16> %argv8i16, <16 x i16> %argv16i16, <32 x i16> %argv32i16, <64 x i16> %argv64i16, i32 %arg32, <4 x i32> %argv4i32, <8 x i32> %argv8i32, <16 x i32> %argv16i32, <32 x i32> %argv32i32, i64 %arg64, <2 x i64> %argv2i64, <4 x i64> %argv4i64, <8 x i64> %argv8i64, <16 x i64> %argv16i64) {
+; SSE42-LABEL: 'scmp_int'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 384 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; AVX1-LABEL: 'scmp_int'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; AVX2-LABEL: 'scmp_int'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; AVX512F-LABEL: 'scmp_int'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 198 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 396 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 204 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; AVX512BW-LABEL: 'scmp_int'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 198 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 396 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 204 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; XOPAVX1-LABEL: 'scmp_int'
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; XOPAVX2-LABEL: 'scmp_int'
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; SLM-LABEL: 'scmp_int'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 93 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 186 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 372 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 744 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 45 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 90 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 180 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 360 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 84 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 168 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+  %I8     = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+  %V16I8  = call <16 x i8> @llvm.scmp.v16i8.16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+  %V32I8  = call <32 x i8> @llvm.scmp.v32i8.32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+  %V64I8  = call <64 x i8> @llvm.scmp.v64i8.64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+  %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+
+  %I16    = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+  %V8I16  = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+  %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+  %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+  %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+
+  %I32    = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+  %V4I32  = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+  %V8I32  = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+  %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+  %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+
+  %I64    = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+  %V2I64  = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+  %V4I64  = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+  %V8I64  = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+  %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+
+  ret i32 undef
+}
+
+define i32 @ucmp_int(i8 %arg8, <16 x i8> %argv16i8, <32 x i8> %argv32i8, <64 x i8> %argv64i8, <128 x i8> %argv128i8, i16 %arg16, <8 x i16> %argv8i16, <16 x i16> %argv16i16, <32 x i16> %argv32i16, <64 x i16> %argv64i16, i32 %arg32, <4 x i32> %argv4i32, <8 x i32> %argv8i32, <16 x i32> %argv16i32, <32 x i32> %argv32i32, i64 %arg64, <2 x i64> %argv2i64, <4 x i64> %argv4i64, <8 x i64> %argv8i64, <16 x i64> %argv16i64) {
+; SSE42-LABEL: 'ucmp_int'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 384 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; AVX1-LABEL: 'ucmp_int'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; AVX2-LABEL: 'ucmp_int'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; AVX512F-LABEL: 'ucmp_int'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 198 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 396 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 204 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; AVX512BW-LABEL: 'ucmp_int'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 198 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 396 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 204 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; XOPAVX1-LABEL: 'ucmp_int'
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; XOPAVX2-LABEL: 'ucmp_int'
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; SLM-LABEL: 'ucmp_int'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 93 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 186 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 372 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 744 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 45 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 90 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 180 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 360 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 84 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 168 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+  %I8     = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+  %V16I8  = call <16 x i8> @llvm.ucmp.v16i8.16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+  %V32I8  = call <32 x i8> @llvm.ucmp.v32i8.32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+  %V64I8  = call <64 x i8> @llvm.ucmp.v64i8.64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+  %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+
+  %I16    = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+  %V8I16  = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+  %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+  %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+  %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+
+  %I32    = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+  %V4I32  = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+  %V8I32  = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+  %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+  %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+
+  %I64    = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+  %V2I64  = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+  %V4I64  = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+  %V8I64  = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+  %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+
+  ret i32 undef
+}
+
diff --git a/llvm/test/Analysis/CostModel/X86/icmp-sizelatency.ll b/llvm/test/Analysis/CostModel/X86/icmp-sizelatency.ll
index 2cac3af5c01b824..0deaad5991fb2f1 100644
--- a/llvm/test/Analysis/CostModel/X86/icmp-sizelatency.ll
+++ b/llvm/test/Analysis/CostModel/X86/icmp-sizelatency.ll
@@ -2089,3 +2089,335 @@ define i32 @cmp_int_ult(i8 %arg8, <16 x i8> %argv16i8, <32 x i8> %argv32i8, <64
   ret i32 undef
 }
 
+define i32 @scmp_int(i8 %arg8, <16 x i8> %argv16i8, <32 x i8> %argv32i8, <64 x i8> %argv64i8, <128 x i8> %argv128i8, i16 %arg16, <8 x i16> %argv8i16, <16 x i16> %argv16i16, <32 x i16> %argv32i16, <64 x i16> %argv64i16, i32 %arg32, <4 x i32> %argv4i32, <8 x i32> %argv8i32, <16 x i32> %argv16i32, <32 x i32> %argv32i32, i64 %arg64, <2 x i64> %argv2i64, <4 x i64> %argv4i64, <8 x i64> %argv8i64, <16 x i64> %argv16i64) {
+; AVX1-LABEL: 'scmp_int'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; AVX2-LABEL: 'scmp_int'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; AVX512F-LABEL: 'scmp_int'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 198 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 396 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 204 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; AVX512BW-LABEL: 'scmp_int'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 198 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 396 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 204 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; XOPAVX1-LABEL: 'scmp_int'
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; XOPAVX2-LABEL: 'scmp_int'
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+  %I8     = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+  %V16I8  = call <16 x i8> @llvm.scmp.v16i8.16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+  %V32I8  = call <32 x i8> @llvm.scmp.v32i8.32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+  %V64I8  = call <64 x i8> @llvm.scmp.v64i8.64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+  %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+
+  %I16    = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+  %V8I16  = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+  %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+  %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+  %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+
+  %I32    = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+  %V4I32  = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+  %V8I32  = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+  %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+  %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+
+  %I64    = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+  %V2I64  = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+  %V4I64  = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+  %V8I64  = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+  %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+
+  ret i32 undef
+}
+
+define i32 @ucmp_int(i8 %arg8, <16 x i8> %argv16i8, <32 x i8> %argv32i8, <64 x i8> %argv64i8, <128 x i8> %argv128i8, i16 %arg16, <8 x i16> %argv8i16, <16 x i16> %argv16i16, <32 x i16> %argv32i16, <64 x i16> %argv64i16, i32 %arg32, <4 x i32> %argv4i32, <8 x i32> %argv8i32, <16 x i32> %argv16i32, <32 x i32> %argv32i32, i64 %arg64, <2 x i64> %argv2i64, <4 x i64> %argv4i64, <8 x i64> %argv8i64, <16 x i64> %argv16i64) {
+; AVX1-LABEL: 'ucmp_int'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; AVX2-LABEL: 'ucmp_int'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; AVX512F-LABEL: 'ucmp_int'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 198 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 396 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 204 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; AVX512BW-LABEL: 'ucmp_int'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 198 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 396 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 204 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; XOPAVX1-LABEL: 'ucmp_int'
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+; XOPAVX2-LABEL: 'ucmp_int'
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+;
+  %I8     = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+  %V16I8  = call <16 x i8> @llvm.ucmp.v16i8.16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+  %V32I8  = call <32 x i8> @llvm.ucmp.v32i8.32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+  %V64I8  = call <64 x i8> @llvm.ucmp.v64i8.64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+  %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+
+  %I16    = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+  %V8I16  = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+  %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+  %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+  %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+
+  %I32    = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+  %V4I32  = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+  %V8I32  = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+  %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+  %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+
+  %I64    = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+  %V2I64  = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+  %V4I64  = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+  %V8I64  = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+  %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+
+  ret i32 undef
+}
+
diff --git a/llvm/test/Analysis/CostModel/X86/icmp.ll b/llvm/test/Analysis/CostModel/X86/icmp.ll
index b1d13ffa605ed08..599895c2b5705a2 100644
--- a/llvm/test/Analysis/CostModel/X86/icmp.ll
+++ b/llvm/test/Analysis/CostModel/X86/icmp.ll
@@ -3055,3 +3055,611 @@ define i32 @cmp_int_ult(i8 %arg8, <16 x i8> %argv16i8, <32 x i8> %argv32i8, <64
   ret i32 undef
 }
 
+define i32 @scmp_int(i8 %arg8, <16 x i8> %argv16i8, <32 x i8> %argv32i8, <64 x i8> %argv64i8, <128 x i8> %argv128i8, i16 %arg16, <8 x i16> %argv8i16, <16 x i16> %argv16i16, <32 x i16> %argv32i16, <64 x i16> %argv64i16, i32 %arg32, <4 x i32> %argv4i32, <8 x i32> %argv8i32, <16 x i32> %argv16i32, <32 x i32> %argv32i32, i64 %arg64, <2 x i64> %argv2i64, <4 x i64> %argv4i64, <8 x i64> %argv8i64, <16 x i64> %argv16i64) {
+; SSE2-LABEL: 'scmp_int'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 78 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 156 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 312 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 624 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 144 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE3-LABEL: 'scmp_int'
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 78 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 156 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 312 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 624 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 144 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSSE3-LABEL: 'scmp_int'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 78 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 156 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 312 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 624 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 144 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE41-LABEL: 'scmp_int'
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 384 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE42-LABEL: 'scmp_int'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 384 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX1-LABEL: 'scmp_int'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX2-LABEL: 'scmp_int'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512F-LABEL: 'scmp_int'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 198 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 396 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 204 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512BW-LABEL: 'scmp_int'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 198 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 396 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 204 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; XOPAVX1-LABEL: 'scmp_int'
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; XOPAVX2-LABEL: 'scmp_int'
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SLM-LABEL: 'scmp_int'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 93 for instruction: %V16I8 = call <16 x i8> @llvm.scmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 186 for instruction: %V32I8 = call <32 x i8> @llvm.scmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 372 for instruction: %V64I8 = call <64 x i8> @llvm.scmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 744 for instruction: %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 45 for instruction: %V8I16 = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 90 for instruction: %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 180 for instruction: %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 360 for instruction: %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V4I32 = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V8I32 = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 84 for instruction: %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 168 for instruction: %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2I64 = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V4I64 = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V8I64 = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %I8     = call i8 @llvm.scmp.i8.i8(i8 %arg8, i8 %arg8)
+  %V16I8  = call <16 x i8> @llvm.scmp.v16i8.16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+  %V32I8  = call <32 x i8> @llvm.scmp.v32i8.32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+  %V64I8  = call <64 x i8> @llvm.scmp.v64i8.64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+  %V128I8 = call <128 x i8> @llvm.scmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+
+  %I16    = call i16 @llvm.scmp.i16.i16(i16 %arg16, i16 %arg16)
+  %V8I16  = call <8 x i16> @llvm.scmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+  %V16I16 = call <16 x i16> @llvm.scmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+  %V32I16 = call <32 x i16> @llvm.scmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+  %V64I16 = call <64 x i16> @llvm.scmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+
+  %I32    = call i32 @llvm.scmp.i32.i32(i32 %arg32, i32 %arg32)
+  %V4I32  = call <4 x i32> @llvm.scmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+  %V8I32  = call <8 x i32> @llvm.scmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+  %V16I32 = call <16 x i32> @llvm.scmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+  %V32I32 = call <32 x i32> @llvm.scmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+
+  %I64    = call i64 @llvm.scmp.i64.i64(i64 %arg64, i64 %arg64)
+  %V2I64  = call <2 x i64> @llvm.scmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+  %V4I64  = call <4 x i64> @llvm.scmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+  %V8I64  = call <8 x i64> @llvm.scmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+  %V16I64 = call <16 x i64> @llvm.scmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+
+  ret i32 undef
+}
+
+define i32 @ucmp_int(i8 %arg8, <16 x i8> %argv16i8, <32 x i8> %argv32i8, <64 x i8> %argv64i8, <128 x i8> %argv128i8, i16 %arg16, <8 x i16> %argv8i16, <16 x i16> %argv16i16, <32 x i16> %argv32i16, <64 x i16> %argv64i16, i32 %arg32, <4 x i32> %argv4i32, <8 x i32> %argv8i32, <16 x i32> %argv16i32, <32 x i32> %argv32i32, i64 %arg64, <2 x i64> %argv2i64, <4 x i64> %argv4i64, <8 x i64> %argv8i64, <16 x i64> %argv16i64) {
+; SSE2-LABEL: 'ucmp_int'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 78 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 156 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 312 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 624 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 144 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE3-LABEL: 'ucmp_int'
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 78 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 156 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 312 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 624 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 144 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSSE3-LABEL: 'ucmp_int'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 78 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 156 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 312 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 624 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 144 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE41-LABEL: 'ucmp_int'
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 384 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE42-LABEL: 'ucmp_int'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 384 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX1-LABEL: 'ucmp_int'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX2-LABEL: 'ucmp_int'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512F-LABEL: 'ucmp_int'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 198 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 396 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 204 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512BW-LABEL: 'ucmp_int'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 198 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 396 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 204 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; XOPAVX1-LABEL: 'ucmp_int'
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; XOPAVX2-LABEL: 'ucmp_int'
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 392 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SLM-LABEL: 'ucmp_int'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 93 for instruction: %V16I8 = call <16 x i8> @llvm.ucmp.v16i8.v16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 186 for instruction: %V32I8 = call <32 x i8> @llvm.ucmp.v32i8.v32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 372 for instruction: %V64I8 = call <64 x i8> @llvm.ucmp.v64i8.v64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 744 for instruction: %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 45 for instruction: %V8I16 = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 90 for instruction: %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 180 for instruction: %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 360 for instruction: %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V4I32 = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V8I32 = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 84 for instruction: %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 168 for instruction: %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2I64 = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V4I64 = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V8I64 = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %I8     = call i8 @llvm.ucmp.i8.i8(i8 %arg8, i8 %arg8)
+  %V16I8  = call <16 x i8> @llvm.ucmp.v16i8.16i8(<16 x i8> %argv16i8, <16 x i8> %argv16i8)
+  %V32I8  = call <32 x i8> @llvm.ucmp.v32i8.32i8(<32 x i8> %argv32i8, <32 x i8> %argv32i8)
+  %V64I8  = call <64 x i8> @llvm.ucmp.v64i8.64i8(<64 x i8> %argv64i8, <64 x i8> %argv64i8)
+  %V128I8 = call <128 x i8> @llvm.ucmp.v128i8.v128i8(<128 x i8> %argv128i8, <128 x i8> %argv128i8)
+
+  %I16    = call i16 @llvm.ucmp.i16.i16(i16 %arg16, i16 %arg16)
+  %V8I16  = call <8 x i16> @llvm.ucmp.v8i16.v8i16(<8 x i16> %argv8i16, <8 x i16> %argv8i16)
+  %V16I16 = call <16 x i16> @llvm.ucmp.v16i16.v16i16(<16 x i16> %argv16i16, <16 x i16> %argv16i16)
+  %V32I16 = call <32 x i16> @llvm.ucmp.v32i16.v32i16(<32 x i16> %argv32i16, <32 x i16> %argv32i16)
+  %V64I16 = call <64 x i16> @llvm.ucmp.v64i16.v64i16(<64 x i16> %argv64i16, <64 x i16> %argv64i16)
+
+  %I32    = call i32 @llvm.ucmp.i32.i32(i32 %arg32, i32 %arg32)
+  %V4I32  = call <4 x i32> @llvm.ucmp.v4i32.v4i32(<4 x i32> %argv4i32, <4 x i32> %argv4i32)
+  %V8I32  = call <8 x i32> @llvm.ucmp.v8i32.v8i32(<8 x i32> %argv8i32, <8 x i32> %argv8i32)
+  %V16I32 = call <16 x i32> @llvm.ucmp.v16i32.v16i32(<16 x i32> %argv16i32, <16 x i32> %argv16i32)
+  %V32I32 = call <32 x i32> @llvm.ucmp.v32i32.v32i32(<32 x i32> %argv32i32, <32 x i32> %argv32i32)
+
+  %I64    = call i64 @llvm.ucmp.i64.i64(i64 %arg64, i64 %arg64)
+  %V2I64  = call <2 x i64> @llvm.ucmp.v2i64.v2i64(<2 x i64> %argv2i64, <2 x i64> %argv2i64)
+  %V4I64  = call <4 x i64> @llvm.ucmp.v4i64.v4i64(<4 x i64> %argv4i64, <4 x i64> %argv4i64)
+  %V8I64  = call <8 x i64> @llvm.ucmp.v8i64.v8i64(<8 x i64> %argv8i64, <8 x i64> %argv8i64)
+  %V16I64 = call <16 x i64> @llvm.ucmp.v16i64.v16i64(<16 x i64> %argv16i64, <16 x i64> %argv16i64)
+
+  ret i32 undef
+}
+

From 0cc6b464f8adb739e6f9bcc9bda8cff9bb1f1c2a Mon Sep 17 00:00:00 2001
From: Piotr Fusik <p.fusik@samsung.com>
Date: Mon, 19 Aug 2024 14:39:17 +0200
Subject: [PATCH 351/441] [docs] Update a filename, fix indentation (#103018)

---
 llvm/docs/ExtendingLLVM.rst | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/llvm/docs/ExtendingLLVM.rst b/llvm/docs/ExtendingLLVM.rst
index 12372996c96dde1..3965112a20f9080 100644
--- a/llvm/docs/ExtendingLLVM.rst
+++ b/llvm/docs/ExtendingLLVM.rst
@@ -90,14 +90,14 @@ complicated behavior in a single node (rotate).
 
    Add an enum value for the new SelectionDAG node.
 
-#. ``lib/CodeGen/SelectionDAG/SelectionDAG.cpp``:
+#. ``lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp``:
 
    Add code to print the node to ``getOperationName``.  If your new node can be
-    evaluated at compile time when given constant arguments (such as an add of a
-    constant with another constant), find the ``getNode`` method that takes the
-    appropriate number of arguments, and add a case for your node to the switch
-    statement that performs constant folding for nodes that take the same number
-    of arguments as your new node.
+   evaluated at compile time when given constant arguments (such as an add of a
+   constant with another constant), find the ``getNode`` method that takes the
+   appropriate number of arguments, and add a case for your node to the switch
+   statement that performs constant folding for nodes that take the same number
+   of arguments as your new node.
 
 #. ``lib/CodeGen/SelectionDAG/LegalizeDAG.cpp``:
 
@@ -115,12 +115,12 @@ complicated behavior in a single node (rotate).
 #. ``lib/CodeGen/SelectionDAG/LegalizeDAG.cpp``:
 
    If targets may support the new node being added only at certain sizes, you
-    will also need to add code to your node's case statement in ``LegalizeOp``
-    to Promote your node's operands to a larger size, and perform the correct
-    operation.  You will also need to add code to ``PromoteOp`` to do this as
-    well.  For a good example, see ``ISD::BSWAP``, which promotes its operand to
-    a wider size, performs the byteswap, and then shifts the correct bytes right
-    to emulate the narrower byteswap in the wider type.
+   will also need to add code to your node's case statement in ``LegalizeOp``
+   to Promote your node's operands to a larger size, and perform the correct
+   operation.  You will also need to add code to ``PromoteOp`` to do this as
+   well.  For a good example, see ``ISD::BSWAP``, which promotes its operand to
+   a wider size, performs the byteswap, and then shifts the correct bytes right
+   to emulate the narrower byteswap in the wider type.
 
 #. ``lib/CodeGen/SelectionDAG/LegalizeDAG.cpp``:
 

From b05c55472bf7cadcd0e4cb1a669b3474695b0524 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Mon, 19 Aug 2024 13:50:07 +0100
Subject: [PATCH 352/441] [X86] LowerABD - simplify i32/i64 to use sub+sub+cmov
 instead of repeating nodes via abs (#102174)

Using X86ISD::SUB nodes directly allows us to drive the X86ISD::CMOV node with exact flags instead of trying to cleanup the generic codegen via ICMP/SUBO nodes.
---
 llvm/lib/Target/X86/X86ISelLowering.cpp |  20 +++-
 llvm/test/CodeGen/X86/abds-neg.ll       |  99 ++++++++-----------
 llvm/test/CodeGen/X86/abds.ll           | 124 ++++++++++--------------
 llvm/test/CodeGen/X86/abdu-neg.ll       |  93 ++++++++----------
 llvm/test/CodeGen/X86/abdu.ll           | 118 ++++++++++------------
 5 files changed, 197 insertions(+), 257 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 20d190bd4aff2f9..169c955f0ba89f0 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -28443,13 +28443,27 @@ static SDValue LowerABD(SDValue Op, const X86Subtarget &Subtarget,
   bool IsSigned = Op.getOpcode() == ISD::ABDS;
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
 
-  // TODO: Move to TargetLowering expandABD() once we have ABD promotion.
   if (VT.isScalarInteger()) {
+    // abds(lhs, rhs) -> select(slt(lhs,rhs),sub(rhs,lhs),sub(lhs,rhs))
+    // abdu(lhs, rhs) -> select(ult(lhs,rhs),sub(rhs,lhs),sub(lhs,rhs))
+    if (Subtarget.canUseCMOV() && VT.bitsGE(MVT::i32)) {
+      SDVTList VTs = DAG.getVTList(VT, MVT::i32);
+      X86::CondCode CC = IsSigned ? X86::COND_L : X86::COND_B;
+      SDValue LHS = DAG.getFreeze(Op.getOperand(0));
+      SDValue RHS = DAG.getFreeze(Op.getOperand(1));
+      SDValue Diff0 = DAG.getNode(X86ISD::SUB, dl, VTs, LHS, RHS);
+      SDValue Diff1 = DAG.getNode(X86ISD::SUB, dl, VTs, RHS, LHS);
+      return DAG.getNode(X86ISD::CMOV, dl, VT, Diff1, Diff0,
+                         DAG.getTargetConstant(CC, dl, MVT::i8),
+                         Diff1.getValue(1));
+    }
+
+    // TODO: Move to TargetLowering expandABD() once we have ABD promotion.
+    // abds(lhs, rhs) -> trunc(abs(sub(sext(lhs), sext(rhs))))
+    // abdu(lhs, rhs) -> trunc(abs(sub(zext(lhs), zext(rhs))))
     unsigned WideBits = std::max<unsigned>(2 * VT.getScalarSizeInBits(), 32u);
     MVT WideVT = MVT::getIntegerVT(WideBits);
     if (TLI.isTypeLegal(WideVT)) {
-      // abds(lhs, rhs) -> trunc(abs(sub(sext(lhs), sext(rhs))))
-      // abdu(lhs, rhs) -> trunc(abs(sub(zext(lhs), zext(rhs))))
       unsigned ExtOpc = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
       SDValue LHS = DAG.getNode(ExtOpc, dl, WideVT, Op.getOperand(0));
       SDValue RHS = DAG.getNode(ExtOpc, dl, WideVT, Op.getOperand(1));
diff --git a/llvm/test/CodeGen/X86/abds-neg.ll b/llvm/test/CodeGen/X86/abds-neg.ll
index 541ed1cbf4fe89b..b72bb46209d2d66 100644
--- a/llvm/test/CodeGen/X86/abds-neg.ll
+++ b/llvm/test/CodeGen/X86/abds-neg.ll
@@ -139,28 +139,25 @@ define i16 @abd_ext_i16(i16 %a, i16 %b) nounwind {
 define i16 @abd_ext_i16_i32(i16 %a, i32 %b) nounwind {
 ; X86-LABEL: abd_ext_i16_i32:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movswl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl %eax, %edx
-; X86-NEXT:    subl %ecx, %edx
-; X86-NEXT:    negl %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movswl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, %edx
+; X86-NEXT:    subl %eax, %edx
 ; X86-NEXT:    subl %ecx, %eax
-; X86-NEXT:    cmovlel %edx, %eax
+; X86-NEXT:    cmovll %edx, %eax
 ; X86-NEXT:    negl %eax
 ; X86-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: abd_ext_i16_i32:
 ; X64:       # %bb.0:
-; X64-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-NEXT:    movswq %di, %rcx
-; X64-NEXT:    movslq %esi, %rax
-; X64-NEXT:    subq %rax, %rcx
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    negq %rax
-; X64-NEXT:    cmovsq %rcx, %rax
+; X64-NEXT:    movswl %di, %ecx
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    subl %esi, %eax
+; X64-NEXT:    subl %ecx, %esi
+; X64-NEXT:    cmovgel %esi, %eax
 ; X64-NEXT:    negl %eax
-; X64-NEXT:    # kill: def $ax killed $ax killed $rax
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X64-NEXT:    retq
   %aext = sext i16 %a to i64
   %bext = sext i32 %b to i64
@@ -205,26 +202,22 @@ define i16 @abd_ext_i16_undef(i16 %a, i16 %b) nounwind {
 define i32 @abd_ext_i32(i32 %a, i32 %b) nounwind {
 ; X86-LABEL: abd_ext_i32:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl %eax, %edx
-; X86-NEXT:    subl %ecx, %edx
-; X86-NEXT:    negl %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, %edx
+; X86-NEXT:    subl %eax, %edx
 ; X86-NEXT:    subl %ecx, %eax
-; X86-NEXT:    cmovlel %edx, %eax
+; X86-NEXT:    cmovll %edx, %eax
 ; X86-NEXT:    negl %eax
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: abd_ext_i32:
 ; X64:       # %bb.0:
-; X64-NEXT:    movslq %esi, %rax
-; X64-NEXT:    movslq %edi, %rcx
-; X64-NEXT:    subq %rax, %rcx
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    negq %rax
-; X64-NEXT:    cmovsq %rcx, %rax
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    subl %esi, %eax
+; X64-NEXT:    subl %edi, %esi
+; X64-NEXT:    cmovgel %esi, %eax
 ; X64-NEXT:    negl %eax
-; X64-NEXT:    # kill: def $eax killed $eax killed $rax
 ; X64-NEXT:    retq
   %aext = sext i32 %a to i64
   %bext = sext i32 %b to i64
@@ -238,27 +231,23 @@ define i32 @abd_ext_i32(i32 %a, i32 %b) nounwind {
 define i32 @abd_ext_i32_i16(i32 %a, i16 %b) nounwind {
 ; X86-LABEL: abd_ext_i32_i16:
 ; X86:       # %bb.0:
-; X86-NEXT:    movswl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl %eax, %edx
-; X86-NEXT:    subl %ecx, %edx
-; X86-NEXT:    negl %edx
+; X86-NEXT:    movswl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, %edx
+; X86-NEXT:    subl %eax, %edx
 ; X86-NEXT:    subl %ecx, %eax
-; X86-NEXT:    cmovlel %edx, %eax
+; X86-NEXT:    cmovll %edx, %eax
 ; X86-NEXT:    negl %eax
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: abd_ext_i32_i16:
 ; X64:       # %bb.0:
-; X64-NEXT:    # kill: def $esi killed $esi def $rsi
-; X64-NEXT:    movswq %si, %rax
-; X64-NEXT:    movslq %edi, %rcx
-; X64-NEXT:    subq %rax, %rcx
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    negq %rax
-; X64-NEXT:    cmovsq %rcx, %rax
+; X64-NEXT:    movswl %si, %eax
+; X64-NEXT:    movl %edi, %ecx
+; X64-NEXT:    subl %eax, %ecx
+; X64-NEXT:    subl %edi, %eax
+; X64-NEXT:    cmovll %ecx, %eax
 ; X64-NEXT:    negl %eax
-; X64-NEXT:    # kill: def $eax killed $eax killed $rax
 ; X64-NEXT:    retq
   %aext = sext i32 %a to i64
   %bext = sext i16 %b to i64
@@ -272,26 +261,22 @@ define i32 @abd_ext_i32_i16(i32 %a, i16 %b) nounwind {
 define i32 @abd_ext_i32_undef(i32 %a, i32 %b) nounwind {
 ; X86-LABEL: abd_ext_i32_undef:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl %eax, %edx
-; X86-NEXT:    subl %ecx, %edx
-; X86-NEXT:    negl %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, %edx
+; X86-NEXT:    subl %eax, %edx
 ; X86-NEXT:    subl %ecx, %eax
-; X86-NEXT:    cmovlel %edx, %eax
+; X86-NEXT:    cmovll %edx, %eax
 ; X86-NEXT:    negl %eax
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: abd_ext_i32_undef:
 ; X64:       # %bb.0:
-; X64-NEXT:    movslq %esi, %rax
-; X64-NEXT:    movslq %edi, %rcx
-; X64-NEXT:    subq %rax, %rcx
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    negq %rax
-; X64-NEXT:    cmovsq %rcx, %rax
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    subl %esi, %eax
+; X64-NEXT:    subl %edi, %esi
+; X64-NEXT:    cmovgel %esi, %eax
 ; X64-NEXT:    negl %eax
-; X64-NEXT:    # kill: def $eax killed $eax killed $rax
 ; X64-NEXT:    retq
   %aext = sext i32 %a to i64
   %bext = sext i32 %b to i64
@@ -332,9 +317,8 @@ define i64 @abd_ext_i64(i64 %a, i64 %b) nounwind {
 ; X64:       # %bb.0:
 ; X64-NEXT:    movq %rdi, %rax
 ; X64-NEXT:    subq %rsi, %rax
-; X64-NEXT:    negq %rax
-; X64-NEXT:    subq %rsi, %rdi
-; X64-NEXT:    cmovgq %rdi, %rax
+; X64-NEXT:    subq %rdi, %rsi
+; X64-NEXT:    cmovgeq %rsi, %rax
 ; X64-NEXT:    negq %rax
 ; X64-NEXT:    retq
   %aext = sext i64 %a to i128
@@ -376,9 +360,8 @@ define i64 @abd_ext_i64_undef(i64 %a, i64 %b) nounwind {
 ; X64:       # %bb.0:
 ; X64-NEXT:    movq %rdi, %rax
 ; X64-NEXT:    subq %rsi, %rax
-; X64-NEXT:    negq %rax
-; X64-NEXT:    subq %rsi, %rdi
-; X64-NEXT:    cmovgq %rdi, %rax
+; X64-NEXT:    subq %rdi, %rsi
+; X64-NEXT:    cmovgeq %rsi, %rax
 ; X64-NEXT:    negq %rax
 ; X64-NEXT:    retq
   %aext = sext i64 %a to i128
diff --git a/llvm/test/CodeGen/X86/abds.ll b/llvm/test/CodeGen/X86/abds.ll
index fd47de5af05ab4d..e972ef3787e4d59 100644
--- a/llvm/test/CodeGen/X86/abds.ll
+++ b/llvm/test/CodeGen/X86/abds.ll
@@ -129,26 +129,23 @@ define i16 @abd_ext_i16(i16 %a, i16 %b) nounwind {
 define i16 @abd_ext_i16_i32(i16 %a, i32 %b) nounwind {
 ; X86-LABEL: abd_ext_i16_i32:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movswl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl %eax, %edx
-; X86-NEXT:    subl %ecx, %edx
-; X86-NEXT:    negl %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movswl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, %edx
+; X86-NEXT:    subl %eax, %edx
 ; X86-NEXT:    subl %ecx, %eax
-; X86-NEXT:    cmovlel %edx, %eax
+; X86-NEXT:    cmovll %edx, %eax
 ; X86-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: abd_ext_i16_i32:
 ; X64:       # %bb.0:
-; X64-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-NEXT:    movswq %di, %rcx
-; X64-NEXT:    movslq %esi, %rax
-; X64-NEXT:    subq %rax, %rcx
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    negq %rax
-; X64-NEXT:    cmovsq %rcx, %rax
-; X64-NEXT:    # kill: def $ax killed $ax killed $rax
+; X64-NEXT:    movswl %di, %ecx
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    subl %esi, %eax
+; X64-NEXT:    subl %ecx, %esi
+; X64-NEXT:    cmovgel %esi, %eax
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X64-NEXT:    retq
   %aext = sext i16 %a to i64
   %bext = sext i32 %b to i64
@@ -191,24 +188,20 @@ define i16 @abd_ext_i16_undef(i16 %a, i16 %b) nounwind {
 define i32 @abd_ext_i32(i32 %a, i32 %b) nounwind {
 ; X86-LABEL: abd_ext_i32:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl %eax, %edx
-; X86-NEXT:    subl %ecx, %edx
-; X86-NEXT:    negl %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, %edx
+; X86-NEXT:    subl %eax, %edx
 ; X86-NEXT:    subl %ecx, %eax
-; X86-NEXT:    cmovlel %edx, %eax
+; X86-NEXT:    cmovll %edx, %eax
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: abd_ext_i32:
 ; X64:       # %bb.0:
-; X64-NEXT:    movslq %esi, %rax
-; X64-NEXT:    movslq %edi, %rcx
-; X64-NEXT:    subq %rax, %rcx
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    negq %rax
-; X64-NEXT:    cmovsq %rcx, %rax
-; X64-NEXT:    # kill: def $eax killed $eax killed $rax
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    subl %esi, %eax
+; X64-NEXT:    subl %edi, %esi
+; X64-NEXT:    cmovgel %esi, %eax
 ; X64-NEXT:    retq
   %aext = sext i32 %a to i64
   %bext = sext i32 %b to i64
@@ -221,25 +214,21 @@ define i32 @abd_ext_i32(i32 %a, i32 %b) nounwind {
 define i32 @abd_ext_i32_i16(i32 %a, i16 %b) nounwind {
 ; X86-LABEL: abd_ext_i32_i16:
 ; X86:       # %bb.0:
-; X86-NEXT:    movswl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl %eax, %edx
-; X86-NEXT:    subl %ecx, %edx
-; X86-NEXT:    negl %edx
+; X86-NEXT:    movswl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, %edx
+; X86-NEXT:    subl %eax, %edx
 ; X86-NEXT:    subl %ecx, %eax
-; X86-NEXT:    cmovlel %edx, %eax
+; X86-NEXT:    cmovll %edx, %eax
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: abd_ext_i32_i16:
 ; X64:       # %bb.0:
-; X64-NEXT:    # kill: def $esi killed $esi def $rsi
-; X64-NEXT:    movswq %si, %rax
-; X64-NEXT:    movslq %edi, %rcx
-; X64-NEXT:    subq %rax, %rcx
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    negq %rax
-; X64-NEXT:    cmovsq %rcx, %rax
-; X64-NEXT:    # kill: def $eax killed $eax killed $rax
+; X64-NEXT:    movswl %si, %eax
+; X64-NEXT:    movl %edi, %ecx
+; X64-NEXT:    subl %eax, %ecx
+; X64-NEXT:    subl %edi, %eax
+; X64-NEXT:    cmovll %ecx, %eax
 ; X64-NEXT:    retq
   %aext = sext i32 %a to i64
   %bext = sext i16 %b to i64
@@ -252,24 +241,20 @@ define i32 @abd_ext_i32_i16(i32 %a, i16 %b) nounwind {
 define i32 @abd_ext_i32_undef(i32 %a, i32 %b) nounwind {
 ; X86-LABEL: abd_ext_i32_undef:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl %eax, %edx
-; X86-NEXT:    subl %ecx, %edx
-; X86-NEXT:    negl %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, %edx
+; X86-NEXT:    subl %eax, %edx
 ; X86-NEXT:    subl %ecx, %eax
-; X86-NEXT:    cmovlel %edx, %eax
+; X86-NEXT:    cmovll %edx, %eax
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: abd_ext_i32_undef:
 ; X64:       # %bb.0:
-; X64-NEXT:    movslq %esi, %rax
-; X64-NEXT:    movslq %edi, %rcx
-; X64-NEXT:    subq %rax, %rcx
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    negq %rax
-; X64-NEXT:    cmovsq %rcx, %rax
-; X64-NEXT:    # kill: def $eax killed $eax killed $rax
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    subl %esi, %eax
+; X64-NEXT:    subl %edi, %esi
+; X64-NEXT:    cmovgel %esi, %eax
 ; X64-NEXT:    retq
   %aext = sext i32 %a to i64
   %bext = sext i32 %b to i64
@@ -306,9 +291,8 @@ define i64 @abd_ext_i64(i64 %a, i64 %b) nounwind {
 ; X64:       # %bb.0:
 ; X64-NEXT:    movq %rdi, %rax
 ; X64-NEXT:    subq %rsi, %rax
-; X64-NEXT:    negq %rax
-; X64-NEXT:    subq %rsi, %rdi
-; X64-NEXT:    cmovgq %rdi, %rax
+; X64-NEXT:    subq %rdi, %rsi
+; X64-NEXT:    cmovgeq %rsi, %rax
 ; X64-NEXT:    retq
   %aext = sext i64 %a to i128
   %bext = sext i64 %b to i128
@@ -345,9 +329,8 @@ define i64 @abd_ext_i64_undef(i64 %a, i64 %b) nounwind {
 ; X64:       # %bb.0:
 ; X64-NEXT:    movq %rdi, %rax
 ; X64-NEXT:    subq %rsi, %rax
-; X64-NEXT:    negq %rax
-; X64-NEXT:    subq %rsi, %rdi
-; X64-NEXT:    cmovgq %rdi, %rax
+; X64-NEXT:    subq %rdi, %rsi
+; X64-NEXT:    cmovgeq %rsi, %rax
 ; X64-NEXT:    retq
   %aext = sext i64 %a to i128
   %bext = sext i64 %b to i128
@@ -542,24 +525,20 @@ define i16 @abd_minmax_i16(i16 %a, i16 %b) nounwind {
 define i32 @abd_minmax_i32(i32 %a, i32 %b) nounwind {
 ; X86-LABEL: abd_minmax_i32:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl %eax, %edx
-; X86-NEXT:    subl %ecx, %edx
-; X86-NEXT:    negl %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, %edx
+; X86-NEXT:    subl %eax, %edx
 ; X86-NEXT:    subl %ecx, %eax
-; X86-NEXT:    cmovlel %edx, %eax
+; X86-NEXT:    cmovll %edx, %eax
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: abd_minmax_i32:
 ; X64:       # %bb.0:
-; X64-NEXT:    movslq %esi, %rax
-; X64-NEXT:    movslq %edi, %rcx
-; X64-NEXT:    subq %rax, %rcx
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    negq %rax
-; X64-NEXT:    cmovsq %rcx, %rax
-; X64-NEXT:    # kill: def $eax killed $eax killed $rax
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    subl %esi, %eax
+; X64-NEXT:    subl %edi, %esi
+; X64-NEXT:    cmovgel %esi, %eax
 ; X64-NEXT:    retq
   %min = call i32 @llvm.smin.i32(i32 %a, i32 %b)
   %max = call i32 @llvm.smax.i32(i32 %a, i32 %b)
@@ -594,9 +573,8 @@ define i64 @abd_minmax_i64(i64 %a, i64 %b) nounwind {
 ; X64:       # %bb.0:
 ; X64-NEXT:    movq %rdi, %rax
 ; X64-NEXT:    subq %rsi, %rax
-; X64-NEXT:    negq %rax
-; X64-NEXT:    subq %rsi, %rdi
-; X64-NEXT:    cmovgq %rdi, %rax
+; X64-NEXT:    subq %rdi, %rsi
+; X64-NEXT:    cmovgeq %rsi, %rax
 ; X64-NEXT:    retq
   %min = call i64 @llvm.smin.i64(i64 %a, i64 %b)
   %max = call i64 @llvm.smax.i64(i64 %a, i64 %b)
diff --git a/llvm/test/CodeGen/X86/abdu-neg.ll b/llvm/test/CodeGen/X86/abdu-neg.ll
index fe4525009ad5c81..9cb3e5e8bf0c263 100644
--- a/llvm/test/CodeGen/X86/abdu-neg.ll
+++ b/llvm/test/CodeGen/X86/abdu-neg.ll
@@ -139,27 +139,25 @@ define i16 @abd_ext_i16(i16 %a, i16 %b) nounwind {
 define i16 @abd_ext_i16_i32(i16 %a, i32 %b) nounwind {
 ; X86-LABEL: abd_ext_i16_i32:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl %eax, %edx
-; X86-NEXT:    subl %ecx, %edx
-; X86-NEXT:    negl %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, %edx
+; X86-NEXT:    subl %eax, %edx
 ; X86-NEXT:    subl %ecx, %eax
-; X86-NEXT:    cmovbel %edx, %eax
+; X86-NEXT:    cmovbl %edx, %eax
 ; X86-NEXT:    negl %eax
 ; X86-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: abd_ext_i16_i32:
 ; X64:       # %bb.0:
-; X64-NEXT:    movl %esi, %eax
 ; X64-NEXT:    movzwl %di, %ecx
-; X64-NEXT:    subq %rax, %rcx
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    negq %rax
-; X64-NEXT:    cmovsq %rcx, %rax
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    subl %esi, %eax
+; X64-NEXT:    subl %ecx, %esi
+; X64-NEXT:    cmovael %esi, %eax
 ; X64-NEXT:    negl %eax
-; X64-NEXT:    # kill: def $ax killed $ax killed $rax
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X64-NEXT:    retq
   %aext = zext i16 %a to i64
   %bext = zext i32 %b to i64
@@ -204,26 +202,22 @@ define i16 @abd_ext_i16_undef(i16 %a, i16 %b) nounwind {
 define i32 @abd_ext_i32(i32 %a, i32 %b) nounwind {
 ; X86-LABEL: abd_ext_i32:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl %eax, %edx
-; X86-NEXT:    subl %ecx, %edx
-; X86-NEXT:    negl %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, %edx
+; X86-NEXT:    subl %eax, %edx
 ; X86-NEXT:    subl %ecx, %eax
-; X86-NEXT:    cmovbel %edx, %eax
+; X86-NEXT:    cmovbl %edx, %eax
 ; X86-NEXT:    negl %eax
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: abd_ext_i32:
 ; X64:       # %bb.0:
-; X64-NEXT:    movl %esi, %eax
-; X64-NEXT:    movl %edi, %ecx
-; X64-NEXT:    subq %rax, %rcx
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    negq %rax
-; X64-NEXT:    cmovsq %rcx, %rax
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    subl %esi, %eax
+; X64-NEXT:    subl %edi, %esi
+; X64-NEXT:    cmovael %esi, %eax
 ; X64-NEXT:    negl %eax
-; X64-NEXT:    # kill: def $eax killed $eax killed $rax
 ; X64-NEXT:    retq
   %aext = zext i32 %a to i64
   %bext = zext i32 %b to i64
@@ -237,26 +231,23 @@ define i32 @abd_ext_i32(i32 %a, i32 %b) nounwind {
 define i32 @abd_ext_i32_i16(i32 %a, i16 %b) nounwind {
 ; X86-LABEL: abd_ext_i32_i16:
 ; X86:       # %bb.0:
-; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl %eax, %edx
-; X86-NEXT:    subl %ecx, %edx
-; X86-NEXT:    negl %edx
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, %edx
+; X86-NEXT:    subl %eax, %edx
 ; X86-NEXT:    subl %ecx, %eax
-; X86-NEXT:    cmovbel %edx, %eax
+; X86-NEXT:    cmovbl %edx, %eax
 ; X86-NEXT:    negl %eax
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: abd_ext_i32_i16:
 ; X64:       # %bb.0:
-; X64-NEXT:    movl %edi, %ecx
 ; X64-NEXT:    movzwl %si, %eax
-; X64-NEXT:    subq %rax, %rcx
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    negq %rax
-; X64-NEXT:    cmovsq %rcx, %rax
+; X64-NEXT:    movl %edi, %ecx
+; X64-NEXT:    subl %eax, %ecx
+; X64-NEXT:    subl %edi, %eax
+; X64-NEXT:    cmovbl %ecx, %eax
 ; X64-NEXT:    negl %eax
-; X64-NEXT:    # kill: def $eax killed $eax killed $rax
 ; X64-NEXT:    retq
   %aext = zext i32 %a to i64
   %bext = zext i16 %b to i64
@@ -270,26 +261,22 @@ define i32 @abd_ext_i32_i16(i32 %a, i16 %b) nounwind {
 define i32 @abd_ext_i32_undef(i32 %a, i32 %b) nounwind {
 ; X86-LABEL: abd_ext_i32_undef:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl %eax, %edx
-; X86-NEXT:    subl %ecx, %edx
-; X86-NEXT:    negl %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, %edx
+; X86-NEXT:    subl %eax, %edx
 ; X86-NEXT:    subl %ecx, %eax
-; X86-NEXT:    cmovbel %edx, %eax
+; X86-NEXT:    cmovbl %edx, %eax
 ; X86-NEXT:    negl %eax
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: abd_ext_i32_undef:
 ; X64:       # %bb.0:
-; X64-NEXT:    movl %esi, %eax
-; X64-NEXT:    movl %edi, %ecx
-; X64-NEXT:    subq %rax, %rcx
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    negq %rax
-; X64-NEXT:    cmovsq %rcx, %rax
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    subl %esi, %eax
+; X64-NEXT:    subl %edi, %esi
+; X64-NEXT:    cmovael %esi, %eax
 ; X64-NEXT:    negl %eax
-; X64-NEXT:    # kill: def $eax killed $eax killed $rax
 ; X64-NEXT:    retq
   %aext = zext i32 %a to i64
   %bext = zext i32 %b to i64
@@ -324,9 +311,8 @@ define i64 @abd_ext_i64(i64 %a, i64 %b) nounwind {
 ; X64:       # %bb.0:
 ; X64-NEXT:    movq %rdi, %rax
 ; X64-NEXT:    subq %rsi, %rax
-; X64-NEXT:    negq %rax
-; X64-NEXT:    subq %rsi, %rdi
-; X64-NEXT:    cmovaq %rdi, %rax
+; X64-NEXT:    subq %rdi, %rsi
+; X64-NEXT:    cmovaeq %rsi, %rax
 ; X64-NEXT:    negq %rax
 ; X64-NEXT:    retq
   %aext = zext i64 %a to i128
@@ -362,9 +348,8 @@ define i64 @abd_ext_i64_undef(i64 %a, i64 %b) nounwind {
 ; X64:       # %bb.0:
 ; X64-NEXT:    movq %rdi, %rax
 ; X64-NEXT:    subq %rsi, %rax
-; X64-NEXT:    negq %rax
-; X64-NEXT:    subq %rsi, %rdi
-; X64-NEXT:    cmovaq %rdi, %rax
+; X64-NEXT:    subq %rdi, %rsi
+; X64-NEXT:    cmovaeq %rsi, %rax
 ; X64-NEXT:    negq %rax
 ; X64-NEXT:    retq
   %aext = zext i64 %a to i128
diff --git a/llvm/test/CodeGen/X86/abdu.ll b/llvm/test/CodeGen/X86/abdu.ll
index 2e2a1debc38cb13..e808e0f21babf28 100644
--- a/llvm/test/CodeGen/X86/abdu.ll
+++ b/llvm/test/CodeGen/X86/abdu.ll
@@ -129,25 +129,23 @@ define i16 @abd_ext_i16(i16 %a, i16 %b) nounwind {
 define i16 @abd_ext_i16_i32(i16 %a, i32 %b) nounwind {
 ; X86-LABEL: abd_ext_i16_i32:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl %eax, %edx
-; X86-NEXT:    subl %ecx, %edx
-; X86-NEXT:    negl %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, %edx
+; X86-NEXT:    subl %eax, %edx
 ; X86-NEXT:    subl %ecx, %eax
-; X86-NEXT:    cmovbel %edx, %eax
+; X86-NEXT:    cmovbl %edx, %eax
 ; X86-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: abd_ext_i16_i32:
 ; X64:       # %bb.0:
-; X64-NEXT:    movl %esi, %eax
 ; X64-NEXT:    movzwl %di, %ecx
-; X64-NEXT:    subq %rax, %rcx
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    negq %rax
-; X64-NEXT:    cmovsq %rcx, %rax
-; X64-NEXT:    # kill: def $ax killed $ax killed $rax
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    subl %esi, %eax
+; X64-NEXT:    subl %ecx, %esi
+; X64-NEXT:    cmovael %esi, %eax
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X64-NEXT:    retq
   %aext = zext i16 %a to i64
   %bext = zext i32 %b to i64
@@ -190,24 +188,20 @@ define i16 @abd_ext_i16_undef(i16 %a, i16 %b) nounwind {
 define i32 @abd_ext_i32(i32 %a, i32 %b) nounwind {
 ; X86-LABEL: abd_ext_i32:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl %eax, %edx
-; X86-NEXT:    subl %ecx, %edx
-; X86-NEXT:    negl %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, %edx
+; X86-NEXT:    subl %eax, %edx
 ; X86-NEXT:    subl %ecx, %eax
-; X86-NEXT:    cmovbel %edx, %eax
+; X86-NEXT:    cmovbl %edx, %eax
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: abd_ext_i32:
 ; X64:       # %bb.0:
-; X64-NEXT:    movl %esi, %eax
-; X64-NEXT:    movl %edi, %ecx
-; X64-NEXT:    subq %rax, %rcx
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    negq %rax
-; X64-NEXT:    cmovsq %rcx, %rax
-; X64-NEXT:    # kill: def $eax killed $eax killed $rax
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    subl %esi, %eax
+; X64-NEXT:    subl %edi, %esi
+; X64-NEXT:    cmovael %esi, %eax
 ; X64-NEXT:    retq
   %aext = zext i32 %a to i64
   %bext = zext i32 %b to i64
@@ -220,24 +214,21 @@ define i32 @abd_ext_i32(i32 %a, i32 %b) nounwind {
 define i32 @abd_ext_i32_i16(i32 %a, i16 %b) nounwind {
 ; X86-LABEL: abd_ext_i32_i16:
 ; X86:       # %bb.0:
-; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl %eax, %edx
-; X86-NEXT:    subl %ecx, %edx
-; X86-NEXT:    negl %edx
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, %edx
+; X86-NEXT:    subl %eax, %edx
 ; X86-NEXT:    subl %ecx, %eax
-; X86-NEXT:    cmovbel %edx, %eax
+; X86-NEXT:    cmovbl %edx, %eax
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: abd_ext_i32_i16:
 ; X64:       # %bb.0:
-; X64-NEXT:    movl %edi, %ecx
 ; X64-NEXT:    movzwl %si, %eax
-; X64-NEXT:    subq %rax, %rcx
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    negq %rax
-; X64-NEXT:    cmovsq %rcx, %rax
-; X64-NEXT:    # kill: def $eax killed $eax killed $rax
+; X64-NEXT:    movl %edi, %ecx
+; X64-NEXT:    subl %eax, %ecx
+; X64-NEXT:    subl %edi, %eax
+; X64-NEXT:    cmovbl %ecx, %eax
 ; X64-NEXT:    retq
   %aext = zext i32 %a to i64
   %bext = zext i16 %b to i64
@@ -250,24 +241,20 @@ define i32 @abd_ext_i32_i16(i32 %a, i16 %b) nounwind {
 define i32 @abd_ext_i32_undef(i32 %a, i32 %b) nounwind {
 ; X86-LABEL: abd_ext_i32_undef:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl %eax, %edx
-; X86-NEXT:    subl %ecx, %edx
-; X86-NEXT:    negl %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, %edx
+; X86-NEXT:    subl %eax, %edx
 ; X86-NEXT:    subl %ecx, %eax
-; X86-NEXT:    cmovbel %edx, %eax
+; X86-NEXT:    cmovbl %edx, %eax
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: abd_ext_i32_undef:
 ; X64:       # %bb.0:
-; X64-NEXT:    movl %esi, %eax
-; X64-NEXT:    movl %edi, %ecx
-; X64-NEXT:    subq %rax, %rcx
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    negq %rax
-; X64-NEXT:    cmovsq %rcx, %rax
-; X64-NEXT:    # kill: def $eax killed $eax killed $rax
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    subl %esi, %eax
+; X64-NEXT:    subl %edi, %esi
+; X64-NEXT:    cmovael %esi, %eax
 ; X64-NEXT:    retq
   %aext = zext i32 %a to i64
   %bext = zext i32 %b to i64
@@ -296,9 +283,8 @@ define i64 @abd_ext_i64(i64 %a, i64 %b) nounwind {
 ; X64:       # %bb.0:
 ; X64-NEXT:    movq %rdi, %rax
 ; X64-NEXT:    subq %rsi, %rax
-; X64-NEXT:    negq %rax
-; X64-NEXT:    subq %rsi, %rdi
-; X64-NEXT:    cmovaq %rdi, %rax
+; X64-NEXT:    subq %rdi, %rsi
+; X64-NEXT:    cmovaeq %rsi, %rax
 ; X64-NEXT:    retq
   %aext = zext i64 %a to i128
   %bext = zext i64 %b to i128
@@ -327,9 +313,8 @@ define i64 @abd_ext_i64_undef(i64 %a, i64 %b) nounwind {
 ; X64:       # %bb.0:
 ; X64-NEXT:    movq %rdi, %rax
 ; X64-NEXT:    subq %rsi, %rax
-; X64-NEXT:    negq %rax
-; X64-NEXT:    subq %rsi, %rdi
-; X64-NEXT:    cmovaq %rdi, %rax
+; X64-NEXT:    subq %rdi, %rsi
+; X64-NEXT:    cmovaeq %rsi, %rax
 ; X64-NEXT:    retq
   %aext = zext i64 %a to i128
   %bext = zext i64 %b to i128
@@ -512,24 +497,20 @@ define i16 @abd_minmax_i16(i16 %a, i16 %b) nounwind {
 define i32 @abd_minmax_i32(i32 %a, i32 %b) nounwind {
 ; X86-LABEL: abd_minmax_i32:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl %eax, %edx
-; X86-NEXT:    subl %ecx, %edx
-; X86-NEXT:    negl %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, %edx
+; X86-NEXT:    subl %eax, %edx
 ; X86-NEXT:    subl %ecx, %eax
-; X86-NEXT:    cmovbel %edx, %eax
+; X86-NEXT:    cmovbl %edx, %eax
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: abd_minmax_i32:
 ; X64:       # %bb.0:
-; X64-NEXT:    movl %esi, %eax
-; X64-NEXT:    movl %edi, %ecx
-; X64-NEXT:    subq %rax, %rcx
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    negq %rax
-; X64-NEXT:    cmovsq %rcx, %rax
-; X64-NEXT:    # kill: def $eax killed $eax killed $rax
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    subl %esi, %eax
+; X64-NEXT:    subl %edi, %esi
+; X64-NEXT:    cmovael %esi, %eax
 ; X64-NEXT:    retq
   %min = call i32 @llvm.umin.i32(i32 %a, i32 %b)
   %max = call i32 @llvm.umax.i32(i32 %a, i32 %b)
@@ -556,9 +537,8 @@ define i64 @abd_minmax_i64(i64 %a, i64 %b) nounwind {
 ; X64:       # %bb.0:
 ; X64-NEXT:    movq %rdi, %rax
 ; X64-NEXT:    subq %rsi, %rax
-; X64-NEXT:    negq %rax
-; X64-NEXT:    subq %rsi, %rdi
-; X64-NEXT:    cmovaq %rdi, %rax
+; X64-NEXT:    subq %rdi, %rsi
+; X64-NEXT:    cmovaeq %rsi, %rax
 ; X64-NEXT:    retq
   %min = call i64 @llvm.umin.i64(i64 %a, i64 %b)
   %max = call i64 @llvm.umax.i64(i64 %a, i64 %b)

From 50daa2397f831f3577cf8f9057f047021867dc88 Mon Sep 17 00:00:00 2001
From: Sergei Barannikov <barannikov88@gmail.com>
Date: Mon, 19 Aug 2024 16:09:46 +0300
Subject: [PATCH 353/441] [DataLayout] Refactor parsing of i/f/v/a
 specifications (#104699)

Split off of #104545 to reduce patch size.
---
 llvm/include/llvm/IR/DataLayout.h             |  23 +-
 llvm/lib/IR/DataLayout.cpp                    | 182 ++++++++--------
 llvm/test/Transforms/InstCombine/crash.ll     |   2 +-
 llvm/test/Transforms/InstCombine/phi.ll       |   2 +-
 llvm/unittests/IR/DataLayoutTest.cpp          | 198 ++++++++++++++----
 .../Target/LLVMIR/Import/import-failure.ll    |   6 -
 6 files changed, 262 insertions(+), 151 deletions(-)

diff --git a/llvm/include/llvm/IR/DataLayout.h b/llvm/include/llvm/IR/DataLayout.h
index 85dae10883c67cc..2f06bda6c30a51f 100644
--- a/llvm/include/llvm/IR/DataLayout.h
+++ b/llvm/include/llvm/IR/DataLayout.h
@@ -115,14 +115,6 @@ class DataLayout {
   // FIXME: `unsigned char` truncates the value parsed by `parseSpecifier`.
   SmallVector<unsigned char, 8> LegalIntWidths;
 
-  /// Type specifier used by some internal functions.
-  enum class TypeSpecifier {
-    Integer = 'i',
-    Float = 'f',
-    Vector = 'v',
-    Aggregate = 'a'
-  };
-
   /// Primitive type specifications. Sorted and uniqued by type bit width.
   SmallVector<PrimitiveSpec, 6> IntSpecs;
   SmallVector<PrimitiveSpec, 4> FloatSpecs;
@@ -145,10 +137,9 @@ class DataLayout {
   /// well-defined bitwise representation.
   SmallVector<unsigned, 8> NonIntegralAddressSpaces;
 
-  /// Attempts to set the specification for the given type.
-  /// Returns an error description on failure.
-  Error setPrimitiveSpec(TypeSpecifier Specifier, uint32_t BitWidth,
-                         Align ABIAlign, Align PrefAlign);
+  /// Sets or updates the specification for the given primitive type.
+  void setPrimitiveSpec(char Specifier, uint32_t BitWidth, Align ABIAlign,
+                        Align PrefAlign);
 
   /// Searches for a pointer specification that matches the given address space.
   /// Returns the default address space specification if not found.
@@ -164,7 +155,13 @@ class DataLayout {
   /// Internal helper method that returns requested alignment for type.
   Align getAlignment(Type *Ty, bool abi_or_pref) const;
 
-  /// Attempts to parse a pointer specification ('p').
+  /// Attempts to parse primitive specification ('i', 'f', or 'v').
+  Error parsePrimitiveSpec(StringRef Spec);
+
+  /// Attempts to parse aggregate specification ('a').
+  Error parseAggregateSpec(StringRef Spec);
+
+  /// Attempts to parse pointer specification ('p').
   Error parsePointerSpec(StringRef Spec);
 
   /// Attempts to parse a single specification.
diff --git a/llvm/lib/IR/DataLayout.cpp b/llvm/lib/IR/DataLayout.cpp
index 425a9024c897d4e..5efdaaccf1fc7fc 100644
--- a/llvm/lib/IR/DataLayout.cpp
+++ b/llvm/lib/IR/DataLayout.cpp
@@ -376,6 +376,82 @@ static Error getAddrSpace(StringRef R, unsigned &AddrSpace) {
   return Error::success();
 }
 
+Error DataLayout::parsePrimitiveSpec(StringRef Spec) {
+  // [ifv]<size>:<abi>[:<pref>]
+  SmallVector<StringRef, 3> Components;
+  char Specifier = Spec.front();
+  assert(Specifier == 'i' || Specifier == 'f' || Specifier == 'v');
+  Spec.drop_front().split(Components, ':');
+
+  if (Components.size() < 2 || Components.size() > 3)
+    return createSpecFormatError(Twine(Specifier) + "<size>:<abi>[:<pref>]");
+
+  // Size. Required, cannot be zero.
+  unsigned BitWidth;
+  if (Error Err = parseSize(Components[0], BitWidth))
+    return Err;
+
+  // ABI alignment.
+  Align ABIAlign;
+  if (Error Err = parseAlignment(Components[1], ABIAlign, "ABI"))
+    return Err;
+
+  if (Specifier == 'i' && BitWidth == 8 && ABIAlign != 1)
+    return createStringError("i8 must be 8-bit aligned");
+
+  // Preferred alignment. Optional, defaults to the ABI alignment.
+  Align PrefAlign = ABIAlign;
+  if (Components.size() > 2)
+    if (Error Err = parseAlignment(Components[2], PrefAlign, "preferred"))
+      return Err;
+
+  if (PrefAlign < ABIAlign)
+    return createStringError(
+        "preferred alignment cannot be less than the ABI alignment");
+
+  setPrimitiveSpec(Specifier, BitWidth, ABIAlign, PrefAlign);
+  return Error::success();
+}
+
+Error DataLayout::parseAggregateSpec(StringRef Spec) {
+  // a<size>:<abi>[:<pref>]
+  SmallVector<StringRef, 3> Components;
+  assert(Spec.front() == 'a');
+  Spec.drop_front().split(Components, ':');
+
+  if (Components.size() < 2 || Components.size() > 3)
+    return createSpecFormatError("a:<abi>[:<pref>]");
+
+  // According to LangRef, <size> component must be absent altogether.
+  // For backward compatibility, allow it to be specified, but require
+  // it to be zero.
+  if (!Components[0].empty()) {
+    unsigned BitWidth;
+    if (!to_integer(Components[0], BitWidth, 10) || BitWidth != 0)
+      return createStringError("size must be zero");
+  }
+
+  // ABI alignment. Required. Can be zero, meaning use one byte alignment.
+  Align ABIAlign;
+  if (Error Err =
+          parseAlignment(Components[1], ABIAlign, "ABI", /*AllowZero=*/true))
+    return Err;
+
+  // Preferred alignment. Optional, defaults to the ABI alignment.
+  Align PrefAlign = ABIAlign;
+  if (Components.size() > 2)
+    if (Error Err = parseAlignment(Components[2], PrefAlign, "preferred"))
+      return Err;
+
+  if (PrefAlign < ABIAlign)
+    return createStringError(
+        "preferred alignment cannot be less than the ABI alignment");
+
+  StructABIAlignment = ABIAlign;
+  StructPrefAlignment = PrefAlign;
+  return Error::success();
+}
+
 Error DataLayout::parsePointerSpec(StringRef Spec) {
   // p[<n>]:<size>:<abi>[:<pref>[:<idx>]]
   SmallVector<StringRef, 5> Components;
@@ -451,6 +527,12 @@ Error DataLayout::parseSpecification(StringRef Spec) {
   assert(!Spec.empty() && "Empty specification is handled by the caller");
   char Specifier = Spec.front();
 
+  if (Specifier == 'i' || Specifier == 'f' || Specifier == 'v')
+    return parsePrimitiveSpec(Spec);
+
+  if (Specifier == 'a')
+    return parseAggregateSpec(Spec);
+
   if (Specifier == 'p')
     return parsePointerSpec(Spec);
 
@@ -477,78 +559,6 @@ Error DataLayout::parseSpecification(StringRef Spec) {
   case 'e':
     BigEndian = false;
     break;
-  case 'i':
-  case 'v':
-  case 'f':
-  case 'a': {
-    TypeSpecifier Specifier;
-    switch (SpecifierChar) {
-    default:
-      llvm_unreachable("Unexpected specifier!");
-    case 'i':
-      Specifier = TypeSpecifier::Integer;
-      break;
-    case 'v':
-      Specifier = TypeSpecifier::Vector;
-      break;
-    case 'f':
-      Specifier = TypeSpecifier::Float;
-      break;
-    case 'a':
-      Specifier = TypeSpecifier::Aggregate;
-      break;
-    }
-
-    // Bit size.
-    unsigned Size = 0;
-    if (!Tok.empty())
-      if (Error Err = getInt(Tok, Size))
-        return Err;
-
-    if (Specifier == TypeSpecifier::Aggregate && Size != 0)
-      return reportError("Sized aggregate specification in datalayout string");
-
-    // ABI alignment.
-    if (Rest.empty())
-      return reportError(
-          "Missing alignment specification in datalayout string");
-    if (Error Err = ::split(Rest, ':', Split))
-      return Err;
-    unsigned ABIAlign;
-    if (Error Err = getIntInBytes(Tok, ABIAlign))
-      return Err;
-    if (Specifier != TypeSpecifier::Aggregate && !ABIAlign)
-      return reportError(
-          "ABI alignment specification must be >0 for non-aggregate types");
-
-    if (!isUInt<16>(ABIAlign))
-      return reportError("Invalid ABI alignment, must be a 16bit integer");
-    if (ABIAlign != 0 && !isPowerOf2_64(ABIAlign))
-      return reportError("Invalid ABI alignment, must be a power of 2");
-    if (Specifier == TypeSpecifier::Integer && Size == 8 && ABIAlign != 1)
-      return reportError("Invalid ABI alignment, i8 must be naturally aligned");
-
-    // Preferred alignment.
-    unsigned PrefAlign = ABIAlign;
-    if (!Rest.empty()) {
-      if (Error Err = ::split(Rest, ':', Split))
-        return Err;
-      if (Error Err = getIntInBytes(Tok, PrefAlign))
-        return Err;
-    }
-
-    if (!isUInt<16>(PrefAlign))
-      return reportError(
-          "Invalid preferred alignment, must be a 16bit integer");
-    if (PrefAlign != 0 && !isPowerOf2_64(PrefAlign))
-      return reportError("Invalid preferred alignment, must be a power of 2");
-
-    if (Error Err = setPrimitiveSpec(Specifier, Size, assumeAligned(ABIAlign),
-                                     assumeAligned(PrefAlign)))
-      return Err;
-
-    break;
-  }
   case 'n': // Native integer types.
     while (true) {
       unsigned Width;
@@ -668,32 +678,19 @@ Error DataLayout::parseLayoutString(StringRef LayoutString) {
   return Error::success();
 }
 
-Error DataLayout::setPrimitiveSpec(TypeSpecifier Specifier, uint32_t BitWidth,
-                                   Align ABIAlign, Align PrefAlign) {
-  // AlignmentsTy::ABIAlign and AlignmentsTy::PrefAlign were once stored as
-  // uint16_t, it is unclear if there are requirements for alignment to be less
-  // than 2^16 other than storage. In the meantime we leave the restriction as
-  // an assert. See D67400 for context.
-  assert(Log2(ABIAlign) < 16 && Log2(PrefAlign) < 16 && "Alignment too big");
-  if (!isUInt<24>(BitWidth))
-    return reportError("Invalid bit width, must be a 24-bit integer");
-  if (PrefAlign < ABIAlign)
-    return reportError(
-        "Preferred alignment cannot be less than the ABI alignment");
-
+void DataLayout::setPrimitiveSpec(char Specifier, uint32_t BitWidth,
+                                  Align ABIAlign, Align PrefAlign) {
   SmallVectorImpl<PrimitiveSpec> *Specs;
   switch (Specifier) {
-  case TypeSpecifier::Aggregate:
-    StructABIAlignment = ABIAlign;
-    StructPrefAlignment = PrefAlign;
-    return Error::success();
-  case TypeSpecifier::Integer:
+  default:
+    llvm_unreachable("Unexpected specifier");
+  case 'i':
     Specs = &IntSpecs;
     break;
-  case TypeSpecifier::Float:
+  case 'f':
     Specs = &FloatSpecs;
     break;
-  case TypeSpecifier::Vector:
+  case 'v':
     Specs = &VectorSpecs;
     break;
   }
@@ -707,7 +704,6 @@ Error DataLayout::setPrimitiveSpec(TypeSpecifier Specifier, uint32_t BitWidth,
     // Insert before I to keep the vector sorted.
     Specs->insert(I, PrimitiveSpec{BitWidth, ABIAlign, PrefAlign});
   }
-  return Error::success();
 }
 
 const DataLayout::PointerSpec &
diff --git a/llvm/test/Transforms/InstCombine/crash.ll b/llvm/test/Transforms/InstCombine/crash.ll
index 5f86069ef736803..9b37d6943b9e52a 100644
--- a/llvm/test/Transforms/InstCombine/crash.ll
+++ b/llvm/test/Transforms/InstCombine/crash.ll
@@ -1,5 +1,5 @@
 ; RUN: opt < %s -passes=instcombine -S
-target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128:n8:16:32"
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32"
 target triple = "i386-apple-darwin10.0"
 
 define i32 @test0(i8 %tmp2) ssp {
diff --git a/llvm/test/Transforms/InstCombine/phi.ll b/llvm/test/Transforms/InstCombine/phi.ll
index b12982dd27e404d..2673b1d74bb6fb8 100644
--- a/llvm/test/Transforms/InstCombine/phi.ll
+++ b/llvm/test/Transforms/InstCombine/phi.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -passes=instcombine -S | FileCheck %s
 
-target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128:n8:16:32:64"
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 define i32 @test1(i32 %A, i1 %b) {
 ; CHECK-LABEL: @test1(
diff --git a/llvm/unittests/IR/DataLayoutTest.cpp b/llvm/unittests/IR/DataLayoutTest.cpp
index 4b236b2d165f9c3..9464b489b65f43c 100644
--- a/llvm/unittests/IR/DataLayoutTest.cpp
+++ b/llvm/unittests/IR/DataLayoutTest.cpp
@@ -19,6 +19,8 @@ using namespace llvm;
 
 namespace {
 
+class DataLayoutTest : public ::testing::Test {};
+
 // TODO: Split into multiple TESTs.
 TEST(DataLayoutTest, ParseErrors) {
   EXPECT_THAT_EXPECTED(
@@ -30,12 +32,6 @@ TEST(DataLayoutTest, ParseErrors) {
   EXPECT_THAT_EXPECTED(
       DataLayout::parse("n0"),
       FailedWithMessage("Zero width native integer type in datalayout string"));
-  EXPECT_THAT_EXPECTED(
-      DataLayout::parse("a1:64"),
-      FailedWithMessage("Sized aggregate specification in datalayout string"));
-  EXPECT_THAT_EXPECTED(
-      DataLayout::parse("a:"),
-      FailedWithMessage("Trailing separator in datalayout string"));
   EXPECT_THAT_EXPECTED(
       DataLayout::parse("m"),
       FailedWithMessage("Expected mangling specifier in datalayout string"));
@@ -43,38 +39,10 @@ TEST(DataLayoutTest, ParseErrors) {
       DataLayout::parse("m."),
       FailedWithMessage("Unexpected trailing characters after mangling "
                         "specifier in datalayout string"));
-  EXPECT_THAT_EXPECTED(
-      DataLayout::parse("f"),
-      FailedWithMessage(
-          "Missing alignment specification in datalayout string"));
   EXPECT_THAT_EXPECTED(
       DataLayout::parse(":32"),
       FailedWithMessage(
           "Expected token before separator in datalayout string"));
-  EXPECT_THAT_EXPECTED(
-      DataLayout::parse("i64:64:16"),
-      FailedWithMessage(
-          "Preferred alignment cannot be less than the ABI alignment"));
-  EXPECT_THAT_EXPECTED(
-      DataLayout::parse("i64:16:16777216"),
-      FailedWithMessage(
-          "Invalid preferred alignment, must be a 16bit integer"));
-  EXPECT_THAT_EXPECTED(
-      DataLayout::parse("i64:16777216:16777216"),
-      FailedWithMessage("Invalid ABI alignment, must be a 16bit integer"));
-  EXPECT_THAT_EXPECTED(
-      DataLayout::parse("i16777216:16:16"),
-      FailedWithMessage("Invalid bit width, must be a 24-bit integer"));
-  EXPECT_THAT_EXPECTED(
-      DataLayout::parse("v128:0:128"),
-      FailedWithMessage(
-          "ABI alignment specification must be >0 for non-aggregate types"));
-  EXPECT_THAT_EXPECTED(
-      DataLayout::parse("i32:24:32"),
-      FailedWithMessage("Invalid ABI alignment, must be a power of 2"));
-  EXPECT_THAT_EXPECTED(
-      DataLayout::parse("i32:32:24"),
-      FailedWithMessage("Invalid preferred alignment, must be a power of 2"));
   EXPECT_THAT_EXPECTED(
       DataLayout::parse("A16777216"),
       FailedWithMessage("Invalid address space, must be a 24-bit integer"));
@@ -87,9 +55,6 @@ TEST(DataLayoutTest, ParseErrors) {
   EXPECT_THAT_EXPECTED(
       DataLayout::parse("Fi24"),
       FailedWithMessage("Alignment is neither 0 nor a power of 2"));
-  EXPECT_THAT_EXPECTED(
-      DataLayout::parse("i8:16"),
-      FailedWithMessage("Invalid ABI alignment, i8 must be naturally aligned"));
   EXPECT_THAT_EXPECTED(
       DataLayout::parse("S24"),
       FailedWithMessage("Alignment is neither 0 nor a power of 2"));
@@ -105,6 +70,165 @@ TEST(DataLayout, LayoutStringFormat) {
         FailedWithMessage("empty specification is not allowed"));
 }
 
+class DataLayoutPrimitiveSpecificationTest
+    : public DataLayoutTest,
+      public ::testing::WithParamInterface<char> {
+  char Specifier;
+
+public:
+  DataLayoutPrimitiveSpecificationTest() : Specifier(GetParam()) {}
+
+  std::string format(StringRef Str) const {
+    std::string Res = Str.str();
+    std::replace(Res.begin(), Res.end(), '!', Specifier);
+    return Res;
+  }
+};
+
+INSTANTIATE_TEST_SUITE_P(PrmitiveSpecifiers,
+                         DataLayoutPrimitiveSpecificationTest,
+                         ::testing::Values('i', 'f', 'v'));
+
+TEST_P(DataLayoutPrimitiveSpecificationTest, ParsePrimitiveSpec) {
+  for (StringRef Str :
+       {"!1:16", "!8:8:8", "!16:32:64", "!16777215:32768:32768"})
+    EXPECT_THAT_EXPECTED(DataLayout::parse(format(Str)), Succeeded());
+
+  for (StringRef Str : {"!", "!1", "!32:32:32:32", "!16:32:64:128"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(format(Str)),
+        FailedWithMessage(format("malformed specification, must be of the form "
+                                 "\"!<size>:<abi>[:<pref>]\"")));
+
+  // size
+  for (StringRef Str : {"!:8", "!:16:16", "!:32:64"})
+    EXPECT_THAT_EXPECTED(DataLayout::parse(format(Str)),
+                         FailedWithMessage("size component cannot be empty"));
+
+  for (StringRef Str :
+       {"!0:8", "!0x8:8", "!x:8:8", "!0:16:32", "!16777216:64:64"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(format(Str)),
+        FailedWithMessage("size must be a non-zero 24-bit integer"));
+
+  // ABI alignment
+  for (StringRef Str : {"!8:", "!16::16", "!32::64"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(format(Str)),
+        FailedWithMessage("ABI alignment component cannot be empty"));
+
+  for (StringRef Str : {"!1:x", "!8:8x:8", "!16:65536:65536"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(format(Str)),
+        FailedWithMessage("ABI alignment must be a 16-bit integer"));
+
+  for (StringRef Str : {"!8:0", "!16:0:16", "!32:0:64"})
+    EXPECT_THAT_EXPECTED(DataLayout::parse(format(Str)),
+                         FailedWithMessage("ABI alignment must be non-zero"));
+
+  for (StringRef Str : {"!1:1", "!8:4", "!16:6:16", "!32:24:64"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(format(Str)),
+        FailedWithMessage(
+            "ABI alignment must be a power of two times the byte width"));
+
+  // preferred alignment
+  for (StringRef Str : {"!1:8:", "!16:16:", "!64:32:"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(format(Str)),
+        FailedWithMessage("preferred alignment component cannot be empty"));
+
+  for (StringRef Str : {"!1:8:x", "!8:8:0x8", "!16:32:65536"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(format(Str)),
+        FailedWithMessage("preferred alignment must be a 16-bit integer"));
+
+  for (StringRef Str : {"!8:8:0", "!32:16:0"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(format(Str)),
+        FailedWithMessage("preferred alignment must be non-zero"));
+
+  for (StringRef Str : {"!1:8:12", "!8:8:17", "!16:32:40"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(format(Str)),
+        FailedWithMessage(
+            "preferred alignment must be a power of two times the byte width"));
+
+  for (StringRef Str : {"!1:16:8", "!64:32:16"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(format(Str)),
+        FailedWithMessage(
+            "preferred alignment cannot be less than the ABI alignment"));
+
+  // Additional check for byte-sized integer.
+  if (GetParam() == 'i') {
+    for (StringRef Str : {"!8:16", "!8:16:8", "!8:16:32"})
+      EXPECT_THAT_EXPECTED(DataLayout::parse(format(Str)),
+                           FailedWithMessage("i8 must be 8-bit aligned"));
+  }
+}
+
+TEST(DataLayoutTest, ParseAggregateSpec) {
+  for (StringRef Str : {"a:8", "a:0:16", "a0:32:64", "a:32768:32768"})
+    EXPECT_THAT_EXPECTED(DataLayout::parse(Str), Succeeded());
+
+  for (StringRef Str : {"a", "a0", "a:32:32:32", "a0:32:64:128"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(Str),
+        FailedWithMessage("malformed specification, must be of the form "
+                          "\"a:<abi>[:<pref>]\""));
+
+  // size
+  for (StringRef Str : {"a1:8", "a0x0:8", "ax:16:32"})
+    EXPECT_THAT_EXPECTED(DataLayout::parse(Str),
+                         FailedWithMessage("size must be zero"));
+
+  // ABI alignment
+  for (StringRef Str : {"a:", "a0:", "a::32"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(Str),
+        FailedWithMessage("ABI alignment component cannot be empty"));
+
+  for (StringRef Str : {"a:x", "a0:0x0", "a:65536", "a0:65536:65536"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(Str),
+        FailedWithMessage("ABI alignment must be a 16-bit integer"));
+
+  for (StringRef Str : {"a:1", "a:4", "a:9:16", "a0:24:32"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(Str),
+        FailedWithMessage(
+            "ABI alignment must be a power of two times the byte width"));
+
+  // preferred alignment
+  for (StringRef Str : {"a:8:", "a0:16:", "a0:0:"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(Str),
+        FailedWithMessage("preferred alignment component cannot be empty"));
+
+  for (StringRef Str : {"a:16:x", "a0:8:0x8", "a:16:65536"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(Str),
+        FailedWithMessage("preferred alignment must be a 16-bit integer"));
+
+  for (StringRef Str : {"a:0:0", "a0:16:0"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(Str),
+        FailedWithMessage("preferred alignment must be non-zero"));
+
+  for (StringRef Str : {"a:8:12", "a:16:17", "a0:32:40"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(Str),
+        FailedWithMessage(
+            "preferred alignment must be a power of two times the byte width"));
+
+  for (StringRef Str : {"a:16:8", "a0:32:16"})
+    EXPECT_THAT_EXPECTED(
+        DataLayout::parse(Str),
+        FailedWithMessage(
+            "preferred alignment cannot be less than the ABI alignment"));
+}
+
 TEST(DataLayout, ParsePointerSpec) {
   for (StringRef Str :
        {"p:16:8", "p:16:16:64", "p:32:64:64:32", "p0:32:64", "p42:64:32:32",
diff --git a/mlir/test/Target/LLVMIR/Import/import-failure.ll b/mlir/test/Target/LLVMIR/Import/import-failure.ll
index 9c24ed2b75746bf..6bde174642d5400 100644
--- a/mlir/test/Target/LLVMIR/Import/import-failure.ll
+++ b/mlir/test/Target/LLVMIR/Import/import-failure.ll
@@ -359,12 +359,6 @@ declare void @llvm.experimental.noalias.scope.decl(metadata)
 
 ; // -----
 
-; CHECK:      import-failure.ll
-; CHECK-SAME: error: cannot translate data layout: i8:8:8:8
-target datalayout = "e-i8:8:8:8"
-
-; // -----
-
 ; CHECK:      import-failure.ll
 ; CHECK-SAME: warning: unhandled data layout token: ni:42
 target datalayout = "e-ni:42-i64:64"

From 3e999bbc2ddfc8be7b2d1a250c9905c905e6ce81 Mon Sep 17 00:00:00 2001
From: Timm Baeder <tbaeder@redhat.com>
Date: Mon, 19 Aug 2024 15:14:44 +0200
Subject: [PATCH 354/441] [clang][bytecode] Use first FieldDecl instead of
 asserting (#104760)

This assertion fails sometimes. We use the first decl for lookup later,
so let's use the first decl here as well.
---
 clang/lib/AST/ByteCode/Program.cpp       | 2 +-
 clang/test/Modules/merge-using-decls.cpp | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/clang/lib/AST/ByteCode/Program.cpp b/clang/lib/AST/ByteCode/Program.cpp
index 25b99115ddd777e..a4f0df8bf6462e0 100644
--- a/clang/lib/AST/ByteCode/Program.cpp
+++ b/clang/lib/AST/ByteCode/Program.cpp
@@ -329,7 +329,7 @@ Record *Program::getOrCreateRecord(const RecordDecl *RD) {
   // Reserve space for fields.
   Record::FieldList Fields;
   for (const FieldDecl *FD : RD->fields()) {
-    assert(FD == FD->getFirstDecl());
+    FD = FD->getFirstDecl();
     // Note that we DO create fields and descriptors
     // for unnamed bitfields here, even though we later ignore
     // them everywhere. That's so the FieldDecl's getFieldIndex() matches.
diff --git a/clang/test/Modules/merge-using-decls.cpp b/clang/test/Modules/merge-using-decls.cpp
index e3bf977f0544993..e9794a40837ef97 100644
--- a/clang/test/Modules/merge-using-decls.cpp
+++ b/clang/test/Modules/merge-using-decls.cpp
@@ -6,7 +6,8 @@
 // RUN: %clang_cc1 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -x c++ -I%S/Inputs/merge-using-decls -verify %s -DORDER=2
 // RUN: %clang_cc1 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -x c++ -I%S/Inputs/merge-using-decls -verify -std=c++98 %s -DORDER=2
 // RUN: %clang_cc1 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -x c++ -I%S/Inputs/merge-using-decls -verify -std=c++11 %s -DORDER=2
-// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -x c++ -I%S/Inputs/merge-using-decls -verify -std=c++17 %s -DORDER=2
+
+// RUN: %clang_cc1 -fexperimental-new-constant-interpreter -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -x c++ -I%S/Inputs/merge-using-decls -verify -std=c++17 %s -DORDER=2
 
 #if ORDER == 1
 #include "a.h"

From 13865b09c6b0c767a8c3d4f49662e6c503260976 Mon Sep 17 00:00:00 2001
From: David Green <david.green@arm.com>
Date: Mon, 19 Aug 2024 14:23:39 +0100
Subject: [PATCH 355/441] [AArch64] Add fneg(fmul) and fmul(fneg) tests. NFC

---
 .../Analysis/CostModel/AArch64/arith-fp.ll    | 116 ++++++++++++++++++
 1 file changed, 116 insertions(+)

diff --git a/llvm/test/Analysis/CostModel/AArch64/arith-fp.ll b/llvm/test/Analysis/CostModel/AArch64/arith-fp.ll
index 497ade4f2f613ca..84150765d77973b 100644
--- a/llvm/test/Analysis/CostModel/AArch64/arith-fp.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/arith-fp.ll
@@ -131,6 +131,122 @@ define i32 @fneg(i32 %arg) {
   ret i32 undef
 }
 
+define i32 @fmulfneg(i32 %arg) {
+; CHECK-LABEL: 'fmulfneg'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F16 = fneg half undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F16M = fmul half %F16, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F16 = fneg <2 x half> undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F16M = fmul <2 x half> %V2F16, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F16 = fneg <4 x half> undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F16M = fmul <4 x half> %V4F16, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F16 = fneg <8 x half> undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F16M = fmul <8 x half> %V8F16, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F16 = fneg <16 x half> undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F16M = fmul <16 x half> %V16F16, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fneg float undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F32M = fmul float %F32, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fneg <2 x float> undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32M = fmul <2 x float> %V2F32, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fneg <4 x float> undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32M = fmul <4 x float> %V4F32, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F32 = fneg <8 x float> undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32M = fmul <8 x float> %V8F32, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fneg double undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F64M = fmul double %F64, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fneg <2 x double> undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64M = fmul <2 x double> %V2F64, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F64 = fneg <4 x double> undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64M = fmul <4 x double> %V4F64, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %F16 = fneg half undef
+  %F16M = fmul half %F16, undef
+  %V2F16 = fneg <2 x half> undef
+  %V2F16M = fmul <2 x half> %V2F16, undef
+  %V4F16 = fneg <4 x half> undef
+  %V4F16M = fmul <4 x half> %V4F16, undef
+  %V8F16 = fneg <8 x half> undef
+  %V8F16M = fmul <8 x half> %V8F16, undef
+  %V16F16 = fneg <16 x half> undef
+  %V16F16M = fmul <16 x half> %V16F16, undef
+
+  %F32 = fneg float undef
+  %F32M = fmul float %F32, undef
+  %V2F32 = fneg <2 x float> undef
+  %V2F32M = fmul <2 x float> %V2F32, undef
+  %V4F32 = fneg <4 x float> undef
+  %V4F32M = fmul <4 x float> %V4F32, undef
+  %V8F32 = fneg <8 x float> undef
+  %V8F32M = fmul <8 x float> %V8F32, undef
+
+  %F64 = fneg double undef
+  %F64M = fmul double %F64, undef
+  %V2F64 = fneg <2 x double> undef
+  %V2F64M = fmul <2 x double> %V2F64, undef
+  %V4F64 = fneg <4 x double> undef
+  %V4F64M = fmul <4 x double> %V4F64, undef
+
+  ret i32 undef
+}
+
+define i32 @fnegfmul(i32 %arg) {
+; CHECK-LABEL: 'fnegfmul'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F16M = fmul half undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F16 = fneg half %F16M
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F16M = fmul <2 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F16 = fneg <2 x half> %V2F16M
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F16M = fmul <4 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F16 = fneg <4 x half> %V4F16M
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F16M = fmul <8 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F16 = fneg <8 x half> %V8F16M
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F16M = fmul <16 x half> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F16 = fneg <16 x half> %V16F16M
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F32M = fmul float undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fneg float %F32M
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32M = fmul <2 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fneg <2 x float> %V2F32M
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32M = fmul <4 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fneg <4 x float> %V4F32M
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32M = fmul <8 x float> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F32 = fneg <8 x float> %V8F32M
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F64M = fmul double undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fneg double %F64M
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64M = fmul <2 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fneg <2 x double> %V2F64M
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64M = fmul <4 x double> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F64 = fneg <4 x double> %V4F64M
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %F16M = fmul half undef, undef
+  %F16 = fneg half %F16M
+  %V2F16M = fmul <2 x half> undef, undef
+  %V2F16 = fneg <2 x half> %V2F16M
+  %V4F16M = fmul <4 x half> undef, undef
+  %V4F16 = fneg <4 x half> %V4F16M
+  %V8F16M = fmul <8 x half> undef, undef
+  %V8F16 = fneg <8 x half> %V8F16M
+  %V16F16M = fmul <16 x half> undef, undef
+  %V16F16 = fneg <16 x half> %V16F16M
+
+  %F32M = fmul float undef, undef
+  %F32 = fneg float %F32M
+  %V2F32M = fmul <2 x float> undef, undef
+  %V2F32 = fneg <2 x float> %V2F32M
+  %V4F32M = fmul <4 x float> undef, undef
+  %V4F32 = fneg <4 x float> %V4F32M
+  %V8F32M = fmul <8 x float> undef, undef
+  %V8F32 = fneg <8 x float> %V8F32M
+
+  %F64M = fmul double undef, undef
+  %F64 = fneg double %F64M
+  %V2F64M = fmul <2 x double> undef, undef
+  %V2F64 = fneg <2 x double> %V2F64M
+  %V4F64M = fmul <4 x double> undef, undef
+  %V4F64 = fneg <4 x double> %V4F64M
+
+  ret i32 undef
+}
+
 define i32 @fmul(i32 %arg) {
 ; CHECK-LABEL: 'fmul'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F16 = fmul half undef, undef

From c6605a08681309188fa260a409bf5d37400876a0 Mon Sep 17 00:00:00 2001
From: Timm Baeder <tbaeder@redhat.com>
Date: Mon, 19 Aug 2024 15:28:12 +0200
Subject: [PATCH 356/441] [clang][bytecode] Fix member pointers to
 IndirectFieldDecls (#104756)

---
 clang/lib/AST/ByteCode/MemberPointer.cpp   | 41 ++++++++++++++--------
 clang/test/AST/ByteCode/memberpointers.cpp | 17 +++++++++
 2 files changed, 43 insertions(+), 15 deletions(-)

diff --git a/clang/lib/AST/ByteCode/MemberPointer.cpp b/clang/lib/AST/ByteCode/MemberPointer.cpp
index 0c1b6edc5f7e105..0fe94db97a3c403 100644
--- a/clang/lib/AST/ByteCode/MemberPointer.cpp
+++ b/clang/lib/AST/ByteCode/MemberPointer.cpp
@@ -18,8 +18,7 @@ namespace interp {
 std::optional<Pointer> MemberPointer::toPointer(const Context &Ctx) const {
   if (!Dcl || isa<FunctionDecl>(Dcl))
     return Base;
-  const FieldDecl *FD = cast<FieldDecl>(Dcl);
-  assert(FD);
+  assert((isa<FieldDecl, IndirectFieldDecl>(Dcl)));
 
   if (!Base.isBlockPointer())
     return std::nullopt;
@@ -31,24 +30,36 @@ std::optional<Pointer> MemberPointer::toPointer(const Context &Ctx) const {
   if (!BaseRecord)
     return std::nullopt;
 
-  assert(BaseRecord);
-  if (FD->getParent() == BaseRecord->getDecl())
-    return CastedBase.atField(BaseRecord->getField(FD)->Offset);
-
-  const RecordDecl *FieldParent = FD->getParent();
-  const Record *FieldRecord = Ctx.getRecord(FieldParent);
-
   unsigned Offset = 0;
-  Offset += FieldRecord->getField(FD)->Offset;
   Offset += CastedBase.block()->getDescriptor()->getMetadataSize();
 
-  if (Offset > CastedBase.block()->getSize())
-    return std::nullopt;
+  if (const auto *FD = dyn_cast<FieldDecl>(Dcl)) {
+    if (FD->getParent() == BaseRecord->getDecl())
+      return CastedBase.atField(BaseRecord->getField(FD)->Offset);
 
-  if (const RecordDecl *BaseDecl = Base.getDeclPtr().getRecord()->getDecl();
-      BaseDecl != FieldParent)
-    Offset += Ctx.collectBaseOffset(FieldParent, BaseDecl);
+    const RecordDecl *FieldParent = FD->getParent();
+    const Record *FieldRecord = Ctx.getRecord(FieldParent);
 
+    Offset += FieldRecord->getField(FD)->Offset;
+    if (Offset > CastedBase.block()->getSize())
+      return std::nullopt;
+
+    if (const RecordDecl *BaseDecl = Base.getDeclPtr().getRecord()->getDecl();
+        BaseDecl != FieldParent)
+      Offset += Ctx.collectBaseOffset(FieldParent, BaseDecl);
+
+  } else {
+    const auto *IFD = cast<IndirectFieldDecl>(Dcl);
+
+    for (const NamedDecl *ND : IFD->chain()) {
+      const FieldDecl *F = cast<FieldDecl>(ND);
+      const RecordDecl *FieldParent = F->getParent();
+      const Record *FieldRecord = Ctx.getRecord(FieldParent);
+      Offset += FieldRecord->getField(F)->Offset;
+    }
+  }
+
+  assert(BaseRecord);
   if (Offset > CastedBase.block()->getSize())
     return std::nullopt;
 
diff --git a/clang/test/AST/ByteCode/memberpointers.cpp b/clang/test/AST/ByteCode/memberpointers.cpp
index f38e94863863159..ea4a725d3fd0dfa 100644
--- a/clang/test/AST/ByteCode/memberpointers.cpp
+++ b/clang/test/AST/ByteCode/memberpointers.cpp
@@ -209,3 +209,20 @@ namespace MemPtrTemporary {
 
   static_assert(apply(A(), &A::f) == 5, "");
 }
+
+namespace IndirectFields {
+  struct I { union { struct { int a, b; }; }; };
+
+  template <typename T, int T::*F>
+  constexpr int ReadField(const T &o) {
+    return F ? o.*F : 0;
+  }
+  void ReadFields() {
+    I i;
+    ReadField<I, &I::a>(i);
+    ReadField<I, &I::b>(i);
+  }
+
+  constexpr I i{12};
+  static_assert(ReadField<I, &I::a>(i) == 12, "");
+}

From a567d4598755219e535eb04d21b43c3624526714 Mon Sep 17 00:00:00 2001
From: Hans Wennborg <hans@chromium.org>
Date: Mon, 19 Aug 2024 15:31:27 +0200
Subject: [PATCH 357/441] Require !windows instead of XFAIL'ing
 ubsan/TestCases/Integer/bit-int.c

The test would unexpectedly pass on Windows when there's 128-bit runtime
routines available. See comment on #104494
---
 compiler-rt/test/ubsan/TestCases/Integer/bit-int.c | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/compiler-rt/test/ubsan/TestCases/Integer/bit-int.c b/compiler-rt/test/ubsan/TestCases/Integer/bit-int.c
index 48f718285b06a1e..d7e2c6dc60aa0b4 100644
--- a/compiler-rt/test/ubsan/TestCases/Integer/bit-int.c
+++ b/compiler-rt/test/ubsan/TestCases/Integer/bit-int.c
@@ -1,9 +1,7 @@
 // REQUIRES: x86_64-target-arch
+// REQUIRES: !windows
 // RUN: %clang -Wno-constant-conversion -Wno-array-bounds -Wno-division-by-zero -Wno-shift-negative-value -Wno-shift-count-negative -Wno-int-to-pointer-cast -O0 -fsanitize=array-bounds,float-cast-overflow,implicit-integer-sign-change,implicit-signed-integer-truncation,implicit-unsigned-integer-truncation,integer-divide-by-zero,pointer-overflow,shift-base,shift-exponent,signed-integer-overflow,unsigned-integer-overflow,unsigned-shift-base,vla-bound %s -o %t1 && %run %t1 2>&1 | FileCheck %s
 
-// FIXME: make the test pass on windows.
-// XFAIL: target={{.*windows-msvc.*}}
-
 #include <stdint.h>
 #include <stdio.h>
 

From 3188e9b4e0f106abd683829906a21a98655bb794 Mon Sep 17 00:00:00 2001
From: Julian Brown <julian.brown@amd.com>
Date: Mon, 19 Aug 2024 14:37:32 +0100
Subject: [PATCH 358/441] [clang][OpenMP] Diagnose badly-formed collapsed
 imperfect loop nests (#60678) (#101305)

This patch fixes a couple of cases where Clang aborts with loop nests
that are being collapsed (via the relevant OpenMP clause) into a new,
combined loop.

The problematic cases happen when a variable declared within the loop
nest is used in the (init, condition, iter) statement of a more
deeply-nested loop. I don't think these cases (generally?) fall under
the non-rectangular loop nest rules as defined in OpenMP 5.0+, but I
could be wrong (and anyway, emitting an error is better than crashing).

In terms of implementation: the crash happens because (to a first
approximation) all the loop bounds calculations are pulled out to the
start of the new, combined loop, but variables declared in the loop nest
"haven't been seen yet". I believe there is special handling for
iteration variables declared in "for" init statements, but not for
variables declared elsewhere in the "imperfect" parts of a loop nest.

So, this patch tries to diagnose the troublesome cases before they can
cause a crash. This is slightly awkward because at the point where we
want to do the diagnosis (SemaOpenMP.cpp), we don't have scope
information readily available. Instead we "manually" scan through the
AST of the loop nest looking for var decls (ForVarDeclFinder), then we
ensure we're not using any of those in loop control subexprs
(ForSubExprChecker). All that is only done when we have a "collapse"
clause.

Range-for loops can also cause crashes at present without this patch, so
are handled too.
---
 .../clang/Basic/DiagnosticSemaKinds.td        |   2 +
 clang/lib/Sema/SemaOpenMP.cpp                 | 141 +++++++++++++++++-
 clang/test/OpenMP/loop_collapse_1.c           |  40 +++++
 clang/test/OpenMP/loop_collapse_2.cpp         |  80 ++++++++++
 4 files changed, 255 insertions(+), 8 deletions(-)
 create mode 100644 clang/test/OpenMP/loop_collapse_1.c
 create mode 100644 clang/test/OpenMP/loop_collapse_2.cpp

diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 94be9c874afe971..0814e15c0f9dd55 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -11165,6 +11165,8 @@ def err_omp_loop_diff_cxx : Error<
   "upper and lower loop bounds">;
 def err_omp_loop_cannot_use_stmt : Error<
   "'%0' statement cannot be used in OpenMP for loop">;
+def err_omp_loop_bad_collapse_var : Error<
+  "cannot use variable %1 in collapsed imperfectly-nested loop %select{init|condition|increment}0 statement">;
 def err_omp_simd_region_cannot_use_stmt : Error<
   "'%0' statement cannot be used in OpenMP simd region">;
 def warn_omp_loop_64_bit_var : Warning<
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index 5950bc272fae3c8..c1442b8a58eda4f 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -21,6 +21,7 @@
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/DeclOpenMP.h"
 #include "clang/AST/OpenMPClause.h"
+#include "clang/AST/RecursiveASTVisitor.h"
 #include "clang/AST/StmtCXX.h"
 #include "clang/AST/StmtOpenMP.h"
 #include "clang/AST/StmtVisitor.h"
@@ -7695,6 +7696,52 @@ struct LoopIterationSpace final {
   Expr *FinalCondition = nullptr;
 };
 
+/// Scan an AST subtree, checking that no decls in the CollapsedLoopVarDecls
+/// set are referenced.  Used for verifying loop nest structure before
+/// performing a loop collapse operation.
+class ForSubExprChecker final : public RecursiveASTVisitor<ForSubExprChecker> {
+  const llvm::SmallPtrSetImpl<const Decl *> &CollapsedLoopVarDecls;
+  VarDecl *ForbiddenVar = nullptr;
+  SourceRange ErrLoc;
+
+public:
+  explicit ForSubExprChecker(
+      const llvm::SmallPtrSetImpl<const Decl *> &CollapsedLoopVarDecls)
+      : CollapsedLoopVarDecls(CollapsedLoopVarDecls) {}
+
+  // We want to visit implicit code, i.e. synthetic initialisation statements
+  // created during range-for lowering.
+  bool shouldVisitImplicitCode() const { return true; }
+
+  bool VisitDeclRefExpr(DeclRefExpr *E) {
+    ValueDecl *VD = E->getDecl();
+    if (!isa<VarDecl, BindingDecl>(VD))
+      return true;
+    VarDecl *V = VD->getPotentiallyDecomposedVarDecl();
+    if (V->getType()->isReferenceType()) {
+      VarDecl *VD = V->getDefinition();
+      if (VD->hasInit()) {
+        Expr *I = VD->getInit();
+        DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(I);
+        if (!DRE)
+          return true;
+        V = DRE->getDecl()->getPotentiallyDecomposedVarDecl();
+      }
+    }
+    Decl *Canon = V->getCanonicalDecl();
+    if (CollapsedLoopVarDecls.contains(Canon)) {
+      ForbiddenVar = V;
+      ErrLoc = E->getSourceRange();
+      return false;
+    }
+
+    return true;
+  }
+
+  VarDecl *getForbiddenVar() const { return ForbiddenVar; }
+  SourceRange getErrRange() const { return ErrLoc; }
+};
+
 /// Helper class for checking canonical form of the OpenMP loops and
 /// extracting iteration space of each loop in the loop nest, that will be used
 /// for IR generation.
@@ -7709,6 +7756,8 @@ class OpenMPIterationSpaceChecker {
   SourceLocation DefaultLoc;
   /// A location for diagnostics (when increment is not compatible).
   SourceLocation ConditionLoc;
+  /// The set of variables declared within the (to be collapsed) loop nest.
+  const llvm::SmallPtrSetImpl<const Decl *> &CollapsedLoopVarDecls;
   /// A source location for referring to loop init later.
   SourceRange InitSrcRange;
   /// A source location for referring to condition later.
@@ -7752,10 +7801,13 @@ class OpenMPIterationSpaceChecker {
   Expr *Condition = nullptr;
 
 public:
-  OpenMPIterationSpaceChecker(Sema &SemaRef, bool SupportsNonRectangular,
-                              DSAStackTy &Stack, SourceLocation DefaultLoc)
+  OpenMPIterationSpaceChecker(
+      Sema &SemaRef, bool SupportsNonRectangular, DSAStackTy &Stack,
+      SourceLocation DefaultLoc,
+      const llvm::SmallPtrSetImpl<const Decl *> &CollapsedLoopDecls)
       : SemaRef(SemaRef), SupportsNonRectangular(SupportsNonRectangular),
-        Stack(Stack), DefaultLoc(DefaultLoc), ConditionLoc(DefaultLoc) {}
+        Stack(Stack), DefaultLoc(DefaultLoc), ConditionLoc(DefaultLoc),
+        CollapsedLoopVarDecls(CollapsedLoopDecls) {}
   /// Check init-expr for canonical loop form and save loop counter
   /// variable - #Var and its initialization value - #LB.
   bool checkAndSetInit(Stmt *S, bool EmitDiags = true);
@@ -8076,6 +8128,16 @@ bool OpenMPIterationSpaceChecker::checkAndSetInit(Stmt *S, bool EmitDiags) {
     if (!ExprTemp->cleanupsHaveSideEffects())
       S = ExprTemp->getSubExpr();
 
+  if (!CollapsedLoopVarDecls.empty()) {
+    ForSubExprChecker FSEC{CollapsedLoopVarDecls};
+    if (!FSEC.TraverseStmt(S)) {
+      SourceRange Range = FSEC.getErrRange();
+      SemaRef.Diag(Range.getBegin(), diag::err_omp_loop_bad_collapse_var)
+          << Range.getEnd() << 0 << FSEC.getForbiddenVar();
+      return true;
+    }
+  }
+
   InitSrcRange = S->getSourceRange();
   if (Expr *E = dyn_cast<Expr>(S))
     S = E->IgnoreParens();
@@ -8179,6 +8241,17 @@ bool OpenMPIterationSpaceChecker::checkAndSetCond(Expr *S) {
   }
   Condition = S;
   S = getExprAsWritten(S);
+
+  if (!CollapsedLoopVarDecls.empty()) {
+    ForSubExprChecker FSEC{CollapsedLoopVarDecls};
+    if (!FSEC.TraverseStmt(S)) {
+      SourceRange Range = FSEC.getErrRange();
+      SemaRef.Diag(Range.getBegin(), diag::err_omp_loop_bad_collapse_var)
+          << Range.getEnd() << 1 << FSEC.getForbiddenVar();
+      return true;
+    }
+  }
+
   SourceLocation CondLoc = S->getBeginLoc();
   auto &&CheckAndSetCond =
       [this, IneqCondIsCanonical](BinaryOperatorKind Opcode, const Expr *LHS,
@@ -8277,6 +8350,16 @@ bool OpenMPIterationSpaceChecker::checkAndSetInc(Expr *S) {
     if (!ExprTemp->cleanupsHaveSideEffects())
       S = ExprTemp->getSubExpr();
 
+  if (!CollapsedLoopVarDecls.empty()) {
+    ForSubExprChecker FSEC{CollapsedLoopVarDecls};
+    if (!FSEC.TraverseStmt(S)) {
+      SourceRange Range = FSEC.getErrRange();
+      SemaRef.Diag(Range.getBegin(), diag::err_omp_loop_bad_collapse_var)
+          << Range.getEnd() << 2 << FSEC.getForbiddenVar();
+      return true;
+    }
+  }
+
   IncrementSrcRange = S->getSourceRange();
   S = S->IgnoreParens();
   if (auto *UO = dyn_cast<UnaryOperator>(S)) {
@@ -8998,8 +9081,9 @@ void SemaOpenMP::ActOnOpenMPLoopInitialization(SourceLocation ForLoc,
     return;
 
   DSAStack->loopStart();
+  llvm::SmallPtrSet<const Decl *, 1> EmptyDeclSet;
   OpenMPIterationSpaceChecker ISC(SemaRef, /*SupportsNonRectangular=*/true,
-                                  *DSAStack, ForLoc);
+                                  *DSAStack, ForLoc, EmptyDeclSet);
   if (!ISC.checkAndSetInit(Init, /*EmitDiags=*/false)) {
     if (ValueDecl *D = ISC.getLoopDecl()) {
       auto *VD = dyn_cast<VarDecl>(D);
@@ -9096,7 +9180,8 @@ static bool checkOpenMPIterationSpace(
     Expr *OrderedLoopCountExpr,
     SemaOpenMP::VarsWithInheritedDSAType &VarsWithImplicitDSA,
     llvm::MutableArrayRef<LoopIterationSpace> ResultIterSpaces,
-    llvm::MapVector<const Expr *, DeclRefExpr *> &Captures) {
+    llvm::MapVector<const Expr *, DeclRefExpr *> &Captures,
+    const llvm::SmallPtrSetImpl<const Decl *> &CollapsedLoopVarDecls) {
   bool SupportsNonRectangular = !isOpenMPLoopTransformationDirective(DKind);
   // OpenMP [2.9.1, Canonical Loop Form]
   //   for (init-expr; test-expr; incr-expr) structured-block
@@ -9135,7 +9220,8 @@ static bool checkOpenMPIterationSpace(
     return false;
 
   OpenMPIterationSpaceChecker ISC(SemaRef, SupportsNonRectangular, DSA,
-                                  For ? For->getForLoc() : CXXFor->getForLoc());
+                                  For ? For->getForLoc() : CXXFor->getForLoc(),
+                                  CollapsedLoopVarDecls);
 
   // Check init.
   Stmt *Init = For ? For->getInit() : CXXFor->getBeginStmt();
@@ -9502,6 +9588,39 @@ static Expr *buildPostUpdate(Sema &S, ArrayRef<Expr *> PostUpdates) {
   return PostUpdate;
 }
 
+/// Look for variables declared in the body parts of a for-loop nest.  Used
+/// for verifying loop nest structure before performing a loop collapse
+/// operation.
+class ForVarDeclFinder final : public RecursiveASTVisitor<ForVarDeclFinder> {
+  int NestingDepth = 0;
+  llvm::SmallPtrSetImpl<const Decl *> &VarDecls;
+
+public:
+  explicit ForVarDeclFinder(llvm::SmallPtrSetImpl<const Decl *> &VD)
+      : VarDecls(VD) {}
+
+  bool VisitForStmt(ForStmt *F) {
+    ++NestingDepth;
+    TraverseStmt(F->getBody());
+    --NestingDepth;
+    return false;
+  }
+
+  bool VisitCXXForRangeStmt(CXXForRangeStmt *RF) {
+    ++NestingDepth;
+    TraverseStmt(RF->getBody());
+    --NestingDepth;
+    return false;
+  }
+
+  bool VisitVarDecl(VarDecl *D) {
+    Decl *C = D->getCanonicalDecl();
+    if (NestingDepth > 0)
+      VarDecls.insert(C);
+    return true;
+  }
+};
+
 /// Called on a for stmt to check itself and nested loops (if any).
 /// \return Returns 0 if one of the collapsed stmts is not canonical for loop,
 /// number of collapsed loops otherwise.
@@ -9514,6 +9633,7 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
   unsigned NestedLoopCount = 1;
   bool SupportsNonPerfectlyNested = (SemaRef.LangOpts.OpenMP >= 50) &&
                                     !isOpenMPLoopTransformationDirective(DKind);
+  llvm::SmallPtrSet<const Decl *, 4> CollapsedLoopVarDecls;
 
   if (CollapseLoopCountExpr) {
     // Found 'collapse' clause - calculate collapse number.
@@ -9521,6 +9641,9 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
     if (!CollapseLoopCountExpr->isValueDependent() &&
         CollapseLoopCountExpr->EvaluateAsInt(Result, SemaRef.getASTContext())) {
       NestedLoopCount = Result.Val.getInt().getLimitedValue();
+
+      ForVarDeclFinder FVDF{CollapsedLoopVarDecls};
+      FVDF.TraverseStmt(AStmt);
     } else {
       Built.clear(/*Size=*/1);
       return 1;
@@ -9558,11 +9681,13 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
           SupportsNonPerfectlyNested, NumLoops,
           [DKind, &SemaRef, &DSA, NumLoops, NestedLoopCount,
            CollapseLoopCountExpr, OrderedLoopCountExpr, &VarsWithImplicitDSA,
-           &IterSpaces, &Captures](unsigned Cnt, Stmt *CurStmt) {
+           &IterSpaces, &Captures,
+           &CollapsedLoopVarDecls](unsigned Cnt, Stmt *CurStmt) {
             if (checkOpenMPIterationSpace(
                     DKind, CurStmt, SemaRef, DSA, Cnt, NestedLoopCount,
                     NumLoops, CollapseLoopCountExpr, OrderedLoopCountExpr,
-                    VarsWithImplicitDSA, IterSpaces, Captures))
+                    VarsWithImplicitDSA, IterSpaces, Captures,
+                    CollapsedLoopVarDecls))
               return true;
             if (Cnt > 0 && Cnt >= NestedLoopCount &&
                 IterSpaces[Cnt].CounterVar) {
diff --git a/clang/test/OpenMP/loop_collapse_1.c b/clang/test/OpenMP/loop_collapse_1.c
new file mode 100644
index 000000000000000..c9877419223dde4
--- /dev/null
+++ b/clang/test/OpenMP/loop_collapse_1.c
@@ -0,0 +1,40 @@
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -verify %s
+
+void func( double *A, int N, int M, int NB ) {
+#pragma omp parallel
+  {
+    int nblks = (N-1)/NB;
+    int lnb = ((N-1)/NB)*NB;
+
+#pragma omp for collapse(2)
+    for (int jblk = 0 ; jblk < nblks ; jblk++ ) {
+      int jb = (jblk == nblks - 1 ? lnb : NB);
+      for (int jk = 0; jk < N; jk+=jb) {  // expected-error{{cannot use variable 'jb' in collapsed imperfectly-nested loop increment statement}}
+      }
+    }
+
+#pragma omp for collapse(2)
+    for (int a = 0; a < N; a++) {
+      for (int b = 0; b < M; b++) {
+        int cx = a+b < NB ? a : b;
+        for (int c = 0; c < cx; c++) {
+        }
+      }
+    }
+
+#pragma omp for collapse(3)
+    for (int a = 0; a < N; a++) {
+      for (int b = 0; b < M; b++) {
+        int cx = a+b < NB ? a : b;
+        for (int c = 0; c < cx; c++) {  // expected-error{{cannot use variable 'cx' in collapsed imperfectly-nested loop condition statement}}
+        }
+      }
+    }
+  }
+}
+
+int main(void) {
+  double arr[256];
+  func (arr, 16, 16, 16);
+  return 0;
+}
diff --git a/clang/test/OpenMP/loop_collapse_2.cpp b/clang/test/OpenMP/loop_collapse_2.cpp
new file mode 100644
index 000000000000000..59deddf65e37b54
--- /dev/null
+++ b/clang/test/OpenMP/loop_collapse_2.cpp
@@ -0,0 +1,80 @@
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -verify %s
+
+// We just want to try out a range for statement... this seems a bit OTT.
+template<typename T>
+class fakevector {
+  T *contents;
+  long size;
+public:
+  fakevector(long sz) : size(sz) {
+    contents = new T[sz];
+  }
+  ~fakevector() {
+    delete[] contents;
+  }
+  T& operator[](long x) { return contents[x]; }
+  typedef T *iterator;
+  fakevector<T>::iterator begin() {
+    return &contents[0];
+  }
+  fakevector<T>::iterator end() {
+    return &contents[size];
+  }
+};
+
+void func( double *A, int N, int M, int NB ) {
+#pragma omp parallel
+  {
+    int nblks = (N-1)/NB;
+    int lnb = ((N-1)/NB)*NB;
+#pragma omp for collapse(2)
+    for (int jblk = 0 ; jblk < nblks ; jblk++ ) {
+      int jb = (jblk == nblks - 1 ? lnb : NB);
+      for (int jk = 0; jk < N; jk+=jb) {  // expected-error{{cannot use variable 'jb' in collapsed imperfectly-nested loop increment statement}}
+      }
+    }
+
+#pragma omp for collapse(2)
+    for (int a = 0; a < N; a++) {
+        for (int b = 0; b < M; b++) {
+          int cx = a+b < NB ? a : b;
+          for (int c = 0; c < cx; c++) {
+          }
+        }
+    }
+
+    fakevector<float> myvec{N};
+#pragma omp for collapse(2)
+    for (auto &a : myvec) {
+      fakevector<float> myvec3{M};
+      for (auto &b : myvec3) {  // expected-error{{cannot use variable 'myvec3' in collapsed imperfectly-nested loop init statement}}
+      }
+    }
+
+    fakevector<float> myvec2{M};
+
+#pragma omp for collapse(3)
+    for (auto &a : myvec) {
+      for (auto &b : myvec2) {
+        int cx = a < b ? N : M;
+        for (int c = 0; c < cx; c++) {  // expected-error {{cannot use variable 'cx' in collapsed imperfectly-nested loop condition statement}}
+        }
+      }
+    }
+
+#pragma omp for collapse(3)
+    for (auto &a : myvec) {
+      int cx = a < 5 ? M : N;
+      for (auto &b : myvec2) {
+        for (int c = 0; c < cx; c++) {  // expected-error{{cannot use variable 'cx' in collapsed imperfectly-nested loop condition statement}}
+        }
+      }
+    }
+  }
+}
+
+int main(void) {
+  double arr[256];
+  func (arr, 16, 16, 16);
+  return 0;
+}

From 7efa068f7a7ed4f42ba09cce73e8c09bb8b4e8ce Mon Sep 17 00:00:00 2001
From: Michael Maitland <michaeltmaitland@gmail.com>
Date: Mon, 19 Aug 2024 09:41:42 -0400
Subject: [PATCH 359/441] [RISCV] Add vector and vector crypto to SiFiveP400
 scheduler model (#102155)

The SiFiveP400 scheduler model did not support vector or vector crypto.
With the addition of the sifive-p470 processor, this model needs to support
these extensions.

The processors who use this model but do not have vector or vector
crypto will never produce these instructions, so there is no impact to these
processors.

Co-authored-by: Min Hsu <min.hsu@sifive.com>
---
 llvm/lib/Target/RISCV/RISCVSchedSiFiveP400.td |  869 +-
 .../tools/llvm-mca/RISCV/SiFiveP400/load.s    |   21 +-
 .../llvm-mca/RISCV/SiFiveP400/vislide-vx.s    |  108 +
 .../llvm-mca/RISCV/SiFiveP400/vlseg-vsseg.s   | 9827 +++++++++++++++++
 .../tools/llvm-mca/RISCV/SiFiveP400/vmv.s     |  895 ++
 .../tools/llvm-mca/RISCV/SiFiveP400/vreduce.s |  438 +
 .../llvm-mca/RISCV/SiFiveP400/vrgather.s      |   86 +
 .../llvm-mca/RISCV/SiFiveP400/vshift-vmul.s   |  132 +
 .../tools/llvm-mca/RISCV/SiFiveP400/zvbb.s    |  460 +
 .../tools/llvm-mca/RISCV/SiFiveP400/zvbc.s    |  112 +
 .../tools/llvm-mca/RISCV/SiFiveP400/zvkg.s    |  127 +
 .../tools/llvm-mca/RISCV/SiFiveP400/zvkned.s  |  203 +
 .../tools/llvm-mca/RISCV/SiFiveP400/zvknhb.s  |  152 +
 .../tools/llvm-mca/RISCV/SiFiveP400/zvksed.s  |  113 +
 .../tools/llvm-mca/RISCV/SiFiveP400/zvksh.s   |   98 +
 15 files changed, 13631 insertions(+), 10 deletions(-)
 create mode 100644 llvm/test/tools/llvm-mca/RISCV/SiFiveP400/vislide-vx.s
 create mode 100644 llvm/test/tools/llvm-mca/RISCV/SiFiveP400/vlseg-vsseg.s
 create mode 100644 llvm/test/tools/llvm-mca/RISCV/SiFiveP400/vmv.s
 create mode 100644 llvm/test/tools/llvm-mca/RISCV/SiFiveP400/vreduce.s
 create mode 100644 llvm/test/tools/llvm-mca/RISCV/SiFiveP400/vrgather.s
 create mode 100644 llvm/test/tools/llvm-mca/RISCV/SiFiveP400/vshift-vmul.s
 create mode 100644 llvm/test/tools/llvm-mca/RISCV/SiFiveP400/zvbb.s
 create mode 100644 llvm/test/tools/llvm-mca/RISCV/SiFiveP400/zvbc.s
 create mode 100644 llvm/test/tools/llvm-mca/RISCV/SiFiveP400/zvkg.s
 create mode 100644 llvm/test/tools/llvm-mca/RISCV/SiFiveP400/zvkned.s
 create mode 100644 llvm/test/tools/llvm-mca/RISCV/SiFiveP400/zvknhb.s
 create mode 100644 llvm/test/tools/llvm-mca/RISCV/SiFiveP400/zvksed.s
 create mode 100644 llvm/test/tools/llvm-mca/RISCV/SiFiveP400/zvksh.s

diff --git a/llvm/lib/Target/RISCV/RISCVSchedSiFiveP400.td b/llvm/lib/Target/RISCV/RISCVSchedSiFiveP400.td
index 9fa455be3852577..6926184e92399c2 100644
--- a/llvm/lib/Target/RISCV/RISCVSchedSiFiveP400.td
+++ b/llvm/lib/Target/RISCV/RISCVSchedSiFiveP400.td
@@ -8,6 +8,115 @@
 
 //===----------------------------------------------------------------------===//
 
+/// c is true if mx has the worst case behavior compared to LMULs in MxList.
+/// On the SiFiveP400, the worst case LMUL is the Largest LMUL
+/// and the worst case sew is the smallest SEW for that LMUL.
+class SiFiveP400IsWorstCaseMX<string mx, list<string> MxList> {
+  string LLMUL = LargestLMUL<MxList>.r;
+  bit c = !eq(mx, LLMUL);
+}
+
+class SiFiveP400IsWorstCaseMXSEW<string mx, int sew, list<string> MxList, bit isF = 0> {
+  string LLMUL = LargestLMUL<MxList>.r;
+  int SSEW = SmallestSEW<mx, isF>.r;
+  bit c = !and(!eq(mx, LLMUL), !eq(sew, SSEW));
+}
+
+// 1 Micro-Op per cycle.
+class SiFiveP400GetLMulCycles<string mx> {
+  int c = !cond(
+    !eq(mx, "M1") : 1,
+    !eq(mx, "M2") : 2,
+    !eq(mx, "M4") : 4,
+    !eq(mx, "M8") : 8,
+    !eq(mx, "MF2") : 1,
+    !eq(mx, "MF4") : 1,
+    !eq(mx, "MF8") : 1
+  );
+}
+
+// Latency for segmented loads and stores are calculated as vl * nf.
+class SiFiveP400GetCyclesSegmented<string mx, int sew, int nf> {
+  defvar VLEN = 128;
+  defvar VLUpperBound = !cond(
+    !eq(mx, "M1") : !div(VLEN, sew),
+    !eq(mx, "M2") : !div(!mul(VLEN, 2), sew),
+    !eq(mx, "M4") : !div(!mul(VLEN, 4), sew),
+    !eq(mx, "M8") : !div(!mul(VLEN, 8), sew),
+    !eq(mx, "MF2") : !div(!div(VLEN, 2), sew),
+    !eq(mx, "MF4") : !div(!div(VLEN, 4), sew),
+    !eq(mx, "MF8") : !div(!div(VLEN, 8), sew),
+  );
+  int c = !mul(VLUpperBound, nf);
+}
+
+// Both variants of floating point vector reductions are based on numbers collected
+// from llvm-exegesis.
+class VFReduceBaseCycles<int sew> {
+  // The latency for simple unordered VFReduce is `C + 6 * log2(LMUL)`,
+  // and `C * LMUL` for ordered VFReduce. This helper class provides the `C`.
+  int val = !cond(!eq(sew, 16): 16,
+                  !eq(sew, 32): 10,
+                  !eq(sew, 64): 6);
+}
+
+class AdvancedVFReduceCycles<int sew, string mx> {
+  // SEW = 64 has lower latencies and RThroughputs than other SEWs.
+  int latency = !cond(!eq(mx, "M1"): !if(!eq(sew, 64), 4, 6),
+                      !eq(mx, "M2"): !if(!eq(sew, 64), 6, 8),
+                      !eq(mx, "M4"): !if(!eq(sew, 64), 8, 10),
+                      !eq(mx, "M8"): !if(!eq(sew, 64), 11, 13),
+                      true: !if(!eq(sew, 64), 4, 6));
+  int rthroughput = !cond(!eq(mx, "M1"): !if(!eq(sew, 64), 2, 3),
+                          !eq(mx, "M2"): !if(!eq(sew, 64), 3, 4),
+                          !eq(mx, "M4"): !if(!eq(sew, 64), 5, 6),
+                          !eq(mx, "M8"): !if(!eq(sew, 64), 10, 12),
+                          true: !if(!eq(sew, 64), 2, 3));
+}
+
+// Both variants of integer vector reductions are based on numbers collected
+// from llvm-exegesis.
+// TODO: Fractional LMUL's latency and rthroughput.
+class SimpleVIReduceCycles<string mx> {
+  defvar LMul = SiFiveP400GetLMulCycles<mx>.c;
+  int latency = !mul(LMul, 2);
+  int rthroughput = !cond(
+                      !eq(mx, "M1"): 1,
+                      !eq(mx, "M2"): 2,
+                      !eq(mx, "M4"): 4,
+                      !eq(mx, "M8"): 9,
+                      true: 1);
+}
+
+class AdvancedVIReduceCycles<int sew, string mx> {
+  // `C - 2 * log2(SEW)`, where `C` = 16.1, 18.1, 20.1, and 23.8 for
+  // M1/2/4/8, respectively.
+  int latency = !cond(!eq(mx, "M1"): !sub(16, !mul(2, !logtwo(sew))),
+                      !eq(mx, "M2"): !sub(18, !mul(2, !logtwo(sew))),
+                      !eq(mx, "M4"): !sub(20, !mul(2, !logtwo(sew))),
+                      !eq(mx, "M8"): !sub(23, !mul(2, !logtwo(sew))),
+                      true: 4);
+  int rthroughput = !cond(
+                      // `8.3 - 1.02 * log2(SEW)`
+                      !eq(mx, "M1"): !sub(8, !logtwo(sew)),
+                      // `10.0 - 1.16 * log2(SEW)`. Note that `9 - log2(SEW)`
+                      // is closer to the floor value of the original formula.
+                      !eq(mx, "M2"): !sub(9, !logtwo(sew)),
+                      // `14.2 - 1.53 * log2(SEW)`
+                      !eq(mx, "M4"): !div(!sub(1420, !mul(153, !logtwo(sew))), 100),
+                      // `24.1 - 2.3 * log2(SEW)`
+                      !eq(mx, "M8"): !div(!sub(241, !mul(23, !logtwo(sew))), 10),
+                      true: 1);
+}
+
+class SiFiveP400VSM3CCycles<string mx> {
+  // c = ceil(LMUL / 2)
+  int c = !cond(!eq(mx, "M2") : 1,
+                !eq(mx, "M4") : 2,
+                !eq(mx, "M8") : 4,
+                true : 1);
+}
+
 def SiFiveP400Model : SchedMachineModel {
   let IssueWidth = 3;         // 3 micro-ops are dispatched per cycle.
   let MicroOpBufferSize = 56; // Max micro-ops that can be buffered.
@@ -45,6 +154,13 @@ defvar SiFiveP400FloatArith  = SiFiveP400FEXQ0;
 defvar SiFiveP400F2I      = SiFiveP400FEXQ0;
 def SiFiveP400FloatDiv    : ProcResource<1>;
 
+// Vector pipeline
+def SiFiveP400VEXQ0        : ProcResource<1>;
+def SiFiveP400VLD          : ProcResource<1>;
+def SiFiveP400VST          : ProcResource<1>;
+def SiFiveP400VDiv         : ProcResource<1>;
+def SiFiveP400VFloatDiv    : ProcResource<1>;
+
 let Latency = 1 in {
 // Integer arithmetic and logic
 def : WriteRes<WriteIALU, [SiFiveP400IntArith]>;
@@ -246,9 +362,549 @@ def : WriteRes<WriteFMovI64ToF64, [SiFiveP400I2F]>;
 def : WriteRes<WriteFMovF64ToI64, [SiFiveP400F2I]>;
 }
 
+// 6. Configuration-Setting Instructions
+def : WriteRes<WriteVSETVLI, [SiFiveP400SYS]>;
+def : WriteRes<WriteVSETIVLI, [SiFiveP400SYS]>;
+def : WriteRes<WriteVSETVL, [SiFiveP400SYS]>;
+
+// 7. Vector Loads and Stores
+// FIXME: This unit is still being improved, currently
+// it is based on stage numbers. Estimates are optimistic,
+// latency may be longer.
+foreach mx = SchedMxList in {
+  defvar LMulLat = SiFiveP400GetLMulCycles<mx>.c;
+  defvar IsWorstCase = SiFiveP400IsWorstCaseMX<mx, SchedMxList>.c;
+  let Latency = 8, ReleaseAtCycles = [LMulLat] in {
+    defm "" : LMULWriteResMX<"WriteVLDE",    [SiFiveP400VLD], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVLDM",    [SiFiveP400VLD], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVLDFF",   [SiFiveP400VLD], mx, IsWorstCase>;
+  }
+  let Latency = 12, ReleaseAtCycles = [LMulLat] in {
+    defm "" : LMULWriteResMX<"WriteVLDS8",   [SiFiveP400VLD], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVLDS16",  [SiFiveP400VLD], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVLDS32",  [SiFiveP400VLD], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVLDS64",  [SiFiveP400VLD], mx, IsWorstCase>;
+  }
+  let Latency = 12, ReleaseAtCycles = [LMulLat] in {
+    defm "" : LMULWriteResMX<"WriteVLDUX8",  [SiFiveP400VLD], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVLDUX16", [SiFiveP400VLD], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVLDUX32", [SiFiveP400VLD], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVLDUX64", [SiFiveP400VLD], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVLDOX8",  [SiFiveP400VLD], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVLDOX16", [SiFiveP400VLD], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVLDOX32", [SiFiveP400VLD], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVLDOX64", [SiFiveP400VLD], mx, IsWorstCase>;
+  }
+}
+
+foreach mx = SchedMxList in {
+  defvar LMulLat = SiFiveP400GetLMulCycles<mx>.c;
+  defvar IsWorstCase = SiFiveP400IsWorstCaseMX<mx, SchedMxList>.c;
+  let Latency = 8, ReleaseAtCycles = [LMulLat] in {
+    defm "" : LMULWriteResMX<"WriteVSTE",    [SiFiveP400VST], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVSTM",    [SiFiveP400VST], mx, IsWorstCase>;
+  }
+  let Latency = 12, ReleaseAtCycles = [LMulLat] in {
+    defm "" : LMULWriteResMX<"WriteVSTS8",   [SiFiveP400VST], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVSTS16",  [SiFiveP400VST], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVSTS32",  [SiFiveP400VST], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVSTS64",  [SiFiveP400VST], mx, IsWorstCase>;
+  }
+  let Latency = 12, ReleaseAtCycles = [LMulLat] in {
+    defm "" : LMULWriteResMX<"WriteVSTUX8",  [SiFiveP400VST], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVSTUX16", [SiFiveP400VST], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVSTUX32", [SiFiveP400VST], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVSTUX64", [SiFiveP400VST], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVSTOX8",  [SiFiveP400VST], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVSTOX16", [SiFiveP400VST], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVSTOX32", [SiFiveP400VST], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVSTOX64", [SiFiveP400VST], mx, IsWorstCase>;
+  }
+}
+
+foreach mx = SchedMxList in {
+  foreach nf=2-8 in {
+    foreach eew = [8, 16, 32, 64] in {
+      defvar IsWorstCase = SiFiveP400IsWorstCaseMX<mx, SchedMxList>.c;
+      defvar LMulLat = SiFiveP400GetCyclesSegmented<mx, eew, nf>.c;
+      let Latency = !add(12, LMulLat), ReleaseAtCycles = [!add(12, LMulLat)] in {
+        defm "" : LMULWriteResMX<"WriteVLSEG" # nf # "e" #eew,   [SiFiveP400VLD], mx, IsWorstCase>;
+        defm "" : LMULWriteResMX<"WriteVLSEGFF" # nf # "e" #eew, [SiFiveP400VLD], mx, IsWorstCase>;
+        defm "" : LMULWriteResMX<"WriteVLSSEG" # nf # "e" #eew,  [SiFiveP400VLD], mx, IsWorstCase>;
+        defm "" : LMULWriteResMX<"WriteVLUXSEG" # nf # "e" #eew, [SiFiveP400VLD], mx, IsWorstCase>;
+        defm "" : LMULWriteResMX<"WriteVLOXSEG" # nf # "e" #eew, [SiFiveP400VLD], mx, IsWorstCase>;
+      }
+      let Latency = !add(1, LMulLat), ReleaseAtCycles = [!add(12, LMulLat)] in {
+        defm "" : LMULWriteResMX<"WriteVSSEG" # nf # "e" #eew,   [SiFiveP400VST], mx, IsWorstCase>;
+        defm "" : LMULWriteResMX<"WriteVSSSEG" # nf # "e" #eew,  [SiFiveP400VST], mx, IsWorstCase>;
+        defm "" : LMULWriteResMX<"WriteVSUXSEG" # nf # "e" #eew, [SiFiveP400VST], mx, IsWorstCase>;
+        defm "" : LMULWriteResMX<"WriteVSOXSEG" # nf # "e" #eew, [SiFiveP400VST], mx, IsWorstCase>;
+      }
+    }
+  }
+}
+
+// Whole register move/load/store
+foreach LMul = [1, 2, 4, 8] in {
+  let Latency = 8, ReleaseAtCycles = [LMul] in {
+    def : WriteRes<!cast<SchedWrite>("WriteVLD" # LMul # "R"), [SiFiveP400VLD]>;
+    def : WriteRes<!cast<SchedWrite>("WriteVST" # LMul # "R"), [SiFiveP400VST]>;
+  }
+  let Latency = 2, ReleaseAtCycles = [LMul] in {
+    def : WriteRes<!cast<SchedWrite>("WriteVMov" # LMul # "V"), [SiFiveP400VEXQ0]>;
+  }
+}
+
+// 11. Vector Integer Arithmetic Instructions
+foreach mx = SchedMxList in {
+  defvar LMulLat = SiFiveP400GetLMulCycles<mx>.c;
+  defvar IsWorstCase = SiFiveP400IsWorstCaseMX<mx, SchedMxList>.c;
+  let Latency = 2, ReleaseAtCycles = [LMulLat] in {
+    defm "" : LMULWriteResMX<"WriteVIALUV",    [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVIALUX",    [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVIALUI",    [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVExtV",     [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVICALUV",   [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVICALUX",   [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVICALUI",   [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVICmpV",    [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVICmpX",    [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVICmpI",    [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVIMinMaxV", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVIMinMaxX", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVIMergeV",  [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVIMergeX",  [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVIMergeI",  [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVIMovV",    [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVIMovX",    [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVIMovI",    [SiFiveP400VEXQ0], mx, IsWorstCase>;
+  }
+
+  let Latency = !if(!lt(LMulLat, 2), 2, LMulLat), ReleaseAtCycles = [LMulLat] in {
+    defm "" : LMULWriteResMX<"WriteVShiftV",   [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVShiftX",   [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVShiftI",   [SiFiveP400VEXQ0], mx, IsWorstCase>;
+  }
+
+  let Latency = !if(!eq(mx, "M8"), 9, 6), ReleaseAtCycles = [LMulLat] in {
+    defm "" : LMULWriteResMX<"WriteVIMulV",    [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVIMulX",    [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVIMulAddV", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVIMulAddX", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+  }
+}
+// Widening
+foreach mx = SchedMxListW in {
+  defvar LMulLat = SiFiveP400GetLMulCycles<mx>.c;
+  defvar IsWorstCase = SiFiveP400IsWorstCaseMX<mx, SchedMxListW>.c;
+  let Latency = 6, ReleaseAtCycles = [LMulLat] in {
+    defm "" : LMULWriteResMX<"WriteVIWALUV",    [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVIWALUX",    [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVIWALUI",    [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVIWMulV",    [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVIWMulX",    [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVIWMulAddV", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVIWMulAddX", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+  }
+}
+
+// Worst case needs 51/45/42/72 * lmul cycles for i8/16/32/64.
+foreach mx = SchedMxList in {
+  foreach sew = SchedSEWSet<mx>.val in {
+    defvar LMulLat = SiFiveP400GetLMulCycles<mx>.c;
+    defvar IsWorstCase = SiFiveP400IsWorstCaseMXSEW<mx, sew, SchedMxList>.c;
+    defvar DivMicroOpLat =
+      !cond(!eq(sew, 8): 51, !eq(sew, 16): 45, !eq(sew, 32): 42,
+      /* SEW=64 */ true: 72);
+    defvar DivLatency = !mul(DivMicroOpLat, LMulLat);
+    let Latency = DivLatency, ReleaseAtCycles = [LMulLat, DivLatency] in {
+      defm "" : LMULSEWWriteResMXSEW<"WriteVIDivV", [SiFiveP400VEXQ0, SiFiveP400VDiv], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVIDivX", [SiFiveP400VEXQ0, SiFiveP400VDiv], mx, sew, IsWorstCase>;
+    }
+  }
+}
+
+// Narrowing Shift and Clips
+foreach mx = SchedMxListW in {
+  defvar LMulLat = SiFiveP400GetLMulCycles<mx>.c;
+  defvar IsWorstCase = SiFiveP400IsWorstCaseMX<mx, SchedMxListW>.c;
+  let Latency = 2, ReleaseAtCycles = [LMulLat] in {
+    defm "" : LMULWriteResMX<"WriteVNShiftV", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVNShiftX", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVNShiftI", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVNClipV",  [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVNClipX",  [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVNClipI",  [SiFiveP400VEXQ0], mx, IsWorstCase>;
+  }
+}
+
+// 12. Vector Fixed-Point Arithmetic Instructions
+foreach mx = SchedMxList in {
+  defvar LMulLat = SiFiveP400GetLMulCycles<mx>.c;
+  defvar IsWorstCase = SiFiveP400IsWorstCaseMX<mx, SchedMxList>.c;
+  let Latency = 6, ReleaseAtCycles = [LMulLat] in {
+    defm "" : LMULWriteResMX<"WriteVSALUV",   [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVSALUX",   [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVSALUI",   [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVAALUV",   [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVAALUX",   [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVSMulV",   [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVSMulX",   [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVSShiftV", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVSShiftX", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVSShiftI", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+  }
+}
+
+// 13. Vector Floating-Point Instructions
+foreach mx = SchedMxListF in {
+  foreach sew = SchedSEWSet<mx, isF=1>.val in {
+    defvar LMulLat = SiFiveP400GetLMulCycles<mx>.c;
+    defvar IsWorstCase = SiFiveP400IsWorstCaseMXSEW<mx, sew, SchedMxListF, isF=1>.c;
+    let Latency = 6, ReleaseAtCycles = [LMulLat] in {
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFALUV",  [SiFiveP400VEXQ0], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFALUF",  [SiFiveP400VEXQ0], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFMulV",  [SiFiveP400VEXQ0], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFMulF",  [SiFiveP400VEXQ0], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFMulAddV", [SiFiveP400VEXQ0], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFMulAddF", [SiFiveP400VEXQ0], mx, sew, IsWorstCase>;
+    }
+  }
+}
+foreach mx = SchedMxListF in {
+  foreach sew = SchedSEWSet<mx, isF=1>.val in {
+    defvar LMulLat = SiFiveP400GetLMulCycles<mx>.c;
+    defvar IsWorstCase = SiFiveP400IsWorstCaseMXSEW<mx, sew, SchedMxListF, isF=1>.c;
+    let Latency = 2, ReleaseAtCycles = [LMulLat] in {
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFRecpV", [SiFiveP400VEXQ0], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFSgnjV", [SiFiveP400VEXQ0], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFSgnjF", [SiFiveP400VEXQ0], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFMinMaxV", [SiFiveP400VEXQ0], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFMinMaxF", [SiFiveP400VEXQ0], mx, sew, IsWorstCase>;
+    }
+    let Latency = 3, ReleaseAtCycles = [LMulLat] in
+    defm "" : LMULSEWWriteResMXSEW<"WriteVFCvtIToFV", [SiFiveP400VEXQ0], mx, sew, IsWorstCase>;
+  }
+}
+foreach mx = SchedMxList in {
+  defvar LMulLat = SiFiveP400GetLMulCycles<mx>.c;
+  defvar IsWorstCase = SiFiveP400IsWorstCaseMX<mx, SchedMxList>.c;
+  let Latency = 2, ReleaseAtCycles = [LMulLat] in {
+    defm "" : LMULWriteResMX<"WriteVFCmpV",  [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVFCmpF",  [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVFClassV",  [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVFMergeV",  [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVFMovV",    [SiFiveP400VEXQ0], mx, IsWorstCase>;
+  }
+  let Latency = 3, ReleaseAtCycles = [LMulLat] in
+  defm "" : LMULWriteResMX<"WriteVFCvtFToIV", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+}
+
+// Widening
+foreach mx = SchedMxListW in {
+  foreach sew = SchedSEWSet<mx, isF=0, isWidening=1>.val in {
+    defvar LMulLat = SiFiveP400GetLMulCycles<mx>.c;
+    defvar IsWorstCase = SiFiveP400IsWorstCaseMXSEW<mx, sew, SchedMxListW>.c;
+    let Latency = 3, ReleaseAtCycles = [LMulLat] in
+    defm "" : LMULSEWWriteResMXSEW<"WriteVFWCvtIToFV", [SiFiveP400VEXQ0], mx, sew, IsWorstCase>;
+  }
+}
+foreach mx = SchedMxListFW in {
+  defvar LMulLat = SiFiveP400GetLMulCycles<mx>.c;
+  defvar IsWorstCase = SiFiveP400IsWorstCaseMX<mx, SchedMxListFW>.c;
+  let Latency = 6, ReleaseAtCycles = [LMulLat] in
+  defm "" : LMULWriteResMX<"WriteVFWCvtFToIV", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+}
+foreach mx = SchedMxListFW in {
+  foreach sew = SchedSEWSet<mx, isF=1, isWidening=1>.val in {
+    defvar LMulLat = SiFiveP400GetLMulCycles<mx>.c;
+    defvar IsWorstCase = SiFiveP400IsWorstCaseMXSEW<mx, sew, SchedMxListFW, isF=1>.c;
+    let Latency = 6, ReleaseAtCycles = [LMulLat] in {
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFWALUV", [SiFiveP400VEXQ0], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFWALUF", [SiFiveP400VEXQ0], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFWMulV", [SiFiveP400VEXQ0], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFWMulF", [SiFiveP400VEXQ0], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFWMulAddV", [SiFiveP400VEXQ0], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFWMulAddF", [SiFiveP400VEXQ0], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFWCvtFToFV", [SiFiveP400VEXQ0], mx, sew, IsWorstCase>;
+    }
+  }
+}
+// Narrowing
+foreach mx = SchedMxListW in {
+  defvar LMulLat = SiFiveP400GetLMulCycles<mx>.c;
+  defvar IsWorstCase = SiFiveP400IsWorstCaseMX<mx, SchedMxListW>.c;
+  let Latency = 3, ReleaseAtCycles = [LMulLat] in
+  defm "" : LMULWriteResMX<"WriteVFNCvtFToIV", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+}
+foreach mx = SchedMxListFW in {
+  foreach sew = SchedSEWSet<mx, isF=1, isWidening=1>.val in {
+  defvar LMulLat = SiFiveP400GetLMulCycles<mx>.c;
+  defvar IsWorstCase = SiFiveP400IsWorstCaseMXSEW<mx, sew, SchedMxListFW, isF=1>.c;
+    let Latency = 3, ReleaseAtCycles = [LMulLat] in {
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFNCvtIToFV", [SiFiveP400VEXQ0], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFNCvtFToFV", [SiFiveP400VEXQ0], mx, sew, IsWorstCase>;
+    }
+  }
+}
+
+// Worst case needs around 29/25/37 * LMUL cycles for f16/32/64.
+foreach mx = SchedMxListF in {
+  foreach sew = SchedSEWSet<mx, 1>.val in {
+    defvar LMulLat = SiFiveP400GetLMulCycles<mx>.c;
+    defvar IsWorstCase = SiFiveP400IsWorstCaseMXSEW<mx, sew, SchedMxListF, 1>.c;
+    defvar DivMicroOpLat =
+      !cond(!eq(sew, 16): 29, !eq(sew, 32): 25, /* SEW=64 */ true: 37);
+    defvar DivLatency = !mul(DivMicroOpLat, LMulLat);
+    let Latency = DivLatency, ReleaseAtCycles = [LMulLat, DivLatency] in {
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFDivV",  [SiFiveP400VEXQ0, SiFiveP400VFloatDiv], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFDivF",  [SiFiveP400VEXQ0, SiFiveP400VFloatDiv], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFSqrtV", [SiFiveP400VEXQ0, SiFiveP400VFloatDiv], mx, sew, IsWorstCase>;
+    }
+  }
+}
+
+// 14. Vector Reduction Operations
+foreach mx = SchedMxList in {
+  foreach sew = SchedSEWSet<mx>.val in {
+    defvar IsWorstCase = SiFiveP400IsWorstCaseMXSEW<mx, sew, SchedMxList>.c;
+
+    // Simple reduction
+    defvar SimpleC = SimpleVIReduceCycles<mx>;
+    let Latency = SimpleC.latency, ReleaseAtCycles = [SimpleC.rthroughput] in
+    defm "" : LMULSEWWriteResMXSEW<"WriteVIRedV_From",  [SiFiveP400VEXQ0], mx, sew, IsWorstCase>;
+
+    // Advanced reduction
+    defvar AdvancedC = AdvancedVIReduceCycles<sew, mx>;
+    let Latency = AdvancedC.latency, ReleaseAtCycles = [AdvancedC.rthroughput] in
+    defm "" : LMULSEWWriteResMXSEW<"WriteVIRedMinMaxV_From", [SiFiveP400VEXQ0],
+                                   mx, sew, IsWorstCase>;
+  }
+}
+
+foreach mx = SchedMxListWRed in {
+  foreach sew = SchedSEWSet<mx, 0, 1>.val in {
+    defvar IsWorstCase = SiFiveP400IsWorstCaseMXSEW<mx, sew, SchedMxListWRed>.c;
+    defvar SimpleC = SimpleVIReduceCycles<mx>;
+    let Latency = SimpleC.latency, ReleaseAtCycles = [SimpleC.rthroughput] in {
+      defm "" : LMULSEWWriteResMXSEW<"WriteVIWRedV_From", [SiFiveP400VEXQ0],
+                                      mx, sew, IsWorstCase>;
+    }
+  }
+}
+
+foreach mx = SchedMxListF in {
+  foreach sew = SchedSEWSet<mx, 1>.val in {
+    defvar LMulLat = SiFiveP400GetLMulCycles<mx>.c;
+    defvar IsWorstCase = SiFiveP400IsWorstCaseMXSEW<mx, sew, SchedMxListF, 1>.c;
+
+    // Simple reduction.
+    defvar BaseC = VFReduceBaseCycles<sew>.val;
+    let Latency = !add(BaseC, !mul(6, !logtwo(LMulLat))), ReleaseAtCycles = [BaseC] in
+    defm "" : LMULSEWWriteResMXSEW<"WriteVFRedV_From", [SiFiveP400VEXQ0],
+                                   mx, sew, IsWorstCase>;
+
+    // Advanced reduction.
+    defvar AdvancedC = AdvancedVFReduceCycles<sew, mx>;
+    let Latency = AdvancedC.latency, ReleaseAtCycles = [AdvancedC.rthroughput] in
+    defm "" : LMULSEWWriteResMXSEW<"WriteVFRedMinMaxV_From",
+                                   [SiFiveP400VEXQ0], mx, sew, IsWorstCase>;
+
+    defvar OrderedRedCycles = !mul(BaseC, LMulLat);
+    let Latency = OrderedRedCycles, ReleaseAtCycles = [OrderedRedCycles] in
+    defm "" : LMULSEWWriteResMXSEW<"WriteVFRedOV_From", [SiFiveP400VEXQ0],
+                                    mx, sew, IsWorstCase>;
+  }
+}
+
+foreach mx = SchedMxListFWRed in {
+  foreach sew = SchedSEWSet<mx, 1, 1>.val in {
+    defvar LMulLat = SiFiveP400GetLMulCycles<mx>.c;
+    defvar IsWorstCase = SiFiveP400IsWorstCaseMXSEW<mx, sew, SchedMxListFWRed, 1>.c;
+    let Latency = !add(6, !mul(6, LMulLat)), ReleaseAtCycles = [LMulLat] in {
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFWRedV_From",  [SiFiveP400VEXQ0],
+                                     mx, sew, IsWorstCase>;
+    }
+
+    defvar OrderedRedCycles = !mul(VFReduceBaseCycles<sew>.val, LMulLat);
+    let Latency = OrderedRedCycles, ReleaseAtCycles = [OrderedRedCycles] in
+    defm "" : LMULSEWWriteResMXSEW<"WriteVFWRedOV_From", [SiFiveP400VEXQ0],
+                                   mx, sew, IsWorstCase>;
+  }
+}
+
+// 15. Vector Mask Instructions
+foreach mx = SchedMxList in {
+  defvar IsWorstCase = SiFiveP400IsWorstCaseMX<mx, SchedMxList>.c;
+  let Latency = 2, ReleaseAtCycles = [1] in {
+    defm "" : LMULWriteResMX<"WriteVMALUV", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVMPopV", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVMFFSV", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVMSFSV", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+  }
+  defvar LMulLat = SiFiveP400GetLMulCycles<mx>.c;
+  let Latency = 2, ReleaseAtCycles = [LMulLat] in {
+    defm "" : LMULWriteResMX<"WriteVIotaV", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVIdxV", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+  }
+}
+
+// 16. Vector Permutation Instructions
+// Simple Slide
+foreach mx = SchedMxList in {
+  defvar LMulLat = SiFiveP400GetLMulCycles<mx>.c;
+  defvar IsWorstCase = SiFiveP400IsWorstCaseMX<mx, SchedMxList>.c;
+  let Latency = 2, ReleaseAtCycles = [LMulLat] in {
+    defm "" : LMULWriteResMX<"WriteVSlideI",  [SiFiveP400VEXQ0], mx, IsWorstCase>;
+  }
+  let Latency = 2, ReleaseAtCycles = [LMulLat] in {
+    defm "" : LMULWriteResMX<"WriteVISlide1X", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVFSlide1F", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+  }
+}
+foreach mx = ["MF8", "MF4", "MF2", "M1"] in {
+  defvar IsWorstCase = SiFiveP400IsWorstCaseMX<mx, SchedMxList>.c;
+  let Latency = 2, ReleaseAtCycles = [1] in {
+    defm "" : LMULWriteResMX<"WriteVSlideUpX",   [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVSlideDownX", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+  }
+}
+
+// Complex Slide
+foreach mx = ["M2", "M4", "M8"] in {
+  defvar LMulLat = SiFiveP400GetLMulCycles<mx>.c;
+  defvar IsWorstCase = SiFiveP400IsWorstCaseMX<mx, SchedMxList>.c;
+
+  defvar UpLatAndCycles = !add(8, LMulLat);
+  let Latency = UpLatAndCycles, ReleaseAtCycles = [UpLatAndCycles] in {
+    defm "" : LMULWriteResMX<"WriteVSlideUpX", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+  }
+  defvar DownLatAndCycles = !add(8, !div(!mul(LMulLat, 3), 2));
+  let Latency = DownLatAndCycles, ReleaseAtCycles = [DownLatAndCycles] in {
+    defm "" : LMULWriteResMX<"WriteVSlideDownX", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+  }
+}
+
+let Latency = 2, ReleaseAtCycles = [2] in {
+  def : WriteRes<WriteVMovXS, [SiFiveP400VEXQ0]>;
+  def : WriteRes<WriteVMovSX, [SiFiveP400VEXQ0]>;
+}
+let Latency = 6, ReleaseAtCycles = [2] in {
+  def : WriteRes<WriteVMovFS, [SiFiveP400VEXQ0]>;
+  def : WriteRes<WriteVMovSF, [SiFiveP400VEXQ0]>;
+}
+
+// Simple Gather and Compress
+foreach mx = ["MF8", "MF4", "MF2", "M1"] in {
+  defvar IsWorstCase = SiFiveP400IsWorstCaseMX<mx, SchedMxList>.c;
+  let Latency = 3, ReleaseAtCycles = [1] in {
+    defm "" : LMULWriteResMX<"WriteVRGatherVX", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+  }
+}
+
+foreach mx = ["MF8", "MF4", "MF2", "M1"] in {
+  foreach sew = SchedSEWSet<mx>.val in {
+    defvar IsWorstCase = SiFiveP400IsWorstCaseMX<mx, SchedMxList>.c;
+    let Latency = 3, ReleaseAtCycles = [1] in {
+      defm "" : LMULSEWWriteResMXSEW<"WriteVRGatherVV", [SiFiveP400VEXQ0], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVRGatherEI16VV", [SiFiveP400VEXQ0], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVCompressV", [SiFiveP400VEXQ0], mx, sew, IsWorstCase>;
+    }
+  }
+}
+
+// Complex Gather and Compress
+foreach mx = ["M2", "M4", "M8"] in {
+  defvar LMulLat = SiFiveP400GetLMulCycles<mx>.c;
+  defvar IsWorstCase = SiFiveP400IsWorstCaseMX<mx, SchedMxList>.c;
+  let Latency = 6, ReleaseAtCycles = [LMulLat] in {
+    defm "" : LMULWriteResMX<"WriteVRGatherVX", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+  }
+}
+
+foreach mx = ["M2", "M4", "M8"] in {
+  foreach sew = SchedSEWSet<mx>.val in {
+    defvar LMulLat = SiFiveP400GetLMulCycles<mx>.c;
+    defvar IsWorstCase = SiFiveP400IsWorstCaseMXSEW<mx, sew, SchedMxList>.c;
+    let Latency = 6, ReleaseAtCycles = [!add(!mul(LMulLat, 2), 8)] in {
+      defm "" : LMULSEWWriteResMXSEW<"WriteVRGatherVV", [SiFiveP400VEXQ0], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVRGatherEI16VV", [SiFiveP400VEXQ0], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVCompressV", [SiFiveP400VEXQ0], mx, sew, IsWorstCase>;
+    }
+  }
+}
+
+// Simple Vrgather.vi
+foreach mx = SchedMxList in {
+  defvar LMulLat = SiFiveP400GetLMulCycles<mx>.c;
+  defvar IsWorstCase = SiFiveP400IsWorstCaseMX<mx, SchedMxList>.c;
+  let Latency = 3, ReleaseAtCycles = [LMulLat] in {
+    defm "" : LMULWriteResMX<"WriteVRGatherVI", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+  }
+}
+
+// Vector Crypto
+foreach mx = SchedMxList in {
+  defvar LMulLat = SiFiveP400GetLMulCycles<mx>.c;
+  defvar IsWorstCase = SiFiveP400IsWorstCaseMX<mx, SchedMxList>.c;
+  // Zvbb
+  let Latency = 2, ReleaseAtCycles = [LMulLat] in {
+    defm "" : LMULWriteResMX<"WriteVBREVV",   [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVCLZV",    [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVCPOPV",   [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVCTZV",    [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVWSLLV",   [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVWSLLX",   [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVWSLLI",   [SiFiveP400VEXQ0], mx, IsWorstCase>;
+  }
+  // Zvbc
+  let Latency = 2, ReleaseAtCycles = [LMulLat] in {
+    defm "" : LMULWriteResMX<"WriteVCLMULV", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVCLMULX", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+  }
+  // Zvkb
+  // VANDN uses WriteVIALU[V|X|I]
+  let Latency = 2, ReleaseAtCycles = [LMulLat] in {
+    defm "" : LMULWriteResMX<"WriteVBREV8V",  [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVREV8V",   [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVRotV",    [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVRotX",    [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVRotI",    [SiFiveP400VEXQ0], mx, IsWorstCase>;
+  }
+  // Zvkg
+  let Latency = 2, ReleaseAtCycles = [LMulLat] in {
+    defm "" : LMULWriteResMX<"WriteVGHSHV",   [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVGMULV",   [SiFiveP400VEXQ0], mx, IsWorstCase>;
+  }
+  // ZvknhaOrZvknhb
+  let Latency = 3, ReleaseAtCycles = [LMulLat] in {
+    defm "" : LMULWriteResMX<"WriteVSHA2CHV", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVSHA2CLV", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVSHA2MSV", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+  }
+  // Zvkned
+  let Latency = 2, ReleaseAtCycles = [LMulLat] in {
+    defm "" : LMULWriteResMX<"WriteVAESMVV",  [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVAESKF1V", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVAESKF2V", [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVAESZV",   [SiFiveP400VEXQ0], mx, IsWorstCase>;
+  }
+  // Zvksed
+  let Latency = 3, ReleaseAtCycles = [SiFiveP400VSM3CCycles<mx>.c] in
+  defm "" : LMULWriteResMX<"WriteVSM3CV",   [SiFiveP400VEXQ0], mx, IsWorstCase>;
+  let Latency = 6, ReleaseAtCycles = [LMulLat] in
+  defm "" : LMULWriteResMX<"WriteVSM3MEV",  [SiFiveP400VEXQ0], mx, IsWorstCase>;
+  let Latency = 3, ReleaseAtCycles = [LMulLat] in {
+    defm "" : LMULWriteResMX<"WriteVSM4KV",   [SiFiveP400VEXQ0], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVSM4RV",   [SiFiveP400VEXQ0], mx, IsWorstCase>;
+  }
+}
+
 // Others
 def : WriteRes<WriteCSR, [SiFiveP400SYS]>;
 def : WriteRes<WriteNop, []>;
+def : WriteRes<WriteRdVLENB, [SiFiveP400SYS]>;
+
 
 // FIXME: This could be better modeled by looking at the regclasses of the operands.
 def : InstRW<[WriteIALU, ReadIALU], (instrs COPY)>;
@@ -356,6 +1012,217 @@ def : ReadAdvance<ReadSHXADD32, 0>;
 def : ReadAdvance<ReadSingleBit, 0>;
 def : ReadAdvance<ReadSingleBitImm, 0>;
 
+// 6. Configuration-Setting Instructions
+def : ReadAdvance<ReadVSETVLI, 0>;
+def : ReadAdvance<ReadVSETVL, 0>;
+
+// 7. Vector Loads and Stores
+def : ReadAdvance<ReadVLDX, 0>;
+def : ReadAdvance<ReadVSTX, 0>;
+defm "" : LMULReadAdvance<"ReadVSTEV", 0>;
+defm "" : LMULReadAdvance<"ReadVSTM", 0>;
+def : ReadAdvance<ReadVLDSX, 0>;
+def : ReadAdvance<ReadVSTSX, 0>;
+defm "" : LMULReadAdvance<"ReadVSTS8V", 0>;
+defm "" : LMULReadAdvance<"ReadVSTS16V", 0>;
+defm "" : LMULReadAdvance<"ReadVSTS32V", 0>;
+defm "" : LMULReadAdvance<"ReadVSTS64V", 0>;
+defm "" : LMULReadAdvance<"ReadVLDUXV", 0>;
+defm "" : LMULReadAdvance<"ReadVLDOXV", 0>;
+defm "" : LMULReadAdvance<"ReadVSTUX8", 0>;
+defm "" : LMULReadAdvance<"ReadVSTUX16", 0>;
+defm "" : LMULReadAdvance<"ReadVSTUX32", 0>;
+defm "" : LMULReadAdvance<"ReadVSTUX64", 0>;
+defm "" : LMULReadAdvance<"ReadVSTUXV", 0>;
+defm "" : LMULReadAdvance<"ReadVSTUX8V", 0>;
+defm "" : LMULReadAdvance<"ReadVSTUX16V", 0>;
+defm "" : LMULReadAdvance<"ReadVSTUX32V", 0>;
+defm "" : LMULReadAdvance<"ReadVSTUX64V", 0>;
+defm "" : LMULReadAdvance<"ReadVSTOX8", 0>;
+defm "" : LMULReadAdvance<"ReadVSTOX16", 0>;
+defm "" : LMULReadAdvance<"ReadVSTOX32", 0>;
+defm "" : LMULReadAdvance<"ReadVSTOX64", 0>;
+defm "" : LMULReadAdvance<"ReadVSTOXV", 0>;
+defm "" : LMULReadAdvance<"ReadVSTOX8V", 0>;
+defm "" : LMULReadAdvance<"ReadVSTOX16V", 0>;
+defm "" : LMULReadAdvance<"ReadVSTOX32V", 0>;
+defm "" : LMULReadAdvance<"ReadVSTOX64V", 0>;
+// LMUL Aware
+def : ReadAdvance<ReadVST1R, 0>;
+def : ReadAdvance<ReadVST2R, 0>;
+def : ReadAdvance<ReadVST4R, 0>;
+def : ReadAdvance<ReadVST8R, 0>;
+
+// 12. Vector Integer Arithmetic Instructions
+defm : LMULReadAdvance<"ReadVIALUV", 0>;
+defm : LMULReadAdvance<"ReadVIALUX", 0>;
+defm : LMULReadAdvanceW<"ReadVIWALUV", 0>;
+defm : LMULReadAdvanceW<"ReadVIWALUX", 0>;
+defm : LMULReadAdvance<"ReadVExtV", 0>;
+defm : LMULReadAdvance<"ReadVICALUV", 0>;
+defm : LMULReadAdvance<"ReadVICALUX", 0>;
+defm : LMULReadAdvance<"ReadVShiftV", 0>;
+defm : LMULReadAdvance<"ReadVShiftX", 0>;
+defm : LMULReadAdvanceW<"ReadVNShiftV", 0>;
+defm : LMULReadAdvanceW<"ReadVNShiftX", 0>;
+defm : LMULReadAdvance<"ReadVICmpV", 0>;
+defm : LMULReadAdvance<"ReadVICmpX", 0>;
+defm : LMULReadAdvance<"ReadVIMinMaxV", 0>;
+defm : LMULReadAdvance<"ReadVIMinMaxX", 0>;
+defm : LMULReadAdvance<"ReadVIMulV", 0>;
+defm : LMULReadAdvance<"ReadVIMulX", 0>;
+defm : LMULSEWReadAdvance<"ReadVIDivV", 0>;
+defm : LMULSEWReadAdvance<"ReadVIDivX", 0>;
+defm : LMULReadAdvanceW<"ReadVIWMulV", 0>;
+defm : LMULReadAdvanceW<"ReadVIWMulX", 0>;
+defm : LMULReadAdvance<"ReadVIMulAddV", 0>;
+defm : LMULReadAdvance<"ReadVIMulAddX", 0>;
+defm : LMULReadAdvanceW<"ReadVIWMulAddV", 0>;
+defm : LMULReadAdvanceW<"ReadVIWMulAddX", 0>;
+defm : LMULReadAdvance<"ReadVIMergeV", 0>;
+defm : LMULReadAdvance<"ReadVIMergeX", 0>;
+defm : LMULReadAdvance<"ReadVIMovV", 0>;
+defm : LMULReadAdvance<"ReadVIMovX", 0>;
+
+// 13. Vector Fixed-Point Arithmetic Instructions
+defm "" : LMULReadAdvance<"ReadVSALUV", 0>;
+defm "" : LMULReadAdvance<"ReadVSALUX", 0>;
+defm "" : LMULReadAdvance<"ReadVAALUV", 0>;
+defm "" : LMULReadAdvance<"ReadVAALUX", 0>;
+defm "" : LMULReadAdvance<"ReadVSMulV", 0>;
+defm "" : LMULReadAdvance<"ReadVSMulX", 0>;
+defm "" : LMULReadAdvance<"ReadVSShiftV", 0>;
+defm "" : LMULReadAdvance<"ReadVSShiftX", 0>;
+defm "" : LMULReadAdvanceW<"ReadVNClipV", 0>;
+defm "" : LMULReadAdvanceW<"ReadVNClipX", 0>;
+
+// 14. Vector Floating-Point Instructions
+defm "" : LMULSEWReadAdvanceF<"ReadVFALUV", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFALUF", 0>;
+defm "" : LMULSEWReadAdvanceFW<"ReadVFWALUV", 0>;
+defm "" : LMULSEWReadAdvanceFW<"ReadVFWALUF", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFMulV", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFMulF", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFDivV", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFDivF", 0>;
+defm "" : LMULSEWReadAdvanceFW<"ReadVFWMulV", 0>;
+defm "" : LMULSEWReadAdvanceFW<"ReadVFWMulF", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFMulAddV", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFMulAddF", 0>;
+defm "" : LMULSEWReadAdvanceFW<"ReadVFWMulAddV", 0>;
+defm "" : LMULSEWReadAdvanceFW<"ReadVFWMulAddF", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFSqrtV", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFRecpV", 0>;
+defm "" : LMULReadAdvance<"ReadVFCmpV", 0>;
+defm "" : LMULReadAdvance<"ReadVFCmpF", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFMinMaxV", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFMinMaxF", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFSgnjV", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFSgnjF", 0>;
+defm "" : LMULReadAdvance<"ReadVFClassV", 0>;
+defm "" : LMULReadAdvance<"ReadVFMergeV", 0>;
+defm "" : LMULReadAdvance<"ReadVFMergeF", 0>;
+defm "" : LMULReadAdvance<"ReadVFMovF", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFCvtIToFV", 0>;
+defm "" : LMULReadAdvance<"ReadVFCvtFToIV", 0>;
+defm "" : LMULSEWReadAdvanceW<"ReadVFWCvtIToFV", 0>;
+defm "" : LMULReadAdvanceFW<"ReadVFWCvtFToIV", 0>;
+defm "" : LMULSEWReadAdvanceFW<"ReadVFWCvtFToFV", 0>;
+defm "" : LMULSEWReadAdvanceFW<"ReadVFNCvtIToFV", 0>;
+defm "" : LMULReadAdvanceW<"ReadVFNCvtFToIV", 0>;
+defm "" : LMULSEWReadAdvanceFW<"ReadVFNCvtFToFV", 0>;
+
+// 15. Vector Reduction Operations
+def : ReadAdvance<ReadVIRedV, 0>;
+def : ReadAdvance<ReadVIRedV0, 0>;
+def : ReadAdvance<ReadVIWRedV, 0>;
+def : ReadAdvance<ReadVIWRedV0, 0>;
+def : ReadAdvance<ReadVFRedV, 0>;
+def : ReadAdvance<ReadVFRedV0, 0>;
+def : ReadAdvance<ReadVFRedOV, 0>;
+def : ReadAdvance<ReadVFRedOV0, 0>;
+def : ReadAdvance<ReadVFWRedV, 0>;
+def : ReadAdvance<ReadVFWRedV0, 0>;
+def : ReadAdvance<ReadVFWRedOV, 0>;
+def : ReadAdvance<ReadVFWRedOV0, 0>;
+
+// 16. Vector Mask Instructions
+defm "" : LMULReadAdvance<"ReadVMALUV", 0>;
+defm "" : LMULReadAdvance<"ReadVMPopV", 0>;
+defm "" : LMULReadAdvance<"ReadVMFFSV", 0>;
+defm "" : LMULReadAdvance<"ReadVMSFSV", 0>;
+defm "" : LMULReadAdvance<"ReadVIotaV", 0>;
+
+// 17. Vector Permutation Instructions
+def : ReadAdvance<ReadVMovXS, 0>;
+def : ReadAdvance<ReadVMovSX_V, 0>;
+def : ReadAdvance<ReadVMovSX_X, 0>;
+def : ReadAdvance<ReadVMovFS, 0>;
+def : ReadAdvance<ReadVMovSF_V, 0>;
+def : ReadAdvance<ReadVMovSF_F, 0>;
+defm "" : LMULReadAdvance<"ReadVISlideV", 0>;
+defm "" : LMULReadAdvance<"ReadVISlideX", 0>;
+defm "" : LMULReadAdvance<"ReadVFSlideV", 0>;
+defm "" : LMULReadAdvance<"ReadVFSlideF", 0>;
+defm "" : LMULSEWReadAdvance<"ReadVRGatherVV_data", 0>;
+defm "" : LMULSEWReadAdvance<"ReadVRGatherVV_index", 0>;
+defm "" : LMULSEWReadAdvance<"ReadVRGatherEI16VV_data", 0>;
+defm "" : LMULSEWReadAdvance<"ReadVRGatherEI16VV_index", 0>;
+defm "" : LMULReadAdvance<"ReadVRGatherVX_data", 0>;
+defm "" : LMULReadAdvance<"ReadVRGatherVX_index", 0>;
+defm "" : LMULReadAdvance<"ReadVRGatherVI_data", 0>;
+defm "" : LMULSEWReadAdvance<"ReadVCompressV", 0>;
+// LMUL Aware
+def : ReadAdvance<ReadVMov1V, 0>;
+def : ReadAdvance<ReadVMov2V, 0>;
+def : ReadAdvance<ReadVMov4V, 0>;
+def : ReadAdvance<ReadVMov8V, 0>;
+
+// Others
+def : ReadAdvance<ReadVMask, 0>;
+def : ReadAdvance<ReadVPassthru_WorstCase, 0>;
+foreach mx = SchedMxList in {
+  def : ReadAdvance<!cast<SchedRead>("ReadVPassthru_" # mx), 0>;
+  foreach sew = SchedSEWSet<mx>.val in
+    def : ReadAdvance<!cast<SchedRead>("ReadVPassthru_" # mx  # "_E" # sew), 0>;
+}
+
+// Vector Crypto Extensions
+// Zvbb
+defm "" : LMULReadAdvance<"ReadVBREVV", 0>;
+defm "" : LMULReadAdvance<"ReadVCLZV", 0>;
+defm "" : LMULReadAdvance<"ReadVCPOPV", 0>;
+defm "" : LMULReadAdvance<"ReadVCTZV", 0>;
+defm "" : LMULReadAdvance<"ReadVWSLLV", 0>;
+defm "" : LMULReadAdvance<"ReadVWSLLX", 0>;
+// Zvbc
+defm "" : LMULReadAdvance<"ReadVCLMULV", 0>;
+defm "" : LMULReadAdvance<"ReadVCLMULX", 0>;
+// Zvkb
+// VANDN uses ReadVIALU[V|X|I]
+defm "" : LMULReadAdvance<"ReadVBREV8V", 0>;
+defm "" : LMULReadAdvance<"ReadVREV8V", 0>;
+defm "" : LMULReadAdvance<"ReadVRotV", 0>;
+defm "" : LMULReadAdvance<"ReadVRotX", 0>;
+// Zvkg
+defm "" : LMULReadAdvance<"ReadVGHSHV", 0>;
+defm "" : LMULReadAdvance<"ReadVGMULV", 0>;
+// Zvknha or Zvknhb
+defm "" : LMULReadAdvance<"ReadVSHA2CHV", 0>;
+defm "" : LMULReadAdvance<"ReadVSHA2CLV", 0>;
+defm "" : LMULReadAdvance<"ReadVSHA2MSV", 0>;
+// Zvkned
+defm "" : LMULReadAdvance<"ReadVAESMVV", 0>;
+defm "" : LMULReadAdvance<"ReadVAESKF1V", 0>;
+defm "" : LMULReadAdvance<"ReadVAESKF2V", 0>;
+defm "" : LMULReadAdvance<"ReadVAESZV", 0>;
+// Zvksed
+defm "" : LMULReadAdvance<"ReadVSM4KV", 0>;
+defm "" : LMULReadAdvance<"ReadVSM4RV", 0>;
+// Zbksh
+defm "" : LMULReadAdvance<"ReadVSM3CV", 0>;
+defm "" : LMULReadAdvance<"ReadVSM3MEV", 0>;
+
 //===----------------------------------------------------------------------===//
 // Unsupported extensions
 defm : UnsupportedSchedZabha;
@@ -364,7 +1231,5 @@ defm : UnsupportedSchedZbkb;
 defm : UnsupportedSchedZbkx;
 defm : UnsupportedSchedSFB;
 defm : UnsupportedSchedZfa;
-defm : UnsupportedSchedV;
 defm : UnsupportedSchedXsfvcp;
-defm : UnsupportedSchedZvk;
 }
diff --git a/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/load.s b/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/load.s
index 3cf207b594ad996..216d2f6c9b57120 100644
--- a/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/load.s
+++ b/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/load.s
@@ -42,15 +42,20 @@ fld ft0, 0(a0)
 # CHECK-NEXT: [5]   - SiFiveP400IEXQ2
 # CHECK-NEXT: [6]   - SiFiveP400Load
 # CHECK-NEXT: [7]   - SiFiveP400Store
+# CHECK-NEXT: [8]   - SiFiveP400VDiv
+# CHECK-NEXT: [9]   - SiFiveP400VEXQ0
+# CHECK-NEXT: [10]  - SiFiveP400VFloatDiv
+# CHECK-NEXT: [11]  - SiFiveP400VLD
+# CHECK-NEXT: [12]  - SiFiveP400VST
 
 # CHECK:      Resource pressure per iteration:
-# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]
-# CHECK-NEXT:  -      -      -      -      -      -     5.00    -
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]
+# CHECK-NEXT:  -      -      -      -      -      -     5.00    -      -      -      -      -      -
 
 # CHECK:      Resource pressure by instruction:
-# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    Instructions:
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     lw	t0, 0(a0)
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     ld	t0, 0(a0)
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     flh	ft0, 0(a0)
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     flw	ft0, 0(a0)
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     fld	ft0, 0(a0)
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -     lw	t0, 0(a0)
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -     ld	t0, 0(a0)
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -     flh	ft0, 0(a0)
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -     flw	ft0, 0(a0)
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -     fld	ft0, 0(a0)
diff --git a/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/vislide-vx.s b/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/vislide-vx.s
new file mode 100644
index 000000000000000..c4934f4ab02095a
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/vislide-vx.s
@@ -0,0 +1,108 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=riscv64 -mcpu=sifive-p470 -iterations=1 < %s | FileCheck %s
+
+vsetvli zero, zero, e32, m2, tu, mu
+vslidedown.vx v5, v7, x6
+
+vsetvli zero, zero, e32, m4, tu, mu
+vslidedown.vx v5, v7, x6
+
+vsetvli zero, zero, e32, m8, tu, mu
+vslidedown.vx v5, v7, x6
+
+vsetvli zero, zero, e32, m2, tu, mu
+vslideup.vx v5, v7, x6
+
+vsetvli zero, zero, e32, m4, tu, mu
+vslideup.vx v5, v7, x6
+
+vsetvli zero, zero, e32, m8, tu, mu
+vslideup.vx v5, v7, x6
+
+vsetvli zero, zero, e32, m2, tu, mu
+vslideup.vx v5, v7, x6, v0.t
+
+vsetvli zero, zero, e32, m4, tu, mu
+vslideup.vx v5, v7, x6, v0.t
+
+vsetvli zero, zero, e32, m8, tu, mu
+vslideup.vx v5, v7, x6, v0.t
+
+# CHECK:      Iterations:        1
+# CHECK-NEXT: Instructions:      18
+# CHECK-NEXT: Total Cycles:      125
+# CHECK-NEXT: Total uOps:        18
+
+# CHECK:      Dispatch Width:    3
+# CHECK-NEXT: uOps Per Cycle:    0.14
+# CHECK-NEXT: IPC:               0.14
+# CHECK-NEXT: Block RThroughput: 121.0
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      11    11.00                       vslidedown.vx v5, v7, t1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      14    14.00                       vslidedown.vx v5, v7, t1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      20    20.00                       vslidedown.vx v5, v7, t1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      10    10.00                       vslideup.vx v5, v7, t1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      12    12.00                       vslideup.vx v5, v7, t1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      16    16.00                       vslideup.vx v5, v7, t1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      10    10.00                       vslideup.vx v5, v7, t1, v0.t
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      12    12.00                       vslideup.vx v5, v7, t1, v0.t
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      16    16.00                       vslideup.vx v5, v7, t1, v0.t
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - SiFiveP400Div
+# CHECK-NEXT: [1]   - SiFiveP400FEXQ0
+# CHECK-NEXT: [2]   - SiFiveP400FloatDiv
+# CHECK-NEXT: [3]   - SiFiveP400IEXQ0
+# CHECK-NEXT: [4]   - SiFiveP400IEXQ1
+# CHECK-NEXT: [5]   - SiFiveP400IEXQ2
+# CHECK-NEXT: [6]   - SiFiveP400Load
+# CHECK-NEXT: [7]   - SiFiveP400Store
+# CHECK-NEXT: [8]   - SiFiveP400VDiv
+# CHECK-NEXT: [9]   - SiFiveP400VEXQ0
+# CHECK-NEXT: [10]  - SiFiveP400VFloatDiv
+# CHECK-NEXT: [11]  - SiFiveP400VLD
+# CHECK-NEXT: [12]  - SiFiveP400VST
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]
+# CHECK-NEXT:  -      -      -      -     9.00    -      -      -      -     121.00  -      -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   Instructions:
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     11.00   -      -      -     vslidedown.vx  v5, v7, t1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     14.00   -      -      -     vslidedown.vx  v5, v7, t1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     20.00   -      -      -     vslidedown.vx  v5, v7, t1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     10.00   -      -      -     vslideup.vx  v5, v7, t1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     12.00   -      -      -     vslideup.vx  v5, v7, t1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     16.00   -      -      -     vslideup.vx  v5, v7, t1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     10.00   -      -      -     vslideup.vx  v5, v7, t1, v0.t
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     12.00   -      -      -     vslideup.vx  v5, v7, t1, v0.t
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     16.00   -      -      -     vslideup.vx  v5, v7, t1, v0.t
+
diff --git a/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/vlseg-vsseg.s b/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/vlseg-vsseg.s
new file mode 100644
index 000000000000000..aba6b2f3e24a5b7
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/vlseg-vsseg.s
@@ -0,0 +1,9827 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=riscv64 -mcpu=sifive-p470 -iterations=1 < %s | FileCheck %s
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vlseg2e8.v  v8,(a0)
+vsetvli zero, zero, e8, mf4, tu, mu
+vlseg2e8.v  v8,(a0)
+vsetvli zero, zero, e8, mf2, tu, mu
+vlseg2e8.v  v8,(a0)
+vsetvli zero, zero, e8, m1, tu, mu
+vlseg2e8.v  v8,(a0)
+vsetvli zero, zero, e8, m2, tu, mu
+vlseg2e8.v  v8,(a0)
+vsetvli zero, zero, e8, m4, tu, mu
+vlseg2e8.v  v8,(a0)
+vsetvli zero, zero, e8, m8, tu, mu
+vlseg2e8.v  v8,(a0)
+vsetvli zero, zero, e16, mf8, tu, mu
+vlseg2e16.v v8,(a0)
+vsetvli zero, zero, e16, mf4, tu, mu
+vlseg2e16.v v8,(a0)
+vsetvli zero, zero, e16, mf2, tu, mu
+vlseg2e16.v v8,(a0)
+vsetvli zero, zero, e16, m1, tu, mu
+vlseg2e16.v v8,(a0)
+vsetvli zero, zero, e16, m2, tu, mu
+vlseg2e16.v v8,(a0)
+vsetvli zero, zero, e16, m4, tu, mu
+vlseg2e16.v v8,(a0)
+vsetvli zero, zero, e16, m8, tu, mu
+vlseg2e16.v v8,(a0)
+vsetvli zero, zero, e32, mf8, tu, mu
+vlseg2e32.v v8,(a0)
+vsetvli zero, zero, e32, mf4, tu, mu
+vlseg2e32.v v8,(a0)
+vsetvli zero, zero, e32, mf2, tu, mu
+vlseg2e32.v v8,(a0)
+vsetvli zero, zero, e32, m1, tu, mu
+vlseg2e32.v v8,(a0)
+vsetvli zero, zero, e32, m2, tu, mu
+vlseg2e32.v v8,(a0)
+vsetvli zero, zero, e32, m4, tu, mu
+vlseg2e32.v v8,(a0)
+vsetvli zero, zero, e32, m8, tu, mu
+vlseg2e32.v v8,(a0)
+vsetvli zero, zero, e64, mf8, tu, mu
+vlseg2e64.v v8,(a0)
+vsetvli zero, zero, e64, mf4, tu, mu
+vlseg2e64.v v8,(a0)
+vsetvli zero, zero, e64, mf2, tu, mu
+vlseg2e64.v v8,(a0)
+vsetvli zero, zero, e64, m1, tu, mu
+vlseg2e64.v v8,(a0)
+vsetvli zero, zero, e64, m2, tu, mu
+vlseg2e64.v v8,(a0)
+vsetvli zero, zero, e64, m4, tu, mu
+vlseg2e64.v v8,(a0)
+vsetvli zero, zero, e64, m8, tu, mu
+vlseg2e64.v v8,(a0)
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vlseg3e8.v  v8,(a0)
+vsetvli zero, zero, e8, mf4, tu, mu
+vlseg3e8.v  v8,(a0)
+vsetvli zero, zero, e8, mf2, tu, mu
+vlseg3e8.v  v8,(a0)
+vsetvli zero, zero, e8, m1, tu, mu
+vlseg3e8.v  v8,(a0)
+vsetvli zero, zero, e8, m2, tu, mu
+vlseg3e8.v  v8,(a0)
+vsetvli zero, zero, e8, m4, tu, mu
+vlseg3e8.v  v8,(a0)
+vsetvli zero, zero, e8, m8, tu, mu
+vlseg3e8.v  v8,(a0)
+vsetvli zero, zero, e16, mf8, tu, mu
+vlseg3e16.v v8,(a0)
+vsetvli zero, zero, e16, mf4, tu, mu
+vlseg3e16.v v8,(a0)
+vsetvli zero, zero, e16, mf2, tu, mu
+vlseg3e16.v v8,(a0)
+vsetvli zero, zero, e16, m1, tu, mu
+vlseg3e16.v v8,(a0)
+vsetvli zero, zero, e16, m2, tu, mu
+vlseg3e16.v v8,(a0)
+vsetvli zero, zero, e16, m4, tu, mu
+vlseg3e16.v v8,(a0)
+vsetvli zero, zero, e16, m8, tu, mu
+vlseg3e16.v v8,(a0)
+vsetvli zero, zero, e32, mf8, tu, mu
+vlseg3e32.v v8,(a0)
+vsetvli zero, zero, e32, mf4, tu, mu
+vlseg3e32.v v8,(a0)
+vsetvli zero, zero, e32, mf2, tu, mu
+vlseg3e32.v v8,(a0)
+vsetvli zero, zero, e32, m1, tu, mu
+vlseg3e32.v v8,(a0)
+vsetvli zero, zero, e32, m2, tu, mu
+vlseg3e32.v v8,(a0)
+vsetvli zero, zero, e32, m4, tu, mu
+vlseg3e32.v v8,(a0)
+vsetvli zero, zero, e32, m8, tu, mu
+vlseg3e32.v v8,(a0)
+vsetvli zero, zero, e64, mf8, tu, mu
+vlseg3e64.v v8,(a0)
+vsetvli zero, zero, e64, mf4, tu, mu
+vlseg3e64.v v8,(a0)
+vsetvli zero, zero, e64, mf2, tu, mu
+vlseg3e64.v v8,(a0)
+vsetvli zero, zero, e64, m1, tu, mu
+vlseg3e64.v v8,(a0)
+vsetvli zero, zero, e64, m2, tu, mu
+vlseg3e64.v v8,(a0)
+vsetvli zero, zero, e64, m4, tu, mu
+vlseg3e64.v v8,(a0)
+vsetvli zero, zero, e64, m8, tu, mu
+vlseg3e64.v v8,(a0)
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vlseg4e8.v  v8,(a0)
+vsetvli zero, zero, e8, mf4, tu, mu
+vlseg4e8.v  v8,(a0)
+vsetvli zero, zero, e8, mf2, tu, mu
+vlseg4e8.v  v8,(a0)
+vsetvli zero, zero, e8, m1, tu, mu
+vlseg4e8.v  v8,(a0)
+vsetvli zero, zero, e8, m2, tu, mu
+vlseg4e8.v  v8,(a0)
+vsetvli zero, zero, e8, m4, tu, mu
+vlseg4e8.v  v8,(a0)
+vsetvli zero, zero, e8, m8, tu, mu
+vlseg4e8.v  v8,(a0)
+vsetvli zero, zero, e16, mf8, tu, mu
+vlseg4e16.v v8,(a0)
+vsetvli zero, zero, e16, mf4, tu, mu
+vlseg4e16.v v8,(a0)
+vsetvli zero, zero, e16, mf2, tu, mu
+vlseg4e16.v v8,(a0)
+vsetvli zero, zero, e16, m1, tu, mu
+vlseg4e16.v v8,(a0)
+vsetvli zero, zero, e16, m2, tu, mu
+vlseg4e16.v v8,(a0)
+vsetvli zero, zero, e16, m4, tu, mu
+vlseg4e16.v v8,(a0)
+vsetvli zero, zero, e16, m8, tu, mu
+vlseg4e16.v v8,(a0)
+vsetvli zero, zero, e32, mf8, tu, mu
+vlseg4e32.v v8,(a0)
+vsetvli zero, zero, e32, mf4, tu, mu
+vlseg4e32.v v8,(a0)
+vsetvli zero, zero, e32, mf2, tu, mu
+vlseg4e32.v v8,(a0)
+vsetvli zero, zero, e32, m1, tu, mu
+vlseg4e32.v v8,(a0)
+vsetvli zero, zero, e32, m2, tu, mu
+vlseg4e32.v v8,(a0)
+vsetvli zero, zero, e32, m4, tu, mu
+vlseg4e32.v v8,(a0)
+vsetvli zero, zero, e32, m8, tu, mu
+vlseg4e32.v v8,(a0)
+vsetvli zero, zero, e64, mf8, tu, mu
+vlseg4e64.v v8,(a0)
+vsetvli zero, zero, e64, mf4, tu, mu
+vlseg4e64.v v8,(a0)
+vsetvli zero, zero, e64, mf2, tu, mu
+vlseg4e64.v v8,(a0)
+vsetvli zero, zero, e64, m1, tu, mu
+vlseg4e64.v v8,(a0)
+vsetvli zero, zero, e64, m2, tu, mu
+vlseg4e64.v v8,(a0)
+vsetvli zero, zero, e64, m4, tu, mu
+vlseg4e64.v v8,(a0)
+vsetvli zero, zero, e64, m8, tu, mu
+vlseg4e64.v v8,(a0)
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vlseg5e8.v  v8,(a0)
+vsetvli zero, zero, e8, mf4, tu, mu
+vlseg5e8.v  v8,(a0)
+vsetvli zero, zero, e8, mf2, tu, mu
+vlseg5e8.v  v8,(a0)
+vsetvli zero, zero, e8, m1, tu, mu
+vlseg5e8.v  v8,(a0)
+vsetvli zero, zero, e8, m2, tu, mu
+vlseg5e8.v  v8,(a0)
+vsetvli zero, zero, e8, m4, tu, mu
+vlseg5e8.v  v8,(a0)
+vsetvli zero, zero, e16, mf8, tu, mu
+vlseg5e16.v v8,(a0)
+vsetvli zero, zero, e16, mf4, tu, mu
+vlseg5e16.v v8,(a0)
+vsetvli zero, zero, e16, mf2, tu, mu
+vlseg5e16.v v8,(a0)
+vsetvli zero, zero, e16, m1, tu, mu
+vlseg5e16.v v8,(a0)
+vsetvli zero, zero, e16, m2, tu, mu
+vlseg5e16.v v8,(a0)
+vsetvli zero, zero, e16, m4, tu, mu
+vlseg5e16.v v8,(a0)
+vsetvli zero, zero, e32, mf8, tu, mu
+vlseg5e32.v v8,(a0)
+vsetvli zero, zero, e32, mf4, tu, mu
+vlseg5e32.v v8,(a0)
+vsetvli zero, zero, e32, mf2, tu, mu
+vlseg5e32.v v8,(a0)
+vsetvli zero, zero, e32, m1, tu, mu
+vlseg5e32.v v8,(a0)
+vsetvli zero, zero, e32, m2, tu, mu
+vlseg5e32.v v8,(a0)
+vsetvli zero, zero, e32, m4, tu, mu
+vlseg5e32.v v8,(a0)
+vsetvli zero, zero, e64, mf8, tu, mu
+vlseg5e64.v v8,(a0)
+vsetvli zero, zero, e64, mf4, tu, mu
+vlseg5e64.v v8,(a0)
+vsetvli zero, zero, e64, mf2, tu, mu
+vlseg5e64.v v8,(a0)
+vsetvli zero, zero, e64, m1, tu, mu
+vlseg5e64.v v8,(a0)
+vsetvli zero, zero, e64, m2, tu, mu
+vlseg5e64.v v8,(a0)
+vsetvli zero, zero, e64, m4, tu, mu
+vlseg5e64.v v8,(a0)
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vlseg6e8.v  v8,(a0)
+vsetvli zero, zero, e8, mf4, tu, mu
+vlseg6e8.v  v8,(a0)
+vsetvli zero, zero, e8, mf2, tu, mu
+vlseg6e8.v  v8,(a0)
+vsetvli zero, zero, e8, m1, tu, mu
+vlseg6e8.v  v8,(a0)
+vsetvli zero, zero, e8, m2, tu, mu
+vlseg6e8.v  v8,(a0)
+vsetvli zero, zero, e8, m4, tu, mu
+vlseg6e8.v  v8,(a0)
+vsetvli zero, zero, e16, mf8, tu, mu
+vlseg6e16.v v8,(a0)
+vsetvli zero, zero, e16, mf4, tu, mu
+vlseg6e16.v v8,(a0)
+vsetvli zero, zero, e16, mf2, tu, mu
+vlseg6e16.v v8,(a0)
+vsetvli zero, zero, e16, m1, tu, mu
+vlseg6e16.v v8,(a0)
+vsetvli zero, zero, e16, m2, tu, mu
+vlseg6e16.v v8,(a0)
+vsetvli zero, zero, e16, m4, tu, mu
+vlseg6e16.v v8,(a0)
+vsetvli zero, zero, e32, mf8, tu, mu
+vlseg6e32.v v8,(a0)
+vsetvli zero, zero, e32, mf4, tu, mu
+vlseg6e32.v v8,(a0)
+vsetvli zero, zero, e32, mf2, tu, mu
+vlseg6e32.v v8,(a0)
+vsetvli zero, zero, e32, m1, tu, mu
+vlseg6e32.v v8,(a0)
+vsetvli zero, zero, e32, m2, tu, mu
+vlseg6e32.v v8,(a0)
+vsetvli zero, zero, e32, m4, tu, mu
+vlseg6e32.v v8,(a0)
+vsetvli zero, zero, e64, mf8, tu, mu
+vlseg6e64.v v8,(a0)
+vsetvli zero, zero, e64, mf4, tu, mu
+vlseg6e64.v v8,(a0)
+vsetvli zero, zero, e64, mf2, tu, mu
+vlseg6e64.v v8,(a0)
+vsetvli zero, zero, e64, m1, tu, mu
+vlseg6e64.v v8,(a0)
+vsetvli zero, zero, e64, m2, tu, mu
+vlseg6e64.v v8,(a0)
+vsetvli zero, zero, e64, m4, tu, mu
+vlseg6e64.v v8,(a0)
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vlseg7e8.v  v8,(a0)
+vsetvli zero, zero, e8, mf4, tu, mu
+vlseg7e8.v  v8,(a0)
+vsetvli zero, zero, e8, mf2, tu, mu
+vlseg7e8.v  v8,(a0)
+vsetvli zero, zero, e8, m1, tu, mu
+vlseg7e8.v  v8,(a0)
+vsetvli zero, zero, e8, m2, tu, mu
+vlseg7e8.v  v8,(a0)
+vsetvli zero, zero, e8, m4, tu, mu
+vlseg7e8.v  v8,(a0)
+vsetvli zero, zero, e16, mf8, tu, mu
+vlseg7e16.v v8,(a0)
+vsetvli zero, zero, e16, mf4, tu, mu
+vlseg7e16.v v8,(a0)
+vsetvli zero, zero, e16, mf2, tu, mu
+vlseg7e16.v v8,(a0)
+vsetvli zero, zero, e16, m1, tu, mu
+vlseg7e16.v v8,(a0)
+vsetvli zero, zero, e16, m2, tu, mu
+vlseg7e16.v v8,(a0)
+vsetvli zero, zero, e16, m4, tu, mu
+vlseg7e16.v v8,(a0)
+vsetvli zero, zero, e32, mf8, tu, mu
+vlseg7e32.v v8,(a0)
+vsetvli zero, zero, e32, mf4, tu, mu
+vlseg7e32.v v8,(a0)
+vsetvli zero, zero, e32, mf2, tu, mu
+vlseg7e32.v v8,(a0)
+vsetvli zero, zero, e32, m1, tu, mu
+vlseg7e32.v v8,(a0)
+vsetvli zero, zero, e32, m2, tu, mu
+vlseg7e32.v v8,(a0)
+vsetvli zero, zero, e32, m4, tu, mu
+vlseg7e32.v v8,(a0)
+vsetvli zero, zero, e64, mf8, tu, mu
+vlseg7e64.v v8,(a0)
+vsetvli zero, zero, e64, mf4, tu, mu
+vlseg7e64.v v8,(a0)
+vsetvli zero, zero, e64, mf2, tu, mu
+vlseg7e64.v v8,(a0)
+vsetvli zero, zero, e64, m1, tu, mu
+vlseg7e64.v v8,(a0)
+vsetvli zero, zero, e64, m2, tu, mu
+vlseg7e64.v v8,(a0)
+vsetvli zero, zero, e64, m4, tu, mu
+vlseg7e64.v v8,(a0)
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vlseg8e8.v  v8,(a0)
+vsetvli zero, zero, e8, mf4, tu, mu
+vlseg8e8.v  v8,(a0)
+vsetvli zero, zero, e8, mf2, tu, mu
+vlseg8e8.v  v8,(a0)
+vsetvli zero, zero, e8, m1, tu, mu
+vlseg8e8.v  v8,(a0)
+vsetvli zero, zero, e8, m2, tu, mu
+vlseg8e8.v  v8,(a0)
+vsetvli zero, zero, e8, m4, tu, mu
+vlseg8e8.v  v8,(a0)
+vsetvli zero, zero, e16, mf8, tu, mu
+vlseg8e16.v v8,(a0)
+vsetvli zero, zero, e16, mf4, tu, mu
+vlseg8e16.v v8,(a0)
+vsetvli zero, zero, e16, mf2, tu, mu
+vlseg8e16.v v8,(a0)
+vsetvli zero, zero, e16, m1, tu, mu
+vlseg8e16.v v8,(a0)
+vsetvli zero, zero, e16, m2, tu, mu
+vlseg8e16.v v8,(a0)
+vsetvli zero, zero, e16, m4, tu, mu
+vlseg8e16.v v8,(a0)
+vsetvli zero, zero, e32, mf8, tu, mu
+vlseg8e32.v v8,(a0)
+vsetvli zero, zero, e32, mf4, tu, mu
+vlseg8e32.v v8,(a0)
+vsetvli zero, zero, e32, mf2, tu, mu
+vlseg8e32.v v8,(a0)
+vsetvli zero, zero, e32, m1, tu, mu
+vlseg8e32.v v8,(a0)
+vsetvli zero, zero, e32, m2, tu, mu
+vlseg8e32.v v8,(a0)
+vsetvli zero, zero, e32, m4, tu, mu
+vlseg8e32.v v8,(a0)
+vsetvli zero, zero, e64, mf8, tu, mu
+vlseg8e64.v v8,(a0)
+vsetvli zero, zero, e64, mf4, tu, mu
+vlseg8e64.v v8,(a0)
+vsetvli zero, zero, e64, mf2, tu, mu
+vlseg8e64.v v8,(a0)
+vsetvli zero, zero, e64, m1, tu, mu
+vlseg8e64.v v8,(a0)
+vsetvli zero, zero, e64, m2, tu, mu
+vlseg8e64.v v8,(a0)
+vsetvli zero, zero, e64, m4, tu, mu
+vlseg8e64.v v8,(a0)
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vsseg2e8.v  v8,(a0)
+vsetvli zero, zero, e8, mf4, tu, mu
+vsseg2e8.v  v8,(a0)
+vsetvli zero, zero, e8, mf2, tu, mu
+vsseg2e8.v  v8,(a0)
+vsetvli zero, zero, e8, m1, tu, mu
+vsseg2e8.v  v8,(a0)
+vsetvli zero, zero, e8, m2, tu, mu
+vsseg2e8.v  v8,(a0)
+vsetvli zero, zero, e8, m4, tu, mu
+vsseg2e8.v  v8,(a0)
+vsetvli zero, zero, e8, m8, tu, mu
+vsseg2e8.v  v8,(a0)
+vsetvli zero, zero, e16, mf8, tu, mu
+vsseg2e16.v v8,(a0)
+vsetvli zero, zero, e16, mf4, tu, mu
+vsseg2e16.v v8,(a0)
+vsetvli zero, zero, e16, mf2, tu, mu
+vsseg2e16.v v8,(a0)
+vsetvli zero, zero, e16, m1, tu, mu
+vsseg2e16.v v8,(a0)
+vsetvli zero, zero, e16, m2, tu, mu
+vsseg2e16.v v8,(a0)
+vsetvli zero, zero, e16, m4, tu, mu
+vsseg2e16.v v8,(a0)
+vsetvli zero, zero, e16, m8, tu, mu
+vsseg2e16.v v8,(a0)
+vsetvli zero, zero, e32, mf8, tu, mu
+vsseg2e32.v v8,(a0)
+vsetvli zero, zero, e32, mf4, tu, mu
+vsseg2e32.v v8,(a0)
+vsetvli zero, zero, e32, mf2, tu, mu
+vsseg2e32.v v8,(a0)
+vsetvli zero, zero, e32, m1, tu, mu
+vsseg2e32.v v8,(a0)
+vsetvli zero, zero, e32, m2, tu, mu
+vsseg2e32.v v8,(a0)
+vsetvli zero, zero, e32, m4, tu, mu
+vsseg2e32.v v8,(a0)
+vsetvli zero, zero, e32, m8, tu, mu
+vsseg2e32.v v8,(a0)
+vsetvli zero, zero, e64, mf8, tu, mu
+vsseg2e64.v v8,(a0)
+vsetvli zero, zero, e64, mf4, tu, mu
+vsseg2e64.v v8,(a0)
+vsetvli zero, zero, e64, mf2, tu, mu
+vsseg2e64.v v8,(a0)
+vsetvli zero, zero, e64, m1, tu, mu
+vsseg2e64.v v8,(a0)
+vsetvli zero, zero, e64, m2, tu, mu
+vsseg2e64.v v8,(a0)
+vsetvli zero, zero, e64, m4, tu, mu
+vsseg2e64.v v8,(a0)
+vsetvli zero, zero, e64, m8, tu, mu
+vsseg2e64.v v8,(a0)
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vsseg3e8.v  v8,(a0)
+vsetvli zero, zero, e8, mf4, tu, mu
+vsseg3e8.v  v8,(a0)
+vsetvli zero, zero, e8, mf2, tu, mu
+vsseg3e8.v  v8,(a0)
+vsetvli zero, zero, e8, m1, tu, mu
+vsseg3e8.v  v8,(a0)
+vsetvli zero, zero, e8, m2, tu, mu
+vsseg3e8.v  v8,(a0)
+vsetvli zero, zero, e8, m4, tu, mu
+vsseg3e8.v  v8,(a0)
+vsetvli zero, zero, e8, m8, tu, mu
+vsseg3e8.v  v8,(a0)
+vsetvli zero, zero, e16, mf8, tu, mu
+vsseg3e16.v v8,(a0)
+vsetvli zero, zero, e16, mf4, tu, mu
+vsseg3e16.v v8,(a0)
+vsetvli zero, zero, e16, mf2, tu, mu
+vsseg3e16.v v8,(a0)
+vsetvli zero, zero, e16, m1, tu, mu
+vsseg3e16.v v8,(a0)
+vsetvli zero, zero, e16, m2, tu, mu
+vsseg3e16.v v8,(a0)
+vsetvli zero, zero, e16, m4, tu, mu
+vsseg3e16.v v8,(a0)
+vsetvli zero, zero, e16, m8, tu, mu
+vsseg3e16.v v8,(a0)
+vsetvli zero, zero, e32, mf8, tu, mu
+vsseg3e32.v v8,(a0)
+vsetvli zero, zero, e32, mf4, tu, mu
+vsseg3e32.v v8,(a0)
+vsetvli zero, zero, e32, mf2, tu, mu
+vsseg3e32.v v8,(a0)
+vsetvli zero, zero, e32, m1, tu, mu
+vsseg3e32.v v8,(a0)
+vsetvli zero, zero, e32, m2, tu, mu
+vsseg3e32.v v8,(a0)
+vsetvli zero, zero, e32, m4, tu, mu
+vsseg3e32.v v8,(a0)
+vsetvli zero, zero, e32, m8, tu, mu
+vsseg3e32.v v8,(a0)
+vsetvli zero, zero, e64, mf8, tu, mu
+vsseg3e64.v v8,(a0)
+vsetvli zero, zero, e64, mf4, tu, mu
+vsseg3e64.v v8,(a0)
+vsetvli zero, zero, e64, mf2, tu, mu
+vsseg3e64.v v8,(a0)
+vsetvli zero, zero, e64, m1, tu, mu
+vsseg3e64.v v8,(a0)
+vsetvli zero, zero, e64, m2, tu, mu
+vsseg3e64.v v8,(a0)
+vsetvli zero, zero, e64, m4, tu, mu
+vsseg3e64.v v8,(a0)
+vsetvli zero, zero, e64, m8, tu, mu
+vsseg3e64.v v8,(a0)
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vsseg4e8.v  v8,(a0)
+vsetvli zero, zero, e8, mf4, tu, mu
+vsseg4e8.v  v8,(a0)
+vsetvli zero, zero, e8, mf2, tu, mu
+vsseg4e8.v  v8,(a0)
+vsetvli zero, zero, e8, m1, tu, mu
+vsseg4e8.v  v8,(a0)
+vsetvli zero, zero, e8, m2, tu, mu
+vsseg4e8.v  v8,(a0)
+vsetvli zero, zero, e8, m4, tu, mu
+vsseg4e8.v  v8,(a0)
+vsetvli zero, zero, e8, m8, tu, mu
+vsseg4e8.v  v8,(a0)
+vsetvli zero, zero, e16, mf8, tu, mu
+vsseg4e16.v v8,(a0)
+vsetvli zero, zero, e16, mf4, tu, mu
+vsseg4e16.v v8,(a0)
+vsetvli zero, zero, e16, mf2, tu, mu
+vsseg4e16.v v8,(a0)
+vsetvli zero, zero, e16, m1, tu, mu
+vsseg4e16.v v8,(a0)
+vsetvli zero, zero, e16, m2, tu, mu
+vsseg4e16.v v8,(a0)
+vsetvli zero, zero, e16, m4, tu, mu
+vsseg4e16.v v8,(a0)
+vsetvli zero, zero, e16, m8, tu, mu
+vsseg4e16.v v8,(a0)
+vsetvli zero, zero, e32, mf8, tu, mu
+vsseg4e32.v v8,(a0)
+vsetvli zero, zero, e32, mf4, tu, mu
+vsseg4e32.v v8,(a0)
+vsetvli zero, zero, e32, mf2, tu, mu
+vsseg4e32.v v8,(a0)
+vsetvli zero, zero, e32, m1, tu, mu
+vsseg4e32.v v8,(a0)
+vsetvli zero, zero, e32, m2, tu, mu
+vsseg4e32.v v8,(a0)
+vsetvli zero, zero, e32, m4, tu, mu
+vsseg4e32.v v8,(a0)
+vsetvli zero, zero, e32, m8, tu, mu
+vsseg4e32.v v8,(a0)
+vsetvli zero, zero, e64, mf8, tu, mu
+vsseg4e64.v v8,(a0)
+vsetvli zero, zero, e64, mf4, tu, mu
+vsseg4e64.v v8,(a0)
+vsetvli zero, zero, e64, mf2, tu, mu
+vsseg4e64.v v8,(a0)
+vsetvli zero, zero, e64, m1, tu, mu
+vsseg4e64.v v8,(a0)
+vsetvli zero, zero, e64, m2, tu, mu
+vsseg4e64.v v8,(a0)
+vsetvli zero, zero, e64, m4, tu, mu
+vsseg4e64.v v8,(a0)
+vsetvli zero, zero, e64, m8, tu, mu
+vsseg4e64.v v8,(a0)
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vsseg5e8.v  v8,(a0)
+vsetvli zero, zero, e8, mf4, tu, mu
+vsseg5e8.v  v8,(a0)
+vsetvli zero, zero, e8, mf2, tu, mu
+vsseg5e8.v  v8,(a0)
+vsetvli zero, zero, e8, m1, tu, mu
+vsseg5e8.v  v8,(a0)
+vsetvli zero, zero, e8, m2, tu, mu
+vsseg5e8.v  v8,(a0)
+vsetvli zero, zero, e8, m4, tu, mu
+vsseg5e8.v  v8,(a0)
+vsetvli zero, zero, e16, mf8, tu, mu
+vsseg5e16.v v8,(a0)
+vsetvli zero, zero, e16, mf4, tu, mu
+vsseg5e16.v v8,(a0)
+vsetvli zero, zero, e16, mf2, tu, mu
+vsseg5e16.v v8,(a0)
+vsetvli zero, zero, e16, m1, tu, mu
+vsseg5e16.v v8,(a0)
+vsetvli zero, zero, e16, m2, tu, mu
+vsseg5e16.v v8,(a0)
+vsetvli zero, zero, e16, m4, tu, mu
+vsseg5e16.v v8,(a0)
+vsetvli zero, zero, e32, mf8, tu, mu
+vsseg5e32.v v8,(a0)
+vsetvli zero, zero, e32, mf4, tu, mu
+vsseg5e32.v v8,(a0)
+vsetvli zero, zero, e32, mf2, tu, mu
+vsseg5e32.v v8,(a0)
+vsetvli zero, zero, e32, m1, tu, mu
+vsseg5e32.v v8,(a0)
+vsetvli zero, zero, e32, m2, tu, mu
+vsseg5e32.v v8,(a0)
+vsetvli zero, zero, e32, m4, tu, mu
+vsseg5e32.v v8,(a0)
+vsetvli zero, zero, e64, mf8, tu, mu
+vsseg5e64.v v8,(a0)
+vsetvli zero, zero, e64, mf4, tu, mu
+vsseg5e64.v v8,(a0)
+vsetvli zero, zero, e64, mf2, tu, mu
+vsseg5e64.v v8,(a0)
+vsetvli zero, zero, e64, m1, tu, mu
+vsseg5e64.v v8,(a0)
+vsetvli zero, zero, e64, m2, tu, mu
+vsseg5e64.v v8,(a0)
+vsetvli zero, zero, e64, m4, tu, mu
+vsseg5e64.v v8,(a0)
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vsseg6e8.v  v8,(a0)
+vsetvli zero, zero, e8, mf4, tu, mu
+vsseg6e8.v  v8,(a0)
+vsetvli zero, zero, e8, mf2, tu, mu
+vsseg6e8.v  v8,(a0)
+vsetvli zero, zero, e8, m1, tu, mu
+vsseg6e8.v  v8,(a0)
+vsetvli zero, zero, e8, m2, tu, mu
+vsseg6e8.v  v8,(a0)
+vsetvli zero, zero, e8, m4, tu, mu
+vsseg6e8.v  v8,(a0)
+vsetvli zero, zero, e16, mf8, tu, mu
+vsseg6e16.v v8,(a0)
+vsetvli zero, zero, e16, mf4, tu, mu
+vsseg6e16.v v8,(a0)
+vsetvli zero, zero, e16, mf2, tu, mu
+vsseg6e16.v v8,(a0)
+vsetvli zero, zero, e16, m1, tu, mu
+vsseg6e16.v v8,(a0)
+vsetvli zero, zero, e16, m2, tu, mu
+vsseg6e16.v v8,(a0)
+vsetvli zero, zero, e16, m4, tu, mu
+vsseg6e16.v v8,(a0)
+vsetvli zero, zero, e32, mf8, tu, mu
+vsseg6e32.v v8,(a0)
+vsetvli zero, zero, e32, mf4, tu, mu
+vsseg6e32.v v8,(a0)
+vsetvli zero, zero, e32, mf2, tu, mu
+vsseg6e32.v v8,(a0)
+vsetvli zero, zero, e32, m1, tu, mu
+vsseg6e32.v v8,(a0)
+vsetvli zero, zero, e32, m2, tu, mu
+vsseg6e32.v v8,(a0)
+vsetvli zero, zero, e32, m4, tu, mu
+vsseg6e32.v v8,(a0)
+vsetvli zero, zero, e64, mf8, tu, mu
+vsseg6e64.v v8,(a0)
+vsetvli zero, zero, e64, mf4, tu, mu
+vsseg6e64.v v8,(a0)
+vsetvli zero, zero, e64, mf2, tu, mu
+vsseg6e64.v v8,(a0)
+vsetvli zero, zero, e64, m1, tu, mu
+vsseg6e64.v v8,(a0)
+vsetvli zero, zero, e64, m2, tu, mu
+vsseg6e64.v v8,(a0)
+vsetvli zero, zero, e64, m4, tu, mu
+vsseg6e64.v v8,(a0)
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vsseg7e8.v  v8,(a0)
+vsetvli zero, zero, e8, mf4, tu, mu
+vsseg7e8.v  v8,(a0)
+vsetvli zero, zero, e8, mf2, tu, mu
+vsseg7e8.v  v8,(a0)
+vsetvli zero, zero, e8, m1, tu, mu
+vsseg7e8.v  v8,(a0)
+vsetvli zero, zero, e8, m2, tu, mu
+vsseg7e8.v  v8,(a0)
+vsetvli zero, zero, e8, m4, tu, mu
+vsseg7e8.v  v8,(a0)
+vsetvli zero, zero, e16, mf8, tu, mu
+vsseg7e16.v v8,(a0)
+vsetvli zero, zero, e16, mf4, tu, mu
+vsseg7e16.v v8,(a0)
+vsetvli zero, zero, e16, mf2, tu, mu
+vsseg7e16.v v8,(a0)
+vsetvli zero, zero, e16, m1, tu, mu
+vsseg7e16.v v8,(a0)
+vsetvli zero, zero, e16, m2, tu, mu
+vsseg7e16.v v8,(a0)
+vsetvli zero, zero, e16, m4, tu, mu
+vsseg7e16.v v8,(a0)
+vsetvli zero, zero, e32, mf8, tu, mu
+vsseg7e32.v v8,(a0)
+vsetvli zero, zero, e32, mf4, tu, mu
+vsseg7e32.v v8,(a0)
+vsetvli zero, zero, e32, mf2, tu, mu
+vsseg7e32.v v8,(a0)
+vsetvli zero, zero, e32, m1, tu, mu
+vsseg7e32.v v8,(a0)
+vsetvli zero, zero, e32, m2, tu, mu
+vsseg7e32.v v8,(a0)
+vsetvli zero, zero, e32, m4, tu, mu
+vsseg7e32.v v8,(a0)
+vsetvli zero, zero, e64, mf8, tu, mu
+vsseg7e64.v v8,(a0)
+vsetvli zero, zero, e64, mf4, tu, mu
+vsseg7e64.v v8,(a0)
+vsetvli zero, zero, e64, mf2, tu, mu
+vsseg7e64.v v8,(a0)
+vsetvli zero, zero, e64, m1, tu, mu
+vsseg7e64.v v8,(a0)
+vsetvli zero, zero, e64, m2, tu, mu
+vsseg7e64.v v8,(a0)
+vsetvli zero, zero, e64, m4, tu, mu
+vsseg7e64.v v8,(a0)
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vsseg8e8.v  v8,(a0)
+vsetvli zero, zero, e8, mf4, tu, mu
+vsseg8e8.v  v8,(a0)
+vsetvli zero, zero, e8, mf2, tu, mu
+vsseg8e8.v  v8,(a0)
+vsetvli zero, zero, e8, m1, tu, mu
+vsseg8e8.v  v8,(a0)
+vsetvli zero, zero, e8, m2, tu, mu
+vsseg8e8.v  v8,(a0)
+vsetvli zero, zero, e8, m4, tu, mu
+vsseg8e8.v  v8,(a0)
+vsetvli zero, zero, e16, mf8, tu, mu
+vsseg8e16.v v8,(a0)
+vsetvli zero, zero, e16, mf4, tu, mu
+vsseg8e16.v v8,(a0)
+vsetvli zero, zero, e16, mf2, tu, mu
+vsseg8e16.v v8,(a0)
+vsetvli zero, zero, e16, m1, tu, mu
+vsseg8e16.v v8,(a0)
+vsetvli zero, zero, e16, m2, tu, mu
+vsseg8e16.v v8,(a0)
+vsetvli zero, zero, e16, m4, tu, mu
+vsseg8e16.v v8,(a0)
+vsetvli zero, zero, e32, mf8, tu, mu
+vsseg8e32.v v8,(a0)
+vsetvli zero, zero, e32, mf4, tu, mu
+vsseg8e32.v v8,(a0)
+vsetvli zero, zero, e32, mf2, tu, mu
+vsseg8e32.v v8,(a0)
+vsetvli zero, zero, e32, m1, tu, mu
+vsseg8e32.v v8,(a0)
+vsetvli zero, zero, e32, m2, tu, mu
+vsseg8e32.v v8,(a0)
+vsetvli zero, zero, e32, m4, tu, mu
+vsseg8e32.v v8,(a0)
+vsetvli zero, zero, e64, mf8, tu, mu
+vsseg8e64.v v8,(a0)
+vsetvli zero, zero, e64, mf4, tu, mu
+vsseg8e64.v v8,(a0)
+vsetvli zero, zero, e64, mf2, tu, mu
+vsseg8e64.v v8,(a0)
+vsetvli zero, zero, e64, m1, tu, mu
+vsseg8e64.v v8,(a0)
+vsetvli zero, zero, e64, m2, tu, mu
+vsseg8e64.v v8,(a0)
+vsetvli zero, zero, e64, m4, tu, mu
+vsseg8e64.v v8,(a0)
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vlsseg2e8.v v8, (a0), a1
+vsetvli zero, zero, e8, mf4, tu, mu
+vlsseg2e8.v v8, (a0), a1
+vsetvli zero, zero, e8, mf2, tu, mu
+vlsseg2e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m1, tu, mu
+vlsseg2e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m2, tu, mu
+vlsseg2e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m4, tu, mu
+vlsseg2e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m8, tu, mu
+vlsseg2e8.v v8, (a0), a1
+vsetvli zero, zero, e16, mf8, tu, mu
+vlsseg2e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, mf4, tu, mu
+vlsseg2e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, mf2, tu, mu
+vlsseg2e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m1, tu, mu
+vlsseg2e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m2, tu, mu
+vlsseg2e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m4, tu, mu
+vlsseg2e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m8, tu, mu
+vlsseg2e16.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf8, tu, mu
+vlsseg2e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf4, tu, mu
+vlsseg2e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf2, tu, mu
+vlsseg2e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m1, tu, mu
+vlsseg2e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m2, tu, mu
+vlsseg2e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m4, tu, mu
+vlsseg2e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m8, tu, mu
+vlsseg2e32.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf8, tu, mu
+vlsseg2e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf4, tu, mu
+vlsseg2e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf2, tu, mu
+vlsseg2e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m1, tu, mu
+vlsseg2e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m2, tu, mu
+vlsseg2e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m4, tu, mu
+vlsseg2e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m8, tu, mu
+vlsseg2e64.v  v8, (a0), a1
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vlsseg3e8.v v8, (a0), a1
+vsetvli zero, zero, e8, mf4, tu, mu
+vlsseg3e8.v v8, (a0), a1
+vsetvli zero, zero, e8, mf2, tu, mu
+vlsseg3e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m1, tu, mu
+vlsseg3e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m2, tu, mu
+vlsseg3e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m4, tu, mu
+vlsseg3e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m8, tu, mu
+vlsseg3e8.v v8, (a0), a1
+vsetvli zero, zero, e16, mf8, tu, mu
+vlsseg3e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, mf4, tu, mu
+vlsseg3e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, mf2, tu, mu
+vlsseg3e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m1, tu, mu
+vlsseg3e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m2, tu, mu
+vlsseg3e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m4, tu, mu
+vlsseg3e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m8, tu, mu
+vlsseg3e16.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf8, tu, mu
+vlsseg3e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf4, tu, mu
+vlsseg3e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf2, tu, mu
+vlsseg3e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m1, tu, mu
+vlsseg3e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m2, tu, mu
+vlsseg3e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m4, tu, mu
+vlsseg3e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m8, tu, mu
+vlsseg3e32.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf8, tu, mu
+vlsseg3e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf4, tu, mu
+vlsseg3e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf2, tu, mu
+vlsseg3e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m1, tu, mu
+vlsseg3e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m2, tu, mu
+vlsseg3e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m4, tu, mu
+vlsseg3e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m8, tu, mu
+vlsseg3e64.v  v8, (a0), a1
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vlsseg4e8.v v8, (a0), a1
+vsetvli zero, zero, e8, mf4, tu, mu
+vlsseg4e8.v v8, (a0), a1
+vsetvli zero, zero, e8, mf2, tu, mu
+vlsseg4e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m1, tu, mu
+vlsseg4e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m2, tu, mu
+vlsseg4e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m4, tu, mu
+vlsseg4e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m8, tu, mu
+vlsseg4e8.v v8, (a0), a1
+vsetvli zero, zero, e16, mf8, tu, mu
+vlsseg4e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, mf4, tu, mu
+vlsseg4e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, mf2, tu, mu
+vlsseg4e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m1, tu, mu
+vlsseg4e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m2, tu, mu
+vlsseg4e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m4, tu, mu
+vlsseg4e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m8, tu, mu
+vlsseg4e16.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf8, tu, mu
+vlsseg4e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf4, tu, mu
+vlsseg4e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf2, tu, mu
+vlsseg4e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m1, tu, mu
+vlsseg4e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m2, tu, mu
+vlsseg4e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m4, tu, mu
+vlsseg4e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m8, tu, mu
+vlsseg4e32.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf8, tu, mu
+vlsseg4e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf4, tu, mu
+vlsseg4e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf2, tu, mu
+vlsseg4e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m1, tu, mu
+vlsseg4e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m2, tu, mu
+vlsseg4e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m4, tu, mu
+vlsseg4e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m8, tu, mu
+vlsseg4e64.v  v8, (a0), a1
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vlsseg5e8.v v8, (a0), a1
+vsetvli zero, zero, e8, mf4, tu, mu
+vlsseg5e8.v v8, (a0), a1
+vsetvli zero, zero, e8, mf2, tu, mu
+vlsseg5e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m1, tu, mu
+vlsseg5e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m2, tu, mu
+vlsseg5e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m4, tu, mu
+vlsseg5e8.v v8, (a0), a1
+vsetvli zero, zero, e16, mf8, tu, mu
+vlsseg5e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, mf4, tu, mu
+vlsseg5e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, mf2, tu, mu
+vlsseg5e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m1, tu, mu
+vlsseg5e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m2, tu, mu
+vlsseg5e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m4, tu, mu
+vlsseg5e16.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf8, tu, mu
+vlsseg5e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf4, tu, mu
+vlsseg5e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf2, tu, mu
+vlsseg5e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m1, tu, mu
+vlsseg5e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m2, tu, mu
+vlsseg5e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m4, tu, mu
+vlsseg5e32.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf8, tu, mu
+vlsseg5e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf4, tu, mu
+vlsseg5e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf2, tu, mu
+vlsseg5e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m1, tu, mu
+vlsseg5e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m2, tu, mu
+vlsseg5e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m4, tu, mu
+vlsseg5e64.v  v8, (a0), a1
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vlsseg6e8.v v8, (a0), a1
+vsetvli zero, zero, e8, mf4, tu, mu
+vlsseg6e8.v v8, (a0), a1
+vsetvli zero, zero, e8, mf2, tu, mu
+vlsseg6e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m1, tu, mu
+vlsseg6e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m2, tu, mu
+vlsseg6e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m4, tu, mu
+vlsseg6e8.v v8, (a0), a1
+vsetvli zero, zero, e16, mf8, tu, mu
+vlsseg6e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, mf4, tu, mu
+vlsseg6e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, mf2, tu, mu
+vlsseg6e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m1, tu, mu
+vlsseg6e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m2, tu, mu
+vlsseg6e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m4, tu, mu
+vlsseg6e16.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf8, tu, mu
+vlsseg6e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf4, tu, mu
+vlsseg6e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf2, tu, mu
+vlsseg6e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m1, tu, mu
+vlsseg6e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m2, tu, mu
+vlsseg6e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m4, tu, mu
+vlsseg6e32.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf8, tu, mu
+vlsseg6e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf4, tu, mu
+vlsseg6e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf2, tu, mu
+vlsseg6e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m1, tu, mu
+vlsseg6e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m2, tu, mu
+vlsseg6e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m4, tu, mu
+vlsseg6e64.v  v8, (a0), a1
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vlsseg7e8.v v8, (a0), a1
+vsetvli zero, zero, e8, mf4, tu, mu
+vlsseg7e8.v v8, (a0), a1
+vsetvli zero, zero, e8, mf2, tu, mu
+vlsseg7e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m1, tu, mu
+vlsseg7e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m2, tu, mu
+vlsseg7e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m4, tu, mu
+vlsseg7e8.v v8, (a0), a1
+vsetvli zero, zero, e16, mf8, tu, mu
+vlsseg7e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, mf4, tu, mu
+vlsseg7e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, mf2, tu, mu
+vlsseg7e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m1, tu, mu
+vlsseg7e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m2, tu, mu
+vlsseg7e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m4, tu, mu
+vlsseg7e16.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf8, tu, mu
+vlsseg7e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf4, tu, mu
+vlsseg7e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf2, tu, mu
+vlsseg7e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m1, tu, mu
+vlsseg7e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m2, tu, mu
+vlsseg7e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m4, tu, mu
+vlsseg7e32.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf8, tu, mu
+vlsseg7e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf4, tu, mu
+vlsseg7e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf2, tu, mu
+vlsseg7e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m1, tu, mu
+vlsseg7e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m2, tu, mu
+vlsseg7e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m4, tu, mu
+vlsseg7e64.v  v8, (a0), a1
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vlsseg8e8.v v8, (a0), a1
+vsetvli zero, zero, e8, mf4, tu, mu
+vlsseg8e8.v v8, (a0), a1
+vsetvli zero, zero, e8, mf2, tu, mu
+vlsseg8e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m1, tu, mu
+vlsseg8e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m2, tu, mu
+vlsseg8e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m4, tu, mu
+vlsseg8e8.v v8, (a0), a1
+vsetvli zero, zero, e16, mf8, tu, mu
+vlsseg8e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, mf4, tu, mu
+vlsseg8e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, mf2, tu, mu
+vlsseg8e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m1, tu, mu
+vlsseg8e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m2, tu, mu
+vlsseg8e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m4, tu, mu
+vlsseg8e16.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf8, tu, mu
+vlsseg8e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf4, tu, mu
+vlsseg8e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf2, tu, mu
+vlsseg8e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m1, tu, mu
+vlsseg8e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m2, tu, mu
+vlsseg8e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m4, tu, mu
+vlsseg8e32.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf8, tu, mu
+vlsseg8e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf4, tu, mu
+vlsseg8e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf2, tu, mu
+vlsseg8e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m1, tu, mu
+vlsseg8e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m2, tu, mu
+vlsseg8e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m4, tu, mu
+vlsseg8e64.v  v8, (a0), a1
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vssseg2e8.v v8, (a0), a1
+vsetvli zero, zero, e8, mf4, tu, mu
+vssseg2e8.v v8, (a0), a1
+vsetvli zero, zero, e8, mf2, tu, mu
+vssseg2e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m1, tu, mu
+vssseg2e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m2, tu, mu
+vssseg2e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m4, tu, mu
+vssseg2e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m8, tu, mu
+vssseg2e8.v v8, (a0), a1
+vsetvli zero, zero, e16, mf8, tu, mu
+vssseg2e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, mf4, tu, mu
+vssseg2e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, mf2, tu, mu
+vssseg2e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m1, tu, mu
+vssseg2e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m2, tu, mu
+vssseg2e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m4, tu, mu
+vssseg2e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m8, tu, mu
+vssseg2e16.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf8, tu, mu
+vssseg2e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf4, tu, mu
+vssseg2e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf2, tu, mu
+vssseg2e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m1, tu, mu
+vssseg2e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m2, tu, mu
+vssseg2e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m4, tu, mu
+vssseg2e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m8, tu, mu
+vssseg2e32.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf8, tu, mu
+vssseg2e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf4, tu, mu
+vssseg2e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf2, tu, mu
+vssseg2e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m1, tu, mu
+vssseg2e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m2, tu, mu
+vssseg2e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m4, tu, mu
+vssseg2e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m8, tu, mu
+vssseg2e64.v  v8, (a0), a1
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vssseg3e8.v v8, (a0), a1
+vsetvli zero, zero, e8, mf4, tu, mu
+vssseg3e8.v v8, (a0), a1
+vsetvli zero, zero, e8, mf2, tu, mu
+vssseg3e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m1, tu, mu
+vssseg3e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m2, tu, mu
+vssseg3e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m4, tu, mu
+vssseg3e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m8, tu, mu
+vssseg3e8.v v8, (a0), a1
+vsetvli zero, zero, e16, mf8, tu, mu
+vssseg3e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, mf4, tu, mu
+vssseg3e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, mf2, tu, mu
+vssseg3e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m1, tu, mu
+vssseg3e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m2, tu, mu
+vssseg3e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m4, tu, mu
+vssseg3e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m8, tu, mu
+vssseg3e16.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf8, tu, mu
+vssseg3e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf4, tu, mu
+vssseg3e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf2, tu, mu
+vssseg3e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m1, tu, mu
+vssseg3e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m2, tu, mu
+vssseg3e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m4, tu, mu
+vssseg3e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m8, tu, mu
+vssseg3e32.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf8, tu, mu
+vssseg3e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf4, tu, mu
+vssseg3e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf2, tu, mu
+vssseg3e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m1, tu, mu
+vssseg3e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m2, tu, mu
+vssseg3e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m4, tu, mu
+vssseg3e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m8, tu, mu
+vssseg3e64.v  v8, (a0), a1
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vssseg4e8.v v8, (a0), a1
+vsetvli zero, zero, e8, mf4, tu, mu
+vssseg4e8.v v8, (a0), a1
+vsetvli zero, zero, e8, mf2, tu, mu
+vssseg4e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m1, tu, mu
+vssseg4e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m2, tu, mu
+vssseg4e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m4, tu, mu
+vssseg4e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m8, tu, mu
+vssseg4e8.v v8, (a0), a1
+vsetvli zero, zero, e16, mf8, tu, mu
+vssseg4e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, mf4, tu, mu
+vssseg4e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, mf2, tu, mu
+vssseg4e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m1, tu, mu
+vssseg4e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m2, tu, mu
+vssseg4e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m4, tu, mu
+vssseg4e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m8, tu, mu
+vssseg4e16.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf8, tu, mu
+vssseg4e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf4, tu, mu
+vssseg4e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf2, tu, mu
+vssseg4e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m1, tu, mu
+vssseg4e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m2, tu, mu
+vssseg4e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m4, tu, mu
+vssseg4e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m8, tu, mu
+vssseg4e32.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf8, tu, mu
+vssseg4e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf4, tu, mu
+vssseg4e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf2, tu, mu
+vssseg4e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m1, tu, mu
+vssseg4e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m2, tu, mu
+vssseg4e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m4, tu, mu
+vssseg4e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m8, tu, mu
+vssseg4e64.v  v8, (a0), a1
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vssseg5e8.v v8, (a0), a1
+vsetvli zero, zero, e8, mf4, tu, mu
+vssseg5e8.v v8, (a0), a1
+vsetvli zero, zero, e8, mf2, tu, mu
+vssseg5e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m1, tu, mu
+vssseg5e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m2, tu, mu
+vssseg5e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m4, tu, mu
+vssseg5e8.v v8, (a0), a1
+vsetvli zero, zero, e16, mf8, tu, mu
+vssseg5e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, mf4, tu, mu
+vssseg5e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, mf2, tu, mu
+vssseg5e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m1, tu, mu
+vssseg5e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m2, tu, mu
+vssseg5e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m4, tu, mu
+vssseg5e16.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf8, tu, mu
+vssseg5e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf4, tu, mu
+vssseg5e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf2, tu, mu
+vssseg5e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m1, tu, mu
+vssseg5e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m2, tu, mu
+vssseg5e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m4, tu, mu
+vssseg5e32.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf8, tu, mu
+vssseg5e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf4, tu, mu
+vssseg5e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf2, tu, mu
+vssseg5e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m1, tu, mu
+vssseg5e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m2, tu, mu
+vssseg5e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m4, tu, mu
+vssseg5e64.v  v8, (a0), a1
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vssseg6e8.v v8, (a0), a1
+vsetvli zero, zero, e8, mf4, tu, mu
+vssseg6e8.v v8, (a0), a1
+vsetvli zero, zero, e8, mf2, tu, mu
+vssseg6e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m1, tu, mu
+vssseg6e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m2, tu, mu
+vssseg6e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m4, tu, mu
+vssseg6e8.v v8, (a0), a1
+vsetvli zero, zero, e16, mf8, tu, mu
+vssseg6e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, mf4, tu, mu
+vssseg6e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, mf2, tu, mu
+vssseg6e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m1, tu, mu
+vssseg6e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m2, tu, mu
+vssseg6e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m4, tu, mu
+vssseg6e16.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf8, tu, mu
+vssseg6e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf4, tu, mu
+vssseg6e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf2, tu, mu
+vssseg6e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m1, tu, mu
+vssseg6e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m2, tu, mu
+vssseg6e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m4, tu, mu
+vssseg6e32.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf8, tu, mu
+vssseg6e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf4, tu, mu
+vssseg6e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf2, tu, mu
+vssseg6e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m1, tu, mu
+vssseg6e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m2, tu, mu
+vssseg6e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m4, tu, mu
+vssseg6e64.v  v8, (a0), a1
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vssseg7e8.v v8, (a0), a1
+vsetvli zero, zero, e8, mf4, tu, mu
+vssseg7e8.v v8, (a0), a1
+vsetvli zero, zero, e8, mf2, tu, mu
+vssseg7e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m1, tu, mu
+vssseg7e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m2, tu, mu
+vssseg7e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m4, tu, mu
+vssseg7e8.v v8, (a0), a1
+vsetvli zero, zero, e16, mf8, tu, mu
+vssseg7e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, mf4, tu, mu
+vssseg7e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, mf2, tu, mu
+vssseg7e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m1, tu, mu
+vssseg7e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m2, tu, mu
+vssseg7e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m4, tu, mu
+vssseg7e16.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf8, tu, mu
+vssseg7e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf4, tu, mu
+vssseg7e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf2, tu, mu
+vssseg7e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m1, tu, mu
+vssseg7e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m2, tu, mu
+vssseg7e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m4, tu, mu
+vssseg7e32.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf8, tu, mu
+vssseg7e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf4, tu, mu
+vssseg7e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf2, tu, mu
+vssseg7e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m1, tu, mu
+vssseg7e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m2, tu, mu
+vssseg7e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m4, tu, mu
+vssseg7e64.v  v8, (a0), a1
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vssseg8e8.v v8, (a0), a1
+vsetvli zero, zero, e8, mf4, tu, mu
+vssseg8e8.v v8, (a0), a1
+vsetvli zero, zero, e8, mf2, tu, mu
+vssseg8e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m1, tu, mu
+vssseg8e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m2, tu, mu
+vssseg8e8.v v8, (a0), a1
+vsetvli zero, zero, e8, m4, tu, mu
+vssseg8e8.v v8, (a0), a1
+vsetvli zero, zero, e16, mf8, tu, mu
+vssseg8e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, mf4, tu, mu
+vssseg8e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, mf2, tu, mu
+vssseg8e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m1, tu, mu
+vssseg8e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m2, tu, mu
+vssseg8e16.v  v8, (a0), a1
+vsetvli zero, zero, e16, m4, tu, mu
+vssseg8e16.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf8, tu, mu
+vssseg8e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf4, tu, mu
+vssseg8e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, mf2, tu, mu
+vssseg8e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m1, tu, mu
+vssseg8e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m2, tu, mu
+vssseg8e32.v  v8, (a0), a1
+vsetvli zero, zero, e32, m4, tu, mu
+vssseg8e32.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf8, tu, mu
+vssseg8e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf4, tu, mu
+vssseg8e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, mf2, tu, mu
+vssseg8e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m1, tu, mu
+vssseg8e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m2, tu, mu
+vssseg8e64.v  v8, (a0), a1
+vsetvli zero, zero, e64, m4, tu, mu
+vssseg8e64.v  v8, (a0), a1
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vlseg2e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, mf4, tu, mu
+vlseg2e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, mf2, tu, mu
+vlseg2e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, m1, tu, mu
+vlseg2e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, m2, tu, mu
+vlseg2e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, m4, tu, mu
+vlseg2e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, m8, tu, mu
+vlseg2e8ff.v  v8,(a0)
+vsetvli zero, zero, e16, mf8, tu, mu
+vlseg2e16ff.v v8,(a0)
+vsetvli zero, zero, e16, mf4, tu, mu
+vlseg2e16ff.v v8,(a0)
+vsetvli zero, zero, e16, mf2, tu, mu
+vlseg2e16ff.v v8,(a0)
+vsetvli zero, zero, e16, m1, tu, mu
+vlseg2e16ff.v v8,(a0)
+vsetvli zero, zero, e16, m2, tu, mu
+vlseg2e16ff.v v8,(a0)
+vsetvli zero, zero, e16, m4, tu, mu
+vlseg2e16ff.v v8,(a0)
+vsetvli zero, zero, e16, m8, tu, mu
+vlseg2e16ff.v v8,(a0)
+vsetvli zero, zero, e32, mf8, tu, mu
+vlseg2e32ff.v v8,(a0)
+vsetvli zero, zero, e32, mf4, tu, mu
+vlseg2e32ff.v v8,(a0)
+vsetvli zero, zero, e32, mf2, tu, mu
+vlseg2e32ff.v v8,(a0)
+vsetvli zero, zero, e32, m1, tu, mu
+vlseg2e32ff.v v8,(a0)
+vsetvli zero, zero, e32, m2, tu, mu
+vlseg2e32ff.v v8,(a0)
+vsetvli zero, zero, e32, m4, tu, mu
+vlseg2e32ff.v v8,(a0)
+vsetvli zero, zero, e32, m8, tu, mu
+vlseg2e32ff.v v8,(a0)
+vsetvli zero, zero, e64, mf8, tu, mu
+vlseg2e64ff.v v8,(a0)
+vsetvli zero, zero, e64, mf4, tu, mu
+vlseg2e64ff.v v8,(a0)
+vsetvli zero, zero, e64, mf2, tu, mu
+vlseg2e64ff.v v8,(a0)
+vsetvli zero, zero, e64, m1, tu, mu
+vlseg2e64ff.v v8,(a0)
+vsetvli zero, zero, e64, m2, tu, mu
+vlseg2e64ff.v v8,(a0)
+vsetvli zero, zero, e64, m4, tu, mu
+vlseg2e64ff.v v8,(a0)
+vsetvli zero, zero, e64, m8, tu, mu
+vlseg2e64ff.v v8,(a0)
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vlseg3e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, mf4, tu, mu
+vlseg3e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, mf2, tu, mu
+vlseg3e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, m1, tu, mu
+vlseg3e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, m2, tu, mu
+vlseg3e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, m4, tu, mu
+vlseg3e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, m8, tu, mu
+vlseg3e8ff.v  v8,(a0)
+vsetvli zero, zero, e16, mf8, tu, mu
+vlseg3e16ff.v v8,(a0)
+vsetvli zero, zero, e16, mf4, tu, mu
+vlseg3e16ff.v v8,(a0)
+vsetvli zero, zero, e16, mf2, tu, mu
+vlseg3e16ff.v v8,(a0)
+vsetvli zero, zero, e16, m1, tu, mu
+vlseg3e16ff.v v8,(a0)
+vsetvli zero, zero, e16, m2, tu, mu
+vlseg3e16ff.v v8,(a0)
+vsetvli zero, zero, e16, m4, tu, mu
+vlseg3e16ff.v v8,(a0)
+vsetvli zero, zero, e16, m8, tu, mu
+vlseg3e16ff.v v8,(a0)
+vsetvli zero, zero, e32, mf8, tu, mu
+vlseg3e32ff.v v8,(a0)
+vsetvli zero, zero, e32, mf4, tu, mu
+vlseg3e32ff.v v8,(a0)
+vsetvli zero, zero, e32, mf2, tu, mu
+vlseg3e32ff.v v8,(a0)
+vsetvli zero, zero, e32, m1, tu, mu
+vlseg3e32ff.v v8,(a0)
+vsetvli zero, zero, e32, m2, tu, mu
+vlseg3e32ff.v v8,(a0)
+vsetvli zero, zero, e32, m4, tu, mu
+vlseg3e32ff.v v8,(a0)
+vsetvli zero, zero, e32, m8, tu, mu
+vlseg3e32ff.v v8,(a0)
+vsetvli zero, zero, e64, mf8, tu, mu
+vlseg3e64ff.v v8,(a0)
+vsetvli zero, zero, e64, mf4, tu, mu
+vlseg3e64ff.v v8,(a0)
+vsetvli zero, zero, e64, mf2, tu, mu
+vlseg3e64ff.v v8,(a0)
+vsetvli zero, zero, e64, m1, tu, mu
+vlseg3e64ff.v v8,(a0)
+vsetvli zero, zero, e64, m2, tu, mu
+vlseg3e64ff.v v8,(a0)
+vsetvli zero, zero, e64, m4, tu, mu
+vlseg3e64ff.v v8,(a0)
+vsetvli zero, zero, e64, m8, tu, mu
+vlseg3e64ff.v v8,(a0)
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vlseg4e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, mf4, tu, mu
+vlseg4e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, mf2, tu, mu
+vlseg4e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, m1, tu, mu
+vlseg4e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, m2, tu, mu
+vlseg4e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, m4, tu, mu
+vlseg4e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, m8, tu, mu
+vlseg4e8ff.v  v8,(a0)
+vsetvli zero, zero, e16, mf8, tu, mu
+vlseg4e16ff.v v8,(a0)
+vsetvli zero, zero, e16, mf4, tu, mu
+vlseg4e16ff.v v8,(a0)
+vsetvli zero, zero, e16, mf2, tu, mu
+vlseg4e16ff.v v8,(a0)
+vsetvli zero, zero, e16, m1, tu, mu
+vlseg4e16ff.v v8,(a0)
+vsetvli zero, zero, e16, m2, tu, mu
+vlseg4e16ff.v v8,(a0)
+vsetvli zero, zero, e16, m4, tu, mu
+vlseg4e16ff.v v8,(a0)
+vsetvli zero, zero, e16, m8, tu, mu
+vlseg4e16ff.v v8,(a0)
+vsetvli zero, zero, e32, mf8, tu, mu
+vlseg4e32ff.v v8,(a0)
+vsetvli zero, zero, e32, mf4, tu, mu
+vlseg4e32ff.v v8,(a0)
+vsetvli zero, zero, e32, mf2, tu, mu
+vlseg4e32ff.v v8,(a0)
+vsetvli zero, zero, e32, m1, tu, mu
+vlseg4e32ff.v v8,(a0)
+vsetvli zero, zero, e32, m2, tu, mu
+vlseg4e32ff.v v8,(a0)
+vsetvli zero, zero, e32, m4, tu, mu
+vlseg4e32ff.v v8,(a0)
+vsetvli zero, zero, e32, m8, tu, mu
+vlseg4e32ff.v v8,(a0)
+vsetvli zero, zero, e64, mf8, tu, mu
+vlseg4e64ff.v v8,(a0)
+vsetvli zero, zero, e64, mf4, tu, mu
+vlseg4e64ff.v v8,(a0)
+vsetvli zero, zero, e64, mf2, tu, mu
+vlseg4e64ff.v v8,(a0)
+vsetvli zero, zero, e64, m1, tu, mu
+vlseg4e64ff.v v8,(a0)
+vsetvli zero, zero, e64, m2, tu, mu
+vlseg4e64ff.v v8,(a0)
+vsetvli zero, zero, e64, m4, tu, mu
+vlseg4e64ff.v v8,(a0)
+vsetvli zero, zero, e64, m8, tu, mu
+vlseg4e64ff.v v8,(a0)
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vlseg5e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, mf4, tu, mu
+vlseg5e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, mf2, tu, mu
+vlseg5e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, m1, tu, mu
+vlseg5e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, m2, tu, mu
+vlseg5e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, m4, tu, mu
+vlseg5e8ff.v  v8,(a0)
+vsetvli zero, zero, e16, mf8, tu, mu
+vlseg5e16ff.v v8,(a0)
+vsetvli zero, zero, e16, mf4, tu, mu
+vlseg5e16ff.v v8,(a0)
+vsetvli zero, zero, e16, mf2, tu, mu
+vlseg5e16ff.v v8,(a0)
+vsetvli zero, zero, e16, m1, tu, mu
+vlseg5e16ff.v v8,(a0)
+vsetvli zero, zero, e16, m2, tu, mu
+vlseg5e16ff.v v8,(a0)
+vsetvli zero, zero, e16, m4, tu, mu
+vlseg5e16ff.v v8,(a0)
+vsetvli zero, zero, e32, mf8, tu, mu
+vlseg5e32ff.v v8,(a0)
+vsetvli zero, zero, e32, mf4, tu, mu
+vlseg5e32ff.v v8,(a0)
+vsetvli zero, zero, e32, mf2, tu, mu
+vlseg5e32ff.v v8,(a0)
+vsetvli zero, zero, e32, m1, tu, mu
+vlseg5e32ff.v v8,(a0)
+vsetvli zero, zero, e32, m2, tu, mu
+vlseg5e32ff.v v8,(a0)
+vsetvli zero, zero, e32, m4, tu, mu
+vlseg5e32ff.v v8,(a0)
+vsetvli zero, zero, e64, mf8, tu, mu
+vlseg5e64ff.v v8,(a0)
+vsetvli zero, zero, e64, mf4, tu, mu
+vlseg5e64ff.v v8,(a0)
+vsetvli zero, zero, e64, mf2, tu, mu
+vlseg5e64ff.v v8,(a0)
+vsetvli zero, zero, e64, m1, tu, mu
+vlseg5e64ff.v v8,(a0)
+vsetvli zero, zero, e64, m2, tu, mu
+vlseg5e64ff.v v8,(a0)
+vsetvli zero, zero, e64, m4, tu, mu
+vlseg5e64ff.v v8,(a0)
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vlseg6e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, mf4, tu, mu
+vlseg6e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, mf2, tu, mu
+vlseg6e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, m1, tu, mu
+vlseg6e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, m2, tu, mu
+vlseg6e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, m4, tu, mu
+vlseg6e8ff.v  v8,(a0)
+vsetvli zero, zero, e16, mf8, tu, mu
+vlseg6e16ff.v v8,(a0)
+vsetvli zero, zero, e16, mf4, tu, mu
+vlseg6e16ff.v v8,(a0)
+vsetvli zero, zero, e16, mf2, tu, mu
+vlseg6e16ff.v v8,(a0)
+vsetvli zero, zero, e16, m1, tu, mu
+vlseg6e16ff.v v8,(a0)
+vsetvli zero, zero, e16, m2, tu, mu
+vlseg6e16ff.v v8,(a0)
+vsetvli zero, zero, e16, m4, tu, mu
+vlseg6e16ff.v v8,(a0)
+vsetvli zero, zero, e32, mf8, tu, mu
+vlseg6e32ff.v v8,(a0)
+vsetvli zero, zero, e32, mf4, tu, mu
+vlseg6e32ff.v v8,(a0)
+vsetvli zero, zero, e32, mf2, tu, mu
+vlseg6e32ff.v v8,(a0)
+vsetvli zero, zero, e32, m1, tu, mu
+vlseg6e32ff.v v8,(a0)
+vsetvli zero, zero, e32, m2, tu, mu
+vlseg6e32ff.v v8,(a0)
+vsetvli zero, zero, e32, m4, tu, mu
+vlseg6e32ff.v v8,(a0)
+vsetvli zero, zero, e64, mf8, tu, mu
+vlseg6e64ff.v v8,(a0)
+vsetvli zero, zero, e64, mf4, tu, mu
+vlseg6e64ff.v v8,(a0)
+vsetvli zero, zero, e64, mf2, tu, mu
+vlseg6e64ff.v v8,(a0)
+vsetvli zero, zero, e64, m1, tu, mu
+vlseg6e64ff.v v8,(a0)
+vsetvli zero, zero, e64, m2, tu, mu
+vlseg6e64ff.v v8,(a0)
+vsetvli zero, zero, e64, m4, tu, mu
+vlseg6e64ff.v v8,(a0)
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vlseg7e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, mf4, tu, mu
+vlseg7e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, mf2, tu, mu
+vlseg7e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, m1, tu, mu
+vlseg7e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, m2, tu, mu
+vlseg7e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, m4, tu, mu
+vlseg7e8ff.v  v8,(a0)
+vsetvli zero, zero, e16, mf8, tu, mu
+vlseg7e16ff.v v8,(a0)
+vsetvli zero, zero, e16, mf4, tu, mu
+vlseg7e16ff.v v8,(a0)
+vsetvli zero, zero, e16, mf2, tu, mu
+vlseg7e16ff.v v8,(a0)
+vsetvli zero, zero, e16, m1, tu, mu
+vlseg7e16ff.v v8,(a0)
+vsetvli zero, zero, e16, m2, tu, mu
+vlseg7e16ff.v v8,(a0)
+vsetvli zero, zero, e16, m4, tu, mu
+vlseg7e16ff.v v8,(a0)
+vsetvli zero, zero, e32, mf8, tu, mu
+vlseg7e32ff.v v8,(a0)
+vsetvli zero, zero, e32, mf4, tu, mu
+vlseg7e32ff.v v8,(a0)
+vsetvli zero, zero, e32, mf2, tu, mu
+vlseg7e32ff.v v8,(a0)
+vsetvli zero, zero, e32, m1, tu, mu
+vlseg7e32ff.v v8,(a0)
+vsetvli zero, zero, e32, m2, tu, mu
+vlseg7e32ff.v v8,(a0)
+vsetvli zero, zero, e32, m4, tu, mu
+vlseg7e32ff.v v8,(a0)
+vsetvli zero, zero, e64, mf8, tu, mu
+vlseg7e64ff.v v8,(a0)
+vsetvli zero, zero, e64, mf4, tu, mu
+vlseg7e64ff.v v8,(a0)
+vsetvli zero, zero, e64, mf2, tu, mu
+vlseg7e64ff.v v8,(a0)
+vsetvli zero, zero, e64, m1, tu, mu
+vlseg7e64ff.v v8,(a0)
+vsetvli zero, zero, e64, m2, tu, mu
+vlseg7e64ff.v v8,(a0)
+vsetvli zero, zero, e64, m4, tu, mu
+vlseg7e64ff.v v8,(a0)
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vlseg8e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, mf4, tu, mu
+vlseg8e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, mf2, tu, mu
+vlseg8e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, m1, tu, mu
+vlseg8e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, m2, tu, mu
+vlseg8e8ff.v  v8,(a0)
+vsetvli zero, zero, e8, m4, tu, mu
+vlseg8e8ff.v  v8,(a0)
+vsetvli zero, zero, e16, mf8, tu, mu
+vlseg8e16ff.v v8,(a0)
+vsetvli zero, zero, e16, mf4, tu, mu
+vlseg8e16ff.v v8,(a0)
+vsetvli zero, zero, e16, mf2, tu, mu
+vlseg8e16ff.v v8,(a0)
+vsetvli zero, zero, e16, m1, tu, mu
+vlseg8e16ff.v v8,(a0)
+vsetvli zero, zero, e16, m2, tu, mu
+vlseg8e16ff.v v8,(a0)
+vsetvli zero, zero, e16, m4, tu, mu
+vlseg8e16ff.v v8,(a0)
+vsetvli zero, zero, e32, mf8, tu, mu
+vlseg8e32ff.v v8,(a0)
+vsetvli zero, zero, e32, mf4, tu, mu
+vlseg8e32ff.v v8,(a0)
+vsetvli zero, zero, e32, mf2, tu, mu
+vlseg8e32ff.v v8,(a0)
+vsetvli zero, zero, e32, m1, tu, mu
+vlseg8e32ff.v v8,(a0)
+vsetvli zero, zero, e32, m2, tu, mu
+vlseg8e32ff.v v8,(a0)
+vsetvli zero, zero, e32, m4, tu, mu
+vlseg8e32ff.v v8,(a0)
+vsetvli zero, zero, e64, mf8, tu, mu
+vlseg8e64ff.v v8,(a0)
+vsetvli zero, zero, e64, mf4, tu, mu
+vlseg8e64ff.v v8,(a0)
+vsetvli zero, zero, e64, mf2, tu, mu
+vlseg8e64ff.v v8,(a0)
+vsetvli zero, zero, e64, m1, tu, mu
+vlseg8e64ff.v v8,(a0)
+vsetvli zero, zero, e64, m2, tu, mu
+vlseg8e64ff.v v8,(a0)
+vsetvli zero, zero, e64, m4, tu, mu
+vlseg8e64ff.v v8,(a0)
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vluxseg2ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vluxseg2ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vluxseg2ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m1, tu, mu
+vluxseg2ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m2, tu, mu
+vluxseg2ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m4, tu, mu
+vluxseg2ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m8, tu, mu
+vluxseg2ei8.v v8, (a0), v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vluxseg2ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vluxseg2ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vluxseg2ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m1, tu, mu
+vluxseg2ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m2, tu, mu
+vluxseg2ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m4, tu, mu
+vluxseg2ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m8, tu, mu
+vluxseg2ei16.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vluxseg2ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vluxseg2ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vluxseg2ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m1, tu, mu
+vluxseg2ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m2, tu, mu
+vluxseg2ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m4, tu, mu
+vluxseg2ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m8, tu, mu
+vluxseg2ei32.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vluxseg2ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vluxseg2ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vluxseg2ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m1, tu, mu
+vluxseg2ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m2, tu, mu
+vluxseg2ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m4, tu, mu
+vluxseg2ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m8, tu, mu
+vluxseg2ei64.v  v8, (a0), v16
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vluxseg3ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vluxseg3ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vluxseg3ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m1, tu, mu
+vluxseg3ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m2, tu, mu
+vluxseg3ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m4, tu, mu
+vluxseg3ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m8, tu, mu
+vluxseg3ei8.v v8, (a0), v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vluxseg3ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vluxseg3ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vluxseg3ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m1, tu, mu
+vluxseg3ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m2, tu, mu
+vluxseg3ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m4, tu, mu
+vluxseg3ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m8, tu, mu
+vluxseg3ei16.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vluxseg3ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vluxseg3ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vluxseg3ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m1, tu, mu
+vluxseg3ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m2, tu, mu
+vluxseg3ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m4, tu, mu
+vluxseg3ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m8, tu, mu
+vluxseg3ei32.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vluxseg3ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vluxseg3ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vluxseg3ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m1, tu, mu
+vluxseg3ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m2, tu, mu
+vluxseg3ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m4, tu, mu
+vluxseg3ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m8, tu, mu
+vluxseg3ei64.v  v8, (a0), v16
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vluxseg4ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vluxseg4ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vluxseg4ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m1, tu, mu
+vluxseg4ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m2, tu, mu
+vluxseg4ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m4, tu, mu
+vluxseg4ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m8, tu, mu
+vluxseg4ei8.v v8, (a0), v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vluxseg4ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vluxseg4ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vluxseg4ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m1, tu, mu
+vluxseg4ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m2, tu, mu
+vluxseg4ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m4, tu, mu
+vluxseg4ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m8, tu, mu
+vluxseg4ei16.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vluxseg4ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vluxseg4ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vluxseg4ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m1, tu, mu
+vluxseg4ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m2, tu, mu
+vluxseg4ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m4, tu, mu
+vluxseg4ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m8, tu, mu
+vluxseg4ei32.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vluxseg4ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vluxseg4ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vluxseg4ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m1, tu, mu
+vluxseg4ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m2, tu, mu
+vluxseg4ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m4, tu, mu
+vluxseg4ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m8, tu, mu
+vluxseg4ei64.v  v8, (a0), v16
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vluxseg5ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vluxseg5ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vluxseg5ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m1, tu, mu
+vluxseg5ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m2, tu, mu
+vluxseg5ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m4, tu, mu
+vluxseg5ei8.v v8, (a0), v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vluxseg5ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vluxseg5ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vluxseg5ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m1, tu, mu
+vluxseg5ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m2, tu, mu
+vluxseg5ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m4, tu, mu
+vluxseg5ei16.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vluxseg5ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vluxseg5ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vluxseg5ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m1, tu, mu
+vluxseg5ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m2, tu, mu
+vluxseg5ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m4, tu, mu
+vluxseg5ei32.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vluxseg5ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vluxseg5ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vluxseg5ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m1, tu, mu
+vluxseg5ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m2, tu, mu
+vluxseg5ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m4, tu, mu
+vluxseg5ei64.v  v8, (a0), v16
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vluxseg6ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vluxseg6ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vluxseg6ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m1, tu, mu
+vluxseg6ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m2, tu, mu
+vluxseg6ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m4, tu, mu
+vluxseg6ei8.v v8, (a0), v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vluxseg6ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vluxseg6ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vluxseg6ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m1, tu, mu
+vluxseg6ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m2, tu, mu
+vluxseg6ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m4, tu, mu
+vluxseg6ei16.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vluxseg6ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vluxseg6ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vluxseg6ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m1, tu, mu
+vluxseg6ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m2, tu, mu
+vluxseg6ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m4, tu, mu
+vluxseg6ei32.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vluxseg6ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vluxseg6ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vluxseg6ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m1, tu, mu
+vluxseg6ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m2, tu, mu
+vluxseg6ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m4, tu, mu
+vluxseg6ei64.v  v8, (a0), v16
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vluxseg7ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vluxseg7ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vluxseg7ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m1, tu, mu
+vluxseg7ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m2, tu, mu
+vluxseg7ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m4, tu, mu
+vluxseg7ei8.v v8, (a0), v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vluxseg7ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vluxseg7ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vluxseg7ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m1, tu, mu
+vluxseg7ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m2, tu, mu
+vluxseg7ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m4, tu, mu
+vluxseg7ei16.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vluxseg7ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vluxseg7ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vluxseg7ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m1, tu, mu
+vluxseg7ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m2, tu, mu
+vluxseg7ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m4, tu, mu
+vluxseg7ei32.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vluxseg7ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vluxseg7ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vluxseg7ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m1, tu, mu
+vluxseg7ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m2, tu, mu
+vluxseg7ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m4, tu, mu
+vluxseg7ei64.v  v8, (a0), v16
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vluxseg8ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vluxseg8ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vluxseg8ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m1, tu, mu
+vluxseg8ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m2, tu, mu
+vluxseg8ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m4, tu, mu
+vluxseg8ei8.v v8, (a0), v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vluxseg8ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vluxseg8ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vluxseg8ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m1, tu, mu
+vluxseg8ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m2, tu, mu
+vluxseg8ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m4, tu, mu
+vluxseg8ei16.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vluxseg8ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vluxseg8ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vluxseg8ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m1, tu, mu
+vluxseg8ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m2, tu, mu
+vluxseg8ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m4, tu, mu
+vluxseg8ei32.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vluxseg8ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vluxseg8ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vluxseg8ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m1, tu, mu
+vluxseg8ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m2, tu, mu
+vluxseg8ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m4, tu, mu
+vluxseg8ei64.v  v8, (a0), v16
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vloxseg2ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vloxseg2ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vloxseg2ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m1, tu, mu
+vloxseg2ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m2, tu, mu
+vloxseg2ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m4, tu, mu
+vloxseg2ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m8, tu, mu
+vloxseg2ei8.v v8, (a0), v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vloxseg2ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vloxseg2ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vloxseg2ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m1, tu, mu
+vloxseg2ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m2, tu, mu
+vloxseg2ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m4, tu, mu
+vloxseg2ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m8, tu, mu
+vloxseg2ei16.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vloxseg2ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vloxseg2ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vloxseg2ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m1, tu, mu
+vloxseg2ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m2, tu, mu
+vloxseg2ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m4, tu, mu
+vloxseg2ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m8, tu, mu
+vloxseg2ei32.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vloxseg2ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vloxseg2ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vloxseg2ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m1, tu, mu
+vloxseg2ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m2, tu, mu
+vloxseg2ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m4, tu, mu
+vloxseg2ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m8, tu, mu
+vloxseg2ei64.v  v8, (a0), v16
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vloxseg3ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vloxseg3ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vloxseg3ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m1, tu, mu
+vloxseg3ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m2, tu, mu
+vloxseg3ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m4, tu, mu
+vloxseg3ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m8, tu, mu
+vloxseg3ei8.v v8, (a0), v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vloxseg3ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vloxseg3ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vloxseg3ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m1, tu, mu
+vloxseg3ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m2, tu, mu
+vloxseg3ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m4, tu, mu
+vloxseg3ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m8, tu, mu
+vloxseg3ei16.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vloxseg3ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vloxseg3ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vloxseg3ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m1, tu, mu
+vloxseg3ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m2, tu, mu
+vloxseg3ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m4, tu, mu
+vloxseg3ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m8, tu, mu
+vloxseg3ei32.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vloxseg3ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vloxseg3ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vloxseg3ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m1, tu, mu
+vloxseg3ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m2, tu, mu
+vloxseg3ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m4, tu, mu
+vloxseg3ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m8, tu, mu
+vloxseg3ei64.v  v8, (a0), v16
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vloxseg4ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vloxseg4ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vloxseg4ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m1, tu, mu
+vloxseg4ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m2, tu, mu
+vloxseg4ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m4, tu, mu
+vloxseg4ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m8, tu, mu
+vloxseg4ei8.v v8, (a0), v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vloxseg4ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vloxseg4ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vloxseg4ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m1, tu, mu
+vloxseg4ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m2, tu, mu
+vloxseg4ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m4, tu, mu
+vloxseg4ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m8, tu, mu
+vloxseg4ei16.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vloxseg4ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vloxseg4ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vloxseg4ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m1, tu, mu
+vloxseg4ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m2, tu, mu
+vloxseg4ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m4, tu, mu
+vloxseg4ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m8, tu, mu
+vloxseg4ei32.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vloxseg4ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vloxseg4ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vloxseg4ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m1, tu, mu
+vloxseg4ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m2, tu, mu
+vloxseg4ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m4, tu, mu
+vloxseg4ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m8, tu, mu
+vloxseg4ei64.v  v8, (a0), v16
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vloxseg5ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vloxseg5ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vloxseg5ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m1, tu, mu
+vloxseg5ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m2, tu, mu
+vloxseg5ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m4, tu, mu
+vloxseg5ei8.v v8, (a0), v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vloxseg5ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vloxseg5ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vloxseg5ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m1, tu, mu
+vloxseg5ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m2, tu, mu
+vloxseg5ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m4, tu, mu
+vloxseg5ei16.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vloxseg5ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vloxseg5ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vloxseg5ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m1, tu, mu
+vloxseg5ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m2, tu, mu
+vloxseg5ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m4, tu, mu
+vloxseg5ei32.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vloxseg5ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vloxseg5ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vloxseg5ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m1, tu, mu
+vloxseg5ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m2, tu, mu
+vloxseg5ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m4, tu, mu
+vloxseg5ei64.v  v8, (a0), v16
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vloxseg6ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vloxseg6ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vloxseg6ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m1, tu, mu
+vloxseg6ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m2, tu, mu
+vloxseg6ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m4, tu, mu
+vloxseg6ei8.v v8, (a0), v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vloxseg6ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vloxseg6ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vloxseg6ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m1, tu, mu
+vloxseg6ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m2, tu, mu
+vloxseg6ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m4, tu, mu
+vloxseg6ei16.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vloxseg6ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vloxseg6ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vloxseg6ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m1, tu, mu
+vloxseg6ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m2, tu, mu
+vloxseg6ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m4, tu, mu
+vloxseg6ei32.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vloxseg6ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vloxseg6ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vloxseg6ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m1, tu, mu
+vloxseg6ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m2, tu, mu
+vloxseg6ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m4, tu, mu
+vloxseg6ei64.v  v8, (a0), v16
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vloxseg7ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vloxseg7ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vloxseg7ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m1, tu, mu
+vloxseg7ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m2, tu, mu
+vloxseg7ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m4, tu, mu
+vloxseg7ei8.v v8, (a0), v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vloxseg7ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vloxseg7ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vloxseg7ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m1, tu, mu
+vloxseg7ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m2, tu, mu
+vloxseg7ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m4, tu, mu
+vloxseg7ei16.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vloxseg7ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vloxseg7ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vloxseg7ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m1, tu, mu
+vloxseg7ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m2, tu, mu
+vloxseg7ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m4, tu, mu
+vloxseg7ei32.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vloxseg7ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vloxseg7ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vloxseg7ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m1, tu, mu
+vloxseg7ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m2, tu, mu
+vloxseg7ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m4, tu, mu
+vloxseg7ei64.v  v8, (a0), v16
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vloxseg8ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vloxseg8ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vloxseg8ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m1, tu, mu
+vloxseg8ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m2, tu, mu
+vloxseg8ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m4, tu, mu
+vloxseg8ei8.v v8, (a0), v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vloxseg8ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vloxseg8ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vloxseg8ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m1, tu, mu
+vloxseg8ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m2, tu, mu
+vloxseg8ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m4, tu, mu
+vloxseg8ei16.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vloxseg8ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vloxseg8ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vloxseg8ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m1, tu, mu
+vloxseg8ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m2, tu, mu
+vloxseg8ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m4, tu, mu
+vloxseg8ei32.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vloxseg8ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vloxseg8ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vloxseg8ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m1, tu, mu
+vloxseg8ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m2, tu, mu
+vloxseg8ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m4, tu, mu
+vloxseg8ei64.v  v8, (a0), v16
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vsuxseg2ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vsuxseg2ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vsuxseg2ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m1, tu, mu
+vsuxseg2ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m2, tu, mu
+vsuxseg2ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m4, tu, mu
+vsuxseg2ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m8, tu, mu
+vsuxseg2ei8.v v8, (a0), v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vsuxseg2ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vsuxseg2ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vsuxseg2ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m1, tu, mu
+vsuxseg2ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m2, tu, mu
+vsuxseg2ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m4, tu, mu
+vsuxseg2ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m8, tu, mu
+vsuxseg2ei16.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vsuxseg2ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vsuxseg2ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vsuxseg2ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m1, tu, mu
+vsuxseg2ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m2, tu, mu
+vsuxseg2ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m4, tu, mu
+vsuxseg2ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m8, tu, mu
+vsuxseg2ei32.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vsuxseg2ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vsuxseg2ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vsuxseg2ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m1, tu, mu
+vsuxseg2ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m2, tu, mu
+vsuxseg2ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m4, tu, mu
+vsuxseg2ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m8, tu, mu
+vsuxseg2ei64.v  v8, (a0), v16
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vsuxseg3ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vsuxseg3ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vsuxseg3ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m1, tu, mu
+vsuxseg3ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m2, tu, mu
+vsuxseg3ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m4, tu, mu
+vsuxseg3ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m8, tu, mu
+vsuxseg3ei8.v v8, (a0), v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vsuxseg3ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vsuxseg3ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vsuxseg3ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m1, tu, mu
+vsuxseg3ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m2, tu, mu
+vsuxseg3ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m4, tu, mu
+vsuxseg3ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m8, tu, mu
+vsuxseg3ei16.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vsuxseg3ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vsuxseg3ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vsuxseg3ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m1, tu, mu
+vsuxseg3ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m2, tu, mu
+vsuxseg3ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m4, tu, mu
+vsuxseg3ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m8, tu, mu
+vsuxseg3ei32.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vsuxseg3ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vsuxseg3ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vsuxseg3ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m1, tu, mu
+vsuxseg3ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m2, tu, mu
+vsuxseg3ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m4, tu, mu
+vsuxseg3ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m8, tu, mu
+vsuxseg3ei64.v  v8, (a0), v16
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vsuxseg4ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vsuxseg4ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vsuxseg4ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m1, tu, mu
+vsuxseg4ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m2, tu, mu
+vsuxseg4ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m4, tu, mu
+vsuxseg4ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m8, tu, mu
+vsuxseg4ei8.v v8, (a0), v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vsuxseg4ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vsuxseg4ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vsuxseg4ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m1, tu, mu
+vsuxseg4ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m2, tu, mu
+vsuxseg4ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m4, tu, mu
+vsuxseg4ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m8, tu, mu
+vsuxseg4ei16.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vsuxseg4ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vsuxseg4ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vsuxseg4ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m1, tu, mu
+vsuxseg4ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m2, tu, mu
+vsuxseg4ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m4, tu, mu
+vsuxseg4ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m8, tu, mu
+vsuxseg4ei32.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vsuxseg4ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vsuxseg4ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vsuxseg4ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m1, tu, mu
+vsuxseg4ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m2, tu, mu
+vsuxseg4ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m4, tu, mu
+vsuxseg4ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m8, tu, mu
+vsuxseg4ei64.v  v8, (a0), v16
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vsuxseg5ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vsuxseg5ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vsuxseg5ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m1, tu, mu
+vsuxseg5ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m2, tu, mu
+vsuxseg5ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m4, tu, mu
+vsuxseg5ei8.v v8, (a0), v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vsuxseg5ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vsuxseg5ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vsuxseg5ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m1, tu, mu
+vsuxseg5ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m2, tu, mu
+vsuxseg5ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m4, tu, mu
+vsuxseg5ei16.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vsuxseg5ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vsuxseg5ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vsuxseg5ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m1, tu, mu
+vsuxseg5ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m2, tu, mu
+vsuxseg5ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m4, tu, mu
+vsuxseg5ei32.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vsuxseg5ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vsuxseg5ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vsuxseg5ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m1, tu, mu
+vsuxseg5ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m2, tu, mu
+vsuxseg5ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m4, tu, mu
+vsuxseg5ei64.v  v8, (a0), v16
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vsuxseg6ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vsuxseg6ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vsuxseg6ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m1, tu, mu
+vsuxseg6ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m2, tu, mu
+vsuxseg6ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m4, tu, mu
+vsuxseg6ei8.v v8, (a0), v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vsuxseg6ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vsuxseg6ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vsuxseg6ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m1, tu, mu
+vsuxseg6ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m2, tu, mu
+vsuxseg6ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m4, tu, mu
+vsuxseg6ei16.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vsuxseg6ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vsuxseg6ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vsuxseg6ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m1, tu, mu
+vsuxseg6ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m2, tu, mu
+vsuxseg6ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m4, tu, mu
+vsuxseg6ei32.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vsuxseg6ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vsuxseg6ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vsuxseg6ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m1, tu, mu
+vsuxseg6ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m2, tu, mu
+vsuxseg6ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m4, tu, mu
+vsuxseg6ei64.v  v8, (a0), v16
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vsuxseg7ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vsuxseg7ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vsuxseg7ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m1, tu, mu
+vsuxseg7ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m2, tu, mu
+vsuxseg7ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m4, tu, mu
+vsuxseg7ei8.v v8, (a0), v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vsuxseg7ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vsuxseg7ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vsuxseg7ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m1, tu, mu
+vsuxseg7ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m2, tu, mu
+vsuxseg7ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m4, tu, mu
+vsuxseg7ei16.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vsuxseg7ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vsuxseg7ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vsuxseg7ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m1, tu, mu
+vsuxseg7ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m2, tu, mu
+vsuxseg7ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m4, tu, mu
+vsuxseg7ei32.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vsuxseg7ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vsuxseg7ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vsuxseg7ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m1, tu, mu
+vsuxseg7ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m2, tu, mu
+vsuxseg7ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m4, tu, mu
+vsuxseg7ei64.v  v8, (a0), v16
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vsuxseg8ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vsuxseg8ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vsuxseg8ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m1, tu, mu
+vsuxseg8ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m2, tu, mu
+vsuxseg8ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m4, tu, mu
+vsuxseg8ei8.v v8, (a0), v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vsuxseg8ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vsuxseg8ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vsuxseg8ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m1, tu, mu
+vsuxseg8ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m2, tu, mu
+vsuxseg8ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m4, tu, mu
+vsuxseg8ei16.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vsuxseg8ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vsuxseg8ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vsuxseg8ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m1, tu, mu
+vsuxseg8ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m2, tu, mu
+vsuxseg8ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m4, tu, mu
+vsuxseg8ei32.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vsuxseg8ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vsuxseg8ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vsuxseg8ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m1, tu, mu
+vsuxseg8ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m2, tu, mu
+vsuxseg8ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m4, tu, mu
+vsuxseg8ei64.v  v8, (a0), v16
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vsoxseg2ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vsoxseg2ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vsoxseg2ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m1, tu, mu
+vsoxseg2ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m2, tu, mu
+vsoxseg2ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m4, tu, mu
+vsoxseg2ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m8, tu, mu
+vsoxseg2ei8.v v8, (a0), v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vsoxseg2ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vsoxseg2ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vsoxseg2ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m1, tu, mu
+vsoxseg2ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m2, tu, mu
+vsoxseg2ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m4, tu, mu
+vsoxseg2ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m8, tu, mu
+vsoxseg2ei16.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vsoxseg2ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vsoxseg2ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vsoxseg2ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m1, tu, mu
+vsoxseg2ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m2, tu, mu
+vsoxseg2ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m4, tu, mu
+vsoxseg2ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m8, tu, mu
+vsoxseg2ei32.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vsoxseg2ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vsoxseg2ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vsoxseg2ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m1, tu, mu
+vsoxseg2ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m2, tu, mu
+vsoxseg2ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m4, tu, mu
+vsoxseg2ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m8, tu, mu
+vsoxseg2ei64.v  v8, (a0), v16
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vsoxseg3ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vsoxseg3ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vsoxseg3ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m1, tu, mu
+vsoxseg3ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m2, tu, mu
+vsoxseg3ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m4, tu, mu
+vsoxseg3ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m8, tu, mu
+vsoxseg3ei8.v v8, (a0), v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vsoxseg3ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vsoxseg3ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vsoxseg3ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m1, tu, mu
+vsoxseg3ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m2, tu, mu
+vsoxseg3ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m4, tu, mu
+vsoxseg3ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m8, tu, mu
+vsoxseg3ei16.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vsoxseg3ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vsoxseg3ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vsoxseg3ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m1, tu, mu
+vsoxseg3ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m2, tu, mu
+vsoxseg3ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m4, tu, mu
+vsoxseg3ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m8, tu, mu
+vsoxseg3ei32.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vsoxseg3ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vsoxseg3ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vsoxseg3ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m1, tu, mu
+vsoxseg3ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m2, tu, mu
+vsoxseg3ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m4, tu, mu
+vsoxseg3ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m8, tu, mu
+vsoxseg3ei64.v  v8, (a0), v16
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vsoxseg4ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vsoxseg4ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vsoxseg4ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m1, tu, mu
+vsoxseg4ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m2, tu, mu
+vsoxseg4ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m4, tu, mu
+vsoxseg4ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m8, tu, mu
+vsoxseg4ei8.v v8, (a0), v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vsoxseg4ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vsoxseg4ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vsoxseg4ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m1, tu, mu
+vsoxseg4ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m2, tu, mu
+vsoxseg4ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m4, tu, mu
+vsoxseg4ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m8, tu, mu
+vsoxseg4ei16.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vsoxseg4ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vsoxseg4ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vsoxseg4ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m1, tu, mu
+vsoxseg4ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m2, tu, mu
+vsoxseg4ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m4, tu, mu
+vsoxseg4ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m8, tu, mu
+vsoxseg4ei32.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vsoxseg4ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vsoxseg4ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vsoxseg4ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m1, tu, mu
+vsoxseg4ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m2, tu, mu
+vsoxseg4ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m4, tu, mu
+vsoxseg4ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m8, tu, mu
+vsoxseg4ei64.v  v8, (a0), v16
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vsoxseg5ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vsoxseg5ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vsoxseg5ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m1, tu, mu
+vsoxseg5ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m2, tu, mu
+vsoxseg5ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m4, tu, mu
+vsoxseg5ei8.v v8, (a0), v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vsoxseg5ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vsoxseg5ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vsoxseg5ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m1, tu, mu
+vsoxseg5ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m2, tu, mu
+vsoxseg5ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m4, tu, mu
+vsoxseg5ei16.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vsoxseg5ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vsoxseg5ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vsoxseg5ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m1, tu, mu
+vsoxseg5ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m2, tu, mu
+vsoxseg5ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m4, tu, mu
+vsoxseg5ei32.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vsoxseg5ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vsoxseg5ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vsoxseg5ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m1, tu, mu
+vsoxseg5ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m2, tu, mu
+vsoxseg5ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m4, tu, mu
+vsoxseg5ei64.v  v8, (a0), v16
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vsoxseg6ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vsoxseg6ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vsoxseg6ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m1, tu, mu
+vsoxseg6ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m2, tu, mu
+vsoxseg6ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m4, tu, mu
+vsoxseg6ei8.v v8, (a0), v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vsoxseg6ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vsoxseg6ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vsoxseg6ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m1, tu, mu
+vsoxseg6ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m2, tu, mu
+vsoxseg6ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m4, tu, mu
+vsoxseg6ei16.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vsoxseg6ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vsoxseg6ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vsoxseg6ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m1, tu, mu
+vsoxseg6ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m2, tu, mu
+vsoxseg6ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m4, tu, mu
+vsoxseg6ei32.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vsoxseg6ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vsoxseg6ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vsoxseg6ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m1, tu, mu
+vsoxseg6ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m2, tu, mu
+vsoxseg6ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m4, tu, mu
+vsoxseg6ei64.v  v8, (a0), v16
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vsoxseg7ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vsoxseg7ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vsoxseg7ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m1, tu, mu
+vsoxseg7ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m2, tu, mu
+vsoxseg7ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m4, tu, mu
+vsoxseg7ei8.v v8, (a0), v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vsoxseg7ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vsoxseg7ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vsoxseg7ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m1, tu, mu
+vsoxseg7ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m2, tu, mu
+vsoxseg7ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m4, tu, mu
+vsoxseg7ei16.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vsoxseg7ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vsoxseg7ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vsoxseg7ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m1, tu, mu
+vsoxseg7ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m2, tu, mu
+vsoxseg7ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m4, tu, mu
+vsoxseg7ei32.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vsoxseg7ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vsoxseg7ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vsoxseg7ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m1, tu, mu
+vsoxseg7ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m2, tu, mu
+vsoxseg7ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m4, tu, mu
+vsoxseg7ei64.v  v8, (a0), v16
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vsoxseg8ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vsoxseg8ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vsoxseg8ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m1, tu, mu
+vsoxseg8ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m2, tu, mu
+vsoxseg8ei8.v v8, (a0), v16
+vsetvli zero, zero, e8, m4, tu, mu
+vsoxseg8ei8.v v8, (a0), v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vsoxseg8ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vsoxseg8ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vsoxseg8ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m1, tu, mu
+vsoxseg8ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m2, tu, mu
+vsoxseg8ei16.v  v8, (a0), v16
+vsetvli zero, zero, e16, m4, tu, mu
+vsoxseg8ei16.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vsoxseg8ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vsoxseg8ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vsoxseg8ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m1, tu, mu
+vsoxseg8ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m2, tu, mu
+vsoxseg8ei32.v  v8, (a0), v16
+vsetvli zero, zero, e32, m4, tu, mu
+vsoxseg8ei32.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vsoxseg8ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vsoxseg8ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vsoxseg8ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m1, tu, mu
+vsoxseg8ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m2, tu, mu
+vsoxseg8ei64.v  v8, (a0), v16
+vsetvli zero, zero, e64, m4, tu, mu
+vsoxseg8ei64.v  v8, (a0), v16
+
+# CHECK:      Iterations:        1
+# CHECK-NEXT: Instructions:      3240
+# CHECK-NEXT: Total Cycles:      217825
+# CHECK-NEXT: Total uOps:        3240
+
+# CHECK:      Dispatch Width:    3
+# CHECK-NEXT: uOps Per Cycle:    0.01
+# CHECK-NEXT: IPC:               0.01
+# CHECK-NEXT: Block RThroughput: 124596.0
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vlseg2e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlseg2e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg2e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg2e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg2e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg2e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vlseg2e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg2e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vlseg2e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlseg2e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg2e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg2e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg2e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg2e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg2e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg2e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vlseg2e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlseg2e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg2e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg2e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg2e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg2e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg2e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg2e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vlseg2e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlseg2e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg2e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg2e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vlseg3e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlseg3e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vlseg3e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlseg3e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlseg3e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      396   396.00  *                   vlseg3e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      396   396.00  *                   vlseg3e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      204   204.00  *                   vlseg3e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vlseg3e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlseg3e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vlseg3e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlseg3e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      204   204.00  *                   vlseg3e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      204   204.00  *                   vlseg3e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlseg3e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlseg3e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vlseg3e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlseg3e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vlseg3e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlseg3e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlseg3e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlseg3e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlseg3e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlseg3e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vlseg3e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlseg3e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlseg3e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlseg3e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlseg4e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg4e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg4e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg4e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg4e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      524   524.00  *                   vlseg4e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      524   524.00  *                   vlseg4e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vlseg4e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlseg4e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg4e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg4e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg4e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vlseg4e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vlseg4e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg4e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg4e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlseg4e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg4e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg4e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg4e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg4e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg4e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg4e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg4e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlseg4e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg4e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg4e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg4e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vlseg5e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      32    32.00   *                   vlseg5e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      52    52.00   *                   vlseg5e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      92    92.00   *                   vlseg5e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      652   652.00  *                   vlseg5e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      652   652.00  *                   vlseg5e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      332   332.00  *                   vlseg5e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vlseg5e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      32    32.00   *                   vlseg5e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      52    52.00   *                   vlseg5e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      332   332.00  *                   vlseg5e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      332   332.00  *                   vlseg5e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      172   172.00  *                   vlseg5e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      172   172.00  *                   vlseg5e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vlseg5e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      32    32.00   *                   vlseg5e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      172   172.00  *                   vlseg5e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      172   172.00  *                   vlseg5e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      92    92.00   *                   vlseg5e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      92    92.00   *                   vlseg5e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      92    92.00   *                   vlseg5e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vlseg5e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      92    92.00   *                   vlseg5e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      92    92.00   *                   vlseg5e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlseg6e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vlseg6e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlseg6e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlseg6e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      780   780.00  *                   vlseg6e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      780   780.00  *                   vlseg6e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      396   396.00  *                   vlseg6e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlseg6e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vlseg6e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlseg6e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      396   396.00  *                   vlseg6e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      396   396.00  *                   vlseg6e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      204   204.00  *                   vlseg6e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      204   204.00  *                   vlseg6e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlseg6e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vlseg6e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      204   204.00  *                   vlseg6e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      204   204.00  *                   vlseg6e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlseg6e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlseg6e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlseg6e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlseg6e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlseg6e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlseg6e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vlseg7e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      40    40.00   *                   vlseg7e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      68    68.00   *                   vlseg7e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      124   124.00  *                   vlseg7e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      908   908.00  *                   vlseg7e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      908   908.00  *                   vlseg7e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      460   460.00  *                   vlseg7e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vlseg7e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      40    40.00   *                   vlseg7e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      68    68.00   *                   vlseg7e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      460   460.00  *                   vlseg7e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      460   460.00  *                   vlseg7e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      236   236.00  *                   vlseg7e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      236   236.00  *                   vlseg7e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vlseg7e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      40    40.00   *                   vlseg7e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      236   236.00  *                   vlseg7e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      236   236.00  *                   vlseg7e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      124   124.00  *                   vlseg7e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      124   124.00  *                   vlseg7e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      124   124.00  *                   vlseg7e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vlseg7e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      124   124.00  *                   vlseg7e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      124   124.00  *                   vlseg7e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg8e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg8e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg8e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg8e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      1036   1036.00  *                   vlseg8e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      1036   1036.00  *                   vlseg8e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      524   524.00  *                   vlseg8e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg8e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg8e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg8e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      524   524.00  *                   vlseg8e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      524   524.00  *                   vlseg8e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vlseg8e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vlseg8e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg8e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg8e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vlseg8e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vlseg8e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg8e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg8e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg8e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg8e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg8e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg8e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      5     16.00          *            vsseg2e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsseg2e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsseg2e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsseg2e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsseg2e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsseg2e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      257   268.00         *            vsseg2e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsseg2e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      5     16.00          *            vsseg2e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsseg2e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsseg2e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsseg2e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsseg2e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsseg2e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsseg2e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsseg2e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      5     16.00          *            vsseg2e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsseg2e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsseg2e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsseg2e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsseg2e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsseg2e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsseg2e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsseg2e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      5     16.00          *            vsseg2e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsseg2e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsseg2e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsseg2e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      7     18.00          *            vsseg3e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsseg3e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vsseg3e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vsseg3e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vsseg3e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      385   396.00         *            vsseg3e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      385   396.00         *            vsseg3e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      193   204.00         *            vsseg3e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      7     18.00          *            vsseg3e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsseg3e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vsseg3e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vsseg3e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      193   204.00         *            vsseg3e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      193   204.00         *            vsseg3e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vsseg3e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vsseg3e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      7     18.00          *            vsseg3e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsseg3e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vsseg3e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vsseg3e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vsseg3e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vsseg3e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vsseg3e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vsseg3e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      7     18.00          *            vsseg3e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsseg3e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vsseg3e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vsseg3e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsseg4e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsseg4e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsseg4e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsseg4e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsseg4e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      513   524.00         *            vsseg4e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      513   524.00         *            vsseg4e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      257   268.00         *            vsseg4e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsseg4e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsseg4e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsseg4e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsseg4e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      257   268.00         *            vsseg4e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      257   268.00         *            vsseg4e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsseg4e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsseg4e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsseg4e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsseg4e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsseg4e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsseg4e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsseg4e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsseg4e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsseg4e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsseg4e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsseg4e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsseg4e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsseg4e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsseg4e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      11    22.00          *            vsseg5e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      21    32.00          *            vsseg5e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      41    52.00          *            vsseg5e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      81    92.00          *            vsseg5e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      641   652.00         *            vsseg5e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      641   652.00         *            vsseg5e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      321   332.00         *            vsseg5e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      11    22.00          *            vsseg5e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      21    32.00          *            vsseg5e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      41    52.00          *            vsseg5e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      321   332.00         *            vsseg5e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      321   332.00         *            vsseg5e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      161   172.00         *            vsseg5e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      161   172.00         *            vsseg5e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      11    22.00          *            vsseg5e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      21    32.00          *            vsseg5e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      161   172.00         *            vsseg5e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      161   172.00         *            vsseg5e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      81    92.00          *            vsseg5e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      81    92.00          *            vsseg5e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      81    92.00          *            vsseg5e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      11    22.00          *            vsseg5e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      81    92.00          *            vsseg5e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      81    92.00          *            vsseg5e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsseg6e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vsseg6e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vsseg6e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vsseg6e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      769   780.00         *            vsseg6e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      769   780.00         *            vsseg6e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      385   396.00         *            vsseg6e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsseg6e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vsseg6e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vsseg6e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      385   396.00         *            vsseg6e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      385   396.00         *            vsseg6e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      193   204.00         *            vsseg6e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      193   204.00         *            vsseg6e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsseg6e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vsseg6e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      193   204.00         *            vsseg6e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      193   204.00         *            vsseg6e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vsseg6e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vsseg6e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vsseg6e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsseg6e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vsseg6e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vsseg6e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      15    26.00          *            vsseg7e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      29    40.00          *            vsseg7e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      57    68.00          *            vsseg7e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      113   124.00         *            vsseg7e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      897   908.00         *            vsseg7e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      897   908.00         *            vsseg7e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      449   460.00         *            vsseg7e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      15    26.00          *            vsseg7e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      29    40.00          *            vsseg7e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      57    68.00          *            vsseg7e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      449   460.00         *            vsseg7e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      449   460.00         *            vsseg7e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      225   236.00         *            vsseg7e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      225   236.00         *            vsseg7e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      15    26.00          *            vsseg7e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      29    40.00          *            vsseg7e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      225   236.00         *            vsseg7e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      225   236.00         *            vsseg7e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      113   124.00         *            vsseg7e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      113   124.00         *            vsseg7e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      113   124.00         *            vsseg7e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      15    26.00          *            vsseg7e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      113   124.00         *            vsseg7e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      113   124.00         *            vsseg7e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsseg8e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsseg8e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsseg8e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsseg8e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      1025   1036.00         *            vsseg8e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      1025   1036.00         *            vsseg8e8.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      513   524.00         *            vsseg8e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsseg8e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsseg8e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsseg8e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      513   524.00         *            vsseg8e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      513   524.00         *            vsseg8e16.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      257   268.00         *            vsseg8e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      257   268.00         *            vsseg8e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsseg8e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsseg8e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      257   268.00         *            vsseg8e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      257   268.00         *            vsseg8e32.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsseg8e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsseg8e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsseg8e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsseg8e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsseg8e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsseg8e64.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vlsseg2e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlsseg2e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlsseg2e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlsseg2e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlsseg2e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlsseg2e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vlsseg2e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlsseg2e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vlsseg2e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlsseg2e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlsseg2e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlsseg2e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlsseg2e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlsseg2e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlsseg2e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlsseg2e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vlsseg2e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlsseg2e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlsseg2e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlsseg2e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlsseg2e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlsseg2e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlsseg2e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlsseg2e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vlsseg2e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlsseg2e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlsseg2e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlsseg2e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vlsseg3e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlsseg3e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vlsseg3e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlsseg3e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlsseg3e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      396   396.00  *                   vlsseg3e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      396   396.00  *                   vlsseg3e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      204   204.00  *                   vlsseg3e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vlsseg3e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlsseg3e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vlsseg3e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlsseg3e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      204   204.00  *                   vlsseg3e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      204   204.00  *                   vlsseg3e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlsseg3e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlsseg3e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vlsseg3e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlsseg3e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vlsseg3e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlsseg3e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlsseg3e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlsseg3e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlsseg3e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlsseg3e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vlsseg3e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlsseg3e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlsseg3e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlsseg3e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlsseg4e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlsseg4e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlsseg4e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlsseg4e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlsseg4e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      524   524.00  *                   vlsseg4e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      524   524.00  *                   vlsseg4e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vlsseg4e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlsseg4e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlsseg4e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlsseg4e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlsseg4e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vlsseg4e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vlsseg4e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlsseg4e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlsseg4e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlsseg4e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlsseg4e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlsseg4e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlsseg4e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlsseg4e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlsseg4e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlsseg4e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlsseg4e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlsseg4e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlsseg4e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlsseg4e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlsseg4e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vlsseg5e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      32    32.00   *                   vlsseg5e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      52    52.00   *                   vlsseg5e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      92    92.00   *                   vlsseg5e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      652   652.00  *                   vlsseg5e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      652   652.00  *                   vlsseg5e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      332   332.00  *                   vlsseg5e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vlsseg5e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      32    32.00   *                   vlsseg5e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      52    52.00   *                   vlsseg5e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      332   332.00  *                   vlsseg5e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      332   332.00  *                   vlsseg5e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      172   172.00  *                   vlsseg5e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      172   172.00  *                   vlsseg5e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vlsseg5e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      32    32.00   *                   vlsseg5e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      172   172.00  *                   vlsseg5e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      172   172.00  *                   vlsseg5e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      92    92.00   *                   vlsseg5e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      92    92.00   *                   vlsseg5e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      92    92.00   *                   vlsseg5e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vlsseg5e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      92    92.00   *                   vlsseg5e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      92    92.00   *                   vlsseg5e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlsseg6e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vlsseg6e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlsseg6e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlsseg6e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      780   780.00  *                   vlsseg6e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      780   780.00  *                   vlsseg6e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      396   396.00  *                   vlsseg6e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlsseg6e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vlsseg6e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlsseg6e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      396   396.00  *                   vlsseg6e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      396   396.00  *                   vlsseg6e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      204   204.00  *                   vlsseg6e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      204   204.00  *                   vlsseg6e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlsseg6e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vlsseg6e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      204   204.00  *                   vlsseg6e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      204   204.00  *                   vlsseg6e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlsseg6e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlsseg6e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlsseg6e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlsseg6e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlsseg6e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlsseg6e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vlsseg7e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      40    40.00   *                   vlsseg7e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      68    68.00   *                   vlsseg7e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      124   124.00  *                   vlsseg7e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      908   908.00  *                   vlsseg7e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      908   908.00  *                   vlsseg7e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      460   460.00  *                   vlsseg7e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vlsseg7e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      40    40.00   *                   vlsseg7e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      68    68.00   *                   vlsseg7e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      460   460.00  *                   vlsseg7e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      460   460.00  *                   vlsseg7e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      236   236.00  *                   vlsseg7e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      236   236.00  *                   vlsseg7e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vlsseg7e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      40    40.00   *                   vlsseg7e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      236   236.00  *                   vlsseg7e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      236   236.00  *                   vlsseg7e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      124   124.00  *                   vlsseg7e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      124   124.00  *                   vlsseg7e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      124   124.00  *                   vlsseg7e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vlsseg7e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      124   124.00  *                   vlsseg7e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      124   124.00  *                   vlsseg7e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlsseg8e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlsseg8e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlsseg8e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlsseg8e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      1036   1036.00  *                   vlsseg8e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      1036   1036.00  *                   vlsseg8e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      524   524.00  *                   vlsseg8e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlsseg8e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlsseg8e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlsseg8e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      524   524.00  *                   vlsseg8e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      524   524.00  *                   vlsseg8e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vlsseg8e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vlsseg8e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlsseg8e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlsseg8e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vlsseg8e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vlsseg8e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlsseg8e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlsseg8e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlsseg8e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlsseg8e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlsseg8e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlsseg8e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      5     16.00          *            vssseg2e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vssseg2e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vssseg2e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vssseg2e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vssseg2e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vssseg2e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      257   268.00         *            vssseg2e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vssseg2e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      5     16.00          *            vssseg2e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vssseg2e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vssseg2e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vssseg2e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vssseg2e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vssseg2e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vssseg2e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vssseg2e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      5     16.00          *            vssseg2e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vssseg2e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vssseg2e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vssseg2e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vssseg2e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vssseg2e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vssseg2e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vssseg2e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      5     16.00          *            vssseg2e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vssseg2e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vssseg2e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vssseg2e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      7     18.00          *            vssseg3e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vssseg3e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vssseg3e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vssseg3e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vssseg3e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      385   396.00         *            vssseg3e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      385   396.00         *            vssseg3e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      193   204.00         *            vssseg3e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      7     18.00          *            vssseg3e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vssseg3e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vssseg3e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vssseg3e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      193   204.00         *            vssseg3e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      193   204.00         *            vssseg3e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vssseg3e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vssseg3e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      7     18.00          *            vssseg3e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vssseg3e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vssseg3e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vssseg3e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vssseg3e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vssseg3e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vssseg3e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vssseg3e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      7     18.00          *            vssseg3e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vssseg3e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vssseg3e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vssseg3e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vssseg4e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vssseg4e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vssseg4e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vssseg4e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vssseg4e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      513   524.00         *            vssseg4e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      513   524.00         *            vssseg4e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      257   268.00         *            vssseg4e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vssseg4e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vssseg4e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vssseg4e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vssseg4e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      257   268.00         *            vssseg4e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      257   268.00         *            vssseg4e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vssseg4e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vssseg4e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vssseg4e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vssseg4e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vssseg4e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vssseg4e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vssseg4e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vssseg4e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vssseg4e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vssseg4e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vssseg4e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vssseg4e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vssseg4e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vssseg4e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      11    22.00          *            vssseg5e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      21    32.00          *            vssseg5e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      41    52.00          *            vssseg5e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      81    92.00          *            vssseg5e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      641   652.00         *            vssseg5e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      641   652.00         *            vssseg5e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      321   332.00         *            vssseg5e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      11    22.00          *            vssseg5e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      21    32.00          *            vssseg5e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      41    52.00          *            vssseg5e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      321   332.00         *            vssseg5e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      321   332.00         *            vssseg5e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      161   172.00         *            vssseg5e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      161   172.00         *            vssseg5e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      11    22.00          *            vssseg5e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      21    32.00          *            vssseg5e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      161   172.00         *            vssseg5e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      161   172.00         *            vssseg5e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      81    92.00          *            vssseg5e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      81    92.00          *            vssseg5e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      81    92.00          *            vssseg5e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      11    22.00          *            vssseg5e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      81    92.00          *            vssseg5e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      81    92.00          *            vssseg5e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vssseg6e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vssseg6e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vssseg6e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vssseg6e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      769   780.00         *            vssseg6e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      769   780.00         *            vssseg6e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      385   396.00         *            vssseg6e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vssseg6e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vssseg6e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vssseg6e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      385   396.00         *            vssseg6e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      385   396.00         *            vssseg6e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      193   204.00         *            vssseg6e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      193   204.00         *            vssseg6e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vssseg6e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vssseg6e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      193   204.00         *            vssseg6e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      193   204.00         *            vssseg6e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vssseg6e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vssseg6e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vssseg6e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vssseg6e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vssseg6e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vssseg6e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      15    26.00          *            vssseg7e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      29    40.00          *            vssseg7e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      57    68.00          *            vssseg7e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      113   124.00         *            vssseg7e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      897   908.00         *            vssseg7e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      897   908.00         *            vssseg7e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      449   460.00         *            vssseg7e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      15    26.00          *            vssseg7e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      29    40.00          *            vssseg7e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      57    68.00          *            vssseg7e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      449   460.00         *            vssseg7e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      449   460.00         *            vssseg7e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      225   236.00         *            vssseg7e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      225   236.00         *            vssseg7e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      15    26.00          *            vssseg7e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      29    40.00          *            vssseg7e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      225   236.00         *            vssseg7e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      225   236.00         *            vssseg7e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      113   124.00         *            vssseg7e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      113   124.00         *            vssseg7e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      113   124.00         *            vssseg7e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      15    26.00          *            vssseg7e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      113   124.00         *            vssseg7e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      113   124.00         *            vssseg7e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vssseg8e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vssseg8e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vssseg8e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vssseg8e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      1025   1036.00         *            vssseg8e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      1025   1036.00         *            vssseg8e8.v v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      513   524.00         *            vssseg8e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vssseg8e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vssseg8e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vssseg8e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      513   524.00         *            vssseg8e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      513   524.00         *            vssseg8e16.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      257   268.00         *            vssseg8e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      257   268.00         *            vssseg8e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vssseg8e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vssseg8e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      257   268.00         *            vssseg8e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      257   268.00         *            vssseg8e32.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vssseg8e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vssseg8e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vssseg8e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vssseg8e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vssseg8e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vssseg8e64.v  v8, (a0), a1
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vlseg2e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlseg2e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg2e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg2e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg2e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg2e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vlseg2e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg2e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vlseg2e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlseg2e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg2e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg2e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg2e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg2e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg2e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg2e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vlseg2e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlseg2e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg2e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg2e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg2e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg2e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg2e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg2e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vlseg2e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlseg2e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg2e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg2e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vlseg3e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlseg3e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vlseg3e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlseg3e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlseg3e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      396   396.00  *                   vlseg3e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      396   396.00  *                   vlseg3e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      204   204.00  *                   vlseg3e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vlseg3e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlseg3e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vlseg3e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlseg3e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      204   204.00  *                   vlseg3e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      204   204.00  *                   vlseg3e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlseg3e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlseg3e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vlseg3e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlseg3e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vlseg3e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlseg3e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlseg3e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlseg3e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlseg3e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlseg3e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vlseg3e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlseg3e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlseg3e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlseg3e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlseg4e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg4e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg4e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg4e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg4e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      524   524.00  *                   vlseg4e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      524   524.00  *                   vlseg4e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vlseg4e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlseg4e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg4e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg4e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg4e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vlseg4e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vlseg4e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg4e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg4e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlseg4e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg4e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg4e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg4e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg4e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg4e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg4e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg4e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vlseg4e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg4e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg4e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg4e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vlseg5e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      32    32.00   *                   vlseg5e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      52    52.00   *                   vlseg5e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      92    92.00   *                   vlseg5e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      652   652.00  *                   vlseg5e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      652   652.00  *                   vlseg5e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      332   332.00  *                   vlseg5e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vlseg5e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      32    32.00   *                   vlseg5e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      52    52.00   *                   vlseg5e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      332   332.00  *                   vlseg5e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      332   332.00  *                   vlseg5e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      172   172.00  *                   vlseg5e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      172   172.00  *                   vlseg5e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vlseg5e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      32    32.00   *                   vlseg5e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      172   172.00  *                   vlseg5e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      172   172.00  *                   vlseg5e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      92    92.00   *                   vlseg5e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      92    92.00   *                   vlseg5e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      92    92.00   *                   vlseg5e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vlseg5e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      92    92.00   *                   vlseg5e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      92    92.00   *                   vlseg5e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlseg6e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vlseg6e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlseg6e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlseg6e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      780   780.00  *                   vlseg6e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      780   780.00  *                   vlseg6e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      396   396.00  *                   vlseg6e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlseg6e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vlseg6e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vlseg6e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      396   396.00  *                   vlseg6e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      396   396.00  *                   vlseg6e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      204   204.00  *                   vlseg6e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      204   204.00  *                   vlseg6e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlseg6e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vlseg6e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      204   204.00  *                   vlseg6e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      204   204.00  *                   vlseg6e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlseg6e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlseg6e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlseg6e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vlseg6e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlseg6e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vlseg6e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vlseg7e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      40    40.00   *                   vlseg7e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      68    68.00   *                   vlseg7e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      124   124.00  *                   vlseg7e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      908   908.00  *                   vlseg7e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      908   908.00  *                   vlseg7e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      460   460.00  *                   vlseg7e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vlseg7e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      40    40.00   *                   vlseg7e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      68    68.00   *                   vlseg7e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      460   460.00  *                   vlseg7e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      460   460.00  *                   vlseg7e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      236   236.00  *                   vlseg7e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      236   236.00  *                   vlseg7e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vlseg7e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      40    40.00   *                   vlseg7e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      236   236.00  *                   vlseg7e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      236   236.00  *                   vlseg7e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      124   124.00  *                   vlseg7e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      124   124.00  *                   vlseg7e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      124   124.00  *                   vlseg7e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vlseg7e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      124   124.00  *                   vlseg7e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      124   124.00  *                   vlseg7e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg8e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg8e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg8e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg8e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      1036   1036.00  *                   vlseg8e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      1036   1036.00  *                   vlseg8e8ff.v  v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      524   524.00  *                   vlseg8e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg8e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg8e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vlseg8e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      524   524.00  *                   vlseg8e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      524   524.00  *                   vlseg8e16ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vlseg8e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vlseg8e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg8e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vlseg8e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vlseg8e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vlseg8e32ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg8e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg8e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg8e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vlseg8e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg8e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vlseg8e64ff.v v8, (a0)
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vluxseg2ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vluxseg2ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vluxseg2ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vluxseg2ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vluxseg2ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vluxseg2ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vluxseg2ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vluxseg2ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vluxseg2ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vluxseg2ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vluxseg2ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vluxseg2ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vluxseg2ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vluxseg2ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vluxseg2ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vluxseg2ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vluxseg2ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vluxseg2ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vluxseg2ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vluxseg2ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vluxseg2ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vluxseg2ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vluxseg2ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vluxseg2ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vluxseg2ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vluxseg2ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vluxseg2ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vluxseg2ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vluxseg3ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vluxseg3ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vluxseg3ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vluxseg3ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vluxseg3ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      396   396.00  *                   vluxseg3ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      396   396.00  *                   vluxseg3ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vluxseg3ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vluxseg3ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vluxseg3ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vluxseg3ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vluxseg3ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      204   204.00  *                   vluxseg3ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      204   204.00  *                   vluxseg3ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vluxseg3ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vluxseg3ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vluxseg3ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vluxseg3ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vluxseg3ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vluxseg3ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vluxseg3ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vluxseg3ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vluxseg3ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vluxseg3ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vluxseg3ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vluxseg3ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vluxseg3ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vluxseg3ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vluxseg4ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vluxseg4ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vluxseg4ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vluxseg4ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vluxseg4ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      524   524.00  *                   vluxseg4ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      524   524.00  *                   vluxseg4ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vluxseg4ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vluxseg4ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vluxseg4ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vluxseg4ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vluxseg4ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vluxseg4ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vluxseg4ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vluxseg4ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vluxseg4ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vluxseg4ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vluxseg4ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vluxseg4ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vluxseg4ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vluxseg4ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vluxseg4ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vluxseg4ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vluxseg4ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vluxseg4ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vluxseg4ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vluxseg4ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vluxseg4ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vluxseg5ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      32    32.00   *                   vluxseg5ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      52    52.00   *                   vluxseg5ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      92    92.00   *                   vluxseg5ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      652   652.00  *                   vluxseg5ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      652   652.00  *                   vluxseg5ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vluxseg5ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      32    32.00   *                   vluxseg5ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      52    52.00   *                   vluxseg5ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      52    52.00   *                   vluxseg5ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      332   332.00  *                   vluxseg5ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      332   332.00  *                   vluxseg5ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vluxseg5ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      32    32.00   *                   vluxseg5ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      32    32.00   *                   vluxseg5ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      32    32.00   *                   vluxseg5ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      172   172.00  *                   vluxseg5ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      172   172.00  *                   vluxseg5ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vluxseg5ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vluxseg5ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vluxseg5ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vluxseg5ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      92    92.00   *                   vluxseg5ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      92    92.00   *                   vluxseg5ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vluxseg6ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vluxseg6ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vluxseg6ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vluxseg6ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      780   780.00  *                   vluxseg6ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      780   780.00  *                   vluxseg6ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vluxseg6ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vluxseg6ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vluxseg6ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vluxseg6ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      396   396.00  *                   vluxseg6ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      396   396.00  *                   vluxseg6ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vluxseg6ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vluxseg6ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vluxseg6ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vluxseg6ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      204   204.00  *                   vluxseg6ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      204   204.00  *                   vluxseg6ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vluxseg6ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vluxseg6ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vluxseg6ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vluxseg6ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vluxseg6ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vluxseg6ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vluxseg7ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      40    40.00   *                   vluxseg7ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      68    68.00   *                   vluxseg7ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      124   124.00  *                   vluxseg7ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      908   908.00  *                   vluxseg7ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      908   908.00  *                   vluxseg7ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vluxseg7ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      40    40.00   *                   vluxseg7ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      68    68.00   *                   vluxseg7ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      68    68.00   *                   vluxseg7ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      460   460.00  *                   vluxseg7ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      460   460.00  *                   vluxseg7ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vluxseg7ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      40    40.00   *                   vluxseg7ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      40    40.00   *                   vluxseg7ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      40    40.00   *                   vluxseg7ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      236   236.00  *                   vluxseg7ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      236   236.00  *                   vluxseg7ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vluxseg7ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vluxseg7ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vluxseg7ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vluxseg7ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      124   124.00  *                   vluxseg7ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      124   124.00  *                   vluxseg7ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vluxseg8ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vluxseg8ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vluxseg8ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vluxseg8ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      1036   1036.00  *                   vluxseg8ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      1036   1036.00  *                   vluxseg8ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vluxseg8ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vluxseg8ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vluxseg8ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vluxseg8ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      524   524.00  *                   vluxseg8ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      524   524.00  *                   vluxseg8ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vluxseg8ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vluxseg8ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vluxseg8ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vluxseg8ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vluxseg8ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vluxseg8ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vluxseg8ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vluxseg8ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vluxseg8ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vluxseg8ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vluxseg8ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vluxseg8ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vloxseg2ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vloxseg2ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vloxseg2ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vloxseg2ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vloxseg2ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vloxseg2ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vloxseg2ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vloxseg2ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vloxseg2ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vloxseg2ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vloxseg2ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vloxseg2ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vloxseg2ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vloxseg2ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vloxseg2ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vloxseg2ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vloxseg2ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vloxseg2ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vloxseg2ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vloxseg2ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vloxseg2ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vloxseg2ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vloxseg2ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vloxseg2ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      16    16.00   *                   vloxseg2ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vloxseg2ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vloxseg2ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vloxseg2ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vloxseg3ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vloxseg3ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vloxseg3ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vloxseg3ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vloxseg3ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      396   396.00  *                   vloxseg3ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      396   396.00  *                   vloxseg3ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vloxseg3ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vloxseg3ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vloxseg3ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vloxseg3ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vloxseg3ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      204   204.00  *                   vloxseg3ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      204   204.00  *                   vloxseg3ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vloxseg3ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vloxseg3ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vloxseg3ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vloxseg3ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vloxseg3ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vloxseg3ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vloxseg3ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vloxseg3ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vloxseg3ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vloxseg3ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      18    18.00   *                   vloxseg3ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vloxseg3ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vloxseg3ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vloxseg3ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vloxseg4ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vloxseg4ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vloxseg4ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vloxseg4ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vloxseg4ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      524   524.00  *                   vloxseg4ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      524   524.00  *                   vloxseg4ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vloxseg4ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vloxseg4ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vloxseg4ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vloxseg4ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vloxseg4ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vloxseg4ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vloxseg4ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vloxseg4ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vloxseg4ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vloxseg4ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vloxseg4ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vloxseg4ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vloxseg4ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vloxseg4ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vloxseg4ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vloxseg4ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vloxseg4ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      20    20.00   *                   vloxseg4ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vloxseg4ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vloxseg4ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vloxseg4ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vloxseg5ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      32    32.00   *                   vloxseg5ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      52    52.00   *                   vloxseg5ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      92    92.00   *                   vloxseg5ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      652   652.00  *                   vloxseg5ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      652   652.00  *                   vloxseg5ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vloxseg5ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      32    32.00   *                   vloxseg5ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      52    52.00   *                   vloxseg5ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      52    52.00   *                   vloxseg5ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      332   332.00  *                   vloxseg5ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      332   332.00  *                   vloxseg5ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vloxseg5ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      32    32.00   *                   vloxseg5ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      32    32.00   *                   vloxseg5ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      32    32.00   *                   vloxseg5ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      172   172.00  *                   vloxseg5ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      172   172.00  *                   vloxseg5ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vloxseg5ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vloxseg5ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vloxseg5ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      22    22.00   *                   vloxseg5ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      92    92.00   *                   vloxseg5ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      92    92.00   *                   vloxseg5ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vloxseg6ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vloxseg6ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vloxseg6ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vloxseg6ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      780   780.00  *                   vloxseg6ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      780   780.00  *                   vloxseg6ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vloxseg6ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vloxseg6ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vloxseg6ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      60    60.00   *                   vloxseg6ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      396   396.00  *                   vloxseg6ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      396   396.00  *                   vloxseg6ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vloxseg6ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vloxseg6ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vloxseg6ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      36    36.00   *                   vloxseg6ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      204   204.00  *                   vloxseg6ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      204   204.00  *                   vloxseg6ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vloxseg6ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vloxseg6ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vloxseg6ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      24    24.00   *                   vloxseg6ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vloxseg6ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      108   108.00  *                   vloxseg6ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vloxseg7ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      40    40.00   *                   vloxseg7ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      68    68.00   *                   vloxseg7ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      124   124.00  *                   vloxseg7ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      908   908.00  *                   vloxseg7ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      908   908.00  *                   vloxseg7ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vloxseg7ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      40    40.00   *                   vloxseg7ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      68    68.00   *                   vloxseg7ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      68    68.00   *                   vloxseg7ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      460   460.00  *                   vloxseg7ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      460   460.00  *                   vloxseg7ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vloxseg7ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      40    40.00   *                   vloxseg7ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      40    40.00   *                   vloxseg7ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      40    40.00   *                   vloxseg7ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      236   236.00  *                   vloxseg7ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      236   236.00  *                   vloxseg7ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vloxseg7ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vloxseg7ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vloxseg7ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      26    26.00   *                   vloxseg7ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      124   124.00  *                   vloxseg7ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      124   124.00  *                   vloxseg7ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vloxseg8ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vloxseg8ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vloxseg8ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vloxseg8ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      1036   1036.00  *                   vloxseg8ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      1036   1036.00  *                   vloxseg8ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vloxseg8ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vloxseg8ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vloxseg8ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      76    76.00   *                   vloxseg8ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      524   524.00  *                   vloxseg8ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      524   524.00  *                   vloxseg8ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vloxseg8ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vloxseg8ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vloxseg8ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      44    44.00   *                   vloxseg8ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vloxseg8ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      268   268.00  *                   vloxseg8ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vloxseg8ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vloxseg8ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vloxseg8ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      28    28.00   *                   vloxseg8ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vloxseg8ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      140   140.00  *                   vloxseg8ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      5     16.00          *            vsuxseg2ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsuxseg2ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsuxseg2ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsuxseg2ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsuxseg2ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsuxseg2ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      257   268.00         *            vsuxseg2ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      3     14.00          *            vsuxseg2ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      5     16.00          *            vsuxseg2ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsuxseg2ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsuxseg2ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsuxseg2ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsuxseg2ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsuxseg2ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsuxseg2ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      3     14.00          *            vsuxseg2ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      5     16.00          *            vsuxseg2ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsuxseg2ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsuxseg2ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsuxseg2ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsuxseg2ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsuxseg2ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsuxseg2ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      3     14.00          *            vsuxseg2ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      5     16.00          *            vsuxseg2ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsuxseg2ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsuxseg2ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsuxseg2ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      7     18.00          *            vsuxseg3ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsuxseg3ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vsuxseg3ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vsuxseg3ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vsuxseg3ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      385   396.00         *            vsuxseg3ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      385   396.00         *            vsuxseg3ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      4     15.00          *            vsuxseg3ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      7     18.00          *            vsuxseg3ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsuxseg3ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vsuxseg3ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vsuxseg3ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      193   204.00         *            vsuxseg3ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      193   204.00         *            vsuxseg3ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsuxseg3ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      4     15.00          *            vsuxseg3ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      7     18.00          *            vsuxseg3ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsuxseg3ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vsuxseg3ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vsuxseg3ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vsuxseg3ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsuxseg3ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsuxseg3ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      4     15.00          *            vsuxseg3ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      7     18.00          *            vsuxseg3ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsuxseg3ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vsuxseg3ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vsuxseg3ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsuxseg4ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsuxseg4ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsuxseg4ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsuxseg4ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsuxseg4ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      513   524.00         *            vsuxseg4ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      513   524.00         *            vsuxseg4ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      5     16.00          *            vsuxseg4ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsuxseg4ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsuxseg4ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsuxseg4ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsuxseg4ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      257   268.00         *            vsuxseg4ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      257   268.00         *            vsuxseg4ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsuxseg4ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      5     16.00          *            vsuxseg4ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsuxseg4ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsuxseg4ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsuxseg4ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsuxseg4ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsuxseg4ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsuxseg4ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsuxseg4ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      5     16.00          *            vsuxseg4ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsuxseg4ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsuxseg4ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsuxseg4ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsuxseg4ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      11    22.00          *            vsuxseg5ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      21    32.00          *            vsuxseg5ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      41    52.00          *            vsuxseg5ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      81    92.00          *            vsuxseg5ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      641   652.00         *            vsuxseg5ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      641   652.00         *            vsuxseg5ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      6     17.00          *            vsuxseg5ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      11    22.00          *            vsuxseg5ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      21    32.00          *            vsuxseg5ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      41    52.00          *            vsuxseg5ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      321   332.00         *            vsuxseg5ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      321   332.00         *            vsuxseg5ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsuxseg5ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      6     17.00          *            vsuxseg5ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      11    22.00          *            vsuxseg5ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      21    32.00          *            vsuxseg5ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      161   172.00         *            vsuxseg5ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      161   172.00         *            vsuxseg5ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsuxseg5ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsuxseg5ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      6     17.00          *            vsuxseg5ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      11    22.00          *            vsuxseg5ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      81    92.00          *            vsuxseg5ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      81    92.00          *            vsuxseg5ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsuxseg6ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vsuxseg6ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vsuxseg6ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vsuxseg6ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      769   780.00         *            vsuxseg6ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      769   780.00         *            vsuxseg6ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      7     18.00          *            vsuxseg6ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsuxseg6ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vsuxseg6ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vsuxseg6ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      385   396.00         *            vsuxseg6ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      385   396.00         *            vsuxseg6ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsuxseg6ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      7     18.00          *            vsuxseg6ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsuxseg6ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vsuxseg6ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      193   204.00         *            vsuxseg6ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      193   204.00         *            vsuxseg6ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsuxseg6ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsuxseg6ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      7     18.00          *            vsuxseg6ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsuxseg6ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vsuxseg6ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vsuxseg6ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      15    26.00          *            vsuxseg7ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      29    40.00          *            vsuxseg7ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      57    68.00          *            vsuxseg7ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      113   124.00         *            vsuxseg7ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      897   908.00         *            vsuxseg7ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      897   908.00         *            vsuxseg7ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      8     19.00          *            vsuxseg7ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      15    26.00          *            vsuxseg7ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      29    40.00          *            vsuxseg7ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      57    68.00          *            vsuxseg7ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      449   460.00         *            vsuxseg7ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      449   460.00         *            vsuxseg7ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsuxseg7ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      8     19.00          *            vsuxseg7ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      15    26.00          *            vsuxseg7ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      29    40.00          *            vsuxseg7ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      225   236.00         *            vsuxseg7ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      225   236.00         *            vsuxseg7ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsuxseg7ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsuxseg7ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      8     19.00          *            vsuxseg7ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      15    26.00          *            vsuxseg7ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      113   124.00         *            vsuxseg7ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      113   124.00         *            vsuxseg7ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsuxseg8ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsuxseg8ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsuxseg8ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsuxseg8ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      1025   1036.00         *            vsuxseg8ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      1025   1036.00         *            vsuxseg8ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsuxseg8ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsuxseg8ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsuxseg8ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsuxseg8ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      513   524.00         *            vsuxseg8ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      513   524.00         *            vsuxseg8ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsuxseg8ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsuxseg8ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsuxseg8ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsuxseg8ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      257   268.00         *            vsuxseg8ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      257   268.00         *            vsuxseg8ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsuxseg8ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsuxseg8ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsuxseg8ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsuxseg8ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsuxseg8ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsuxseg8ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      5     16.00          *            vsoxseg2ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsoxseg2ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsoxseg2ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsoxseg2ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsoxseg2ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsoxseg2ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      257   268.00         *            vsoxseg2ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      3     14.00          *            vsoxseg2ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      5     16.00          *            vsoxseg2ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsoxseg2ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsoxseg2ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsoxseg2ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsoxseg2ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsoxseg2ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsoxseg2ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      3     14.00          *            vsoxseg2ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      5     16.00          *            vsoxseg2ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsoxseg2ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsoxseg2ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsoxseg2ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsoxseg2ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsoxseg2ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsoxseg2ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      3     14.00          *            vsoxseg2ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      5     16.00          *            vsoxseg2ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsoxseg2ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsoxseg2ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsoxseg2ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      7     18.00          *            vsoxseg3ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsoxseg3ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vsoxseg3ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vsoxseg3ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vsoxseg3ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      385   396.00         *            vsoxseg3ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      385   396.00         *            vsoxseg3ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      4     15.00          *            vsoxseg3ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      7     18.00          *            vsoxseg3ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsoxseg3ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vsoxseg3ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vsoxseg3ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      193   204.00         *            vsoxseg3ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      193   204.00         *            vsoxseg3ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsoxseg3ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      4     15.00          *            vsoxseg3ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      7     18.00          *            vsoxseg3ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsoxseg3ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vsoxseg3ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vsoxseg3ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vsoxseg3ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsoxseg3ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsoxseg3ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      4     15.00          *            vsoxseg3ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      7     18.00          *            vsoxseg3ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsoxseg3ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vsoxseg3ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vsoxseg3ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsoxseg4ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsoxseg4ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsoxseg4ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsoxseg4ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsoxseg4ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      513   524.00         *            vsoxseg4ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      513   524.00         *            vsoxseg4ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      5     16.00          *            vsoxseg4ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsoxseg4ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsoxseg4ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsoxseg4ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsoxseg4ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      257   268.00         *            vsoxseg4ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      257   268.00         *            vsoxseg4ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsoxseg4ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      5     16.00          *            vsoxseg4ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsoxseg4ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsoxseg4ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsoxseg4ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsoxseg4ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsoxseg4ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsoxseg4ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsoxseg4ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      5     16.00          *            vsoxseg4ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsoxseg4ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsoxseg4ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsoxseg4ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsoxseg4ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      11    22.00          *            vsoxseg5ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      21    32.00          *            vsoxseg5ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      41    52.00          *            vsoxseg5ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      81    92.00          *            vsoxseg5ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      641   652.00         *            vsoxseg5ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      641   652.00         *            vsoxseg5ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      6     17.00          *            vsoxseg5ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      11    22.00          *            vsoxseg5ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      21    32.00          *            vsoxseg5ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      41    52.00          *            vsoxseg5ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      321   332.00         *            vsoxseg5ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      321   332.00         *            vsoxseg5ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsoxseg5ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      6     17.00          *            vsoxseg5ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      11    22.00          *            vsoxseg5ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      21    32.00          *            vsoxseg5ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      161   172.00         *            vsoxseg5ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      161   172.00         *            vsoxseg5ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsoxseg5ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsoxseg5ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      6     17.00          *            vsoxseg5ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      11    22.00          *            vsoxseg5ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      81    92.00          *            vsoxseg5ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      81    92.00          *            vsoxseg5ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsoxseg6ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vsoxseg6ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vsoxseg6ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vsoxseg6ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      769   780.00         *            vsoxseg6ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      769   780.00         *            vsoxseg6ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      7     18.00          *            vsoxseg6ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsoxseg6ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vsoxseg6ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      49    60.00          *            vsoxseg6ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      385   396.00         *            vsoxseg6ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      385   396.00         *            vsoxseg6ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsoxseg6ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      7     18.00          *            vsoxseg6ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsoxseg6ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      25    36.00          *            vsoxseg6ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      193   204.00         *            vsoxseg6ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      193   204.00         *            vsoxseg6ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsoxseg6ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsoxseg6ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      7     18.00          *            vsoxseg6ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      13    24.00          *            vsoxseg6ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vsoxseg6ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      97    108.00         *            vsoxseg6ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      15    26.00          *            vsoxseg7ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      29    40.00          *            vsoxseg7ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      57    68.00          *            vsoxseg7ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      113   124.00         *            vsoxseg7ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      897   908.00         *            vsoxseg7ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      897   908.00         *            vsoxseg7ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      8     19.00          *            vsoxseg7ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      15    26.00          *            vsoxseg7ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      29    40.00          *            vsoxseg7ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      57    68.00          *            vsoxseg7ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      449   460.00         *            vsoxseg7ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      449   460.00         *            vsoxseg7ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsoxseg7ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      8     19.00          *            vsoxseg7ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      15    26.00          *            vsoxseg7ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      29    40.00          *            vsoxseg7ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      225   236.00         *            vsoxseg7ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      225   236.00         *            vsoxseg7ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsoxseg7ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsoxseg7ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      8     19.00          *            vsoxseg7ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      15    26.00          *            vsoxseg7ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      113   124.00         *            vsoxseg7ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      113   124.00         *            vsoxseg7ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsoxseg8ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsoxseg8ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsoxseg8ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsoxseg8ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      1025   1036.00         *            vsoxseg8ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      1025   1036.00         *            vsoxseg8ei8.v v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsoxseg8ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsoxseg8ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsoxseg8ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      65    76.00          *            vsoxseg8ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      513   524.00         *            vsoxseg8ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      513   524.00         *            vsoxseg8ei16.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsoxseg8ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsoxseg8ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsoxseg8ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      33    44.00          *            vsoxseg8ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      257   268.00         *            vsoxseg8ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      257   268.00         *            vsoxseg8ei32.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsoxseg8ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      1     12.00          *            vsoxseg8ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      9     20.00          *            vsoxseg8ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      17    28.00          *            vsoxseg8ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsoxseg8ei64.v  v8, (a0), v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      129   140.00         *            vsoxseg8ei64.v  v8, (a0), v16
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - SiFiveP400Div
+# CHECK-NEXT: [1]   - SiFiveP400FEXQ0
+# CHECK-NEXT: [2]   - SiFiveP400FloatDiv
+# CHECK-NEXT: [3]   - SiFiveP400IEXQ0
+# CHECK-NEXT: [4]   - SiFiveP400IEXQ1
+# CHECK-NEXT: [5]   - SiFiveP400IEXQ2
+# CHECK-NEXT: [6]   - SiFiveP400Load
+# CHECK-NEXT: [7]   - SiFiveP400Store
+# CHECK-NEXT: [8]   - SiFiveP400VDiv
+# CHECK-NEXT: [9]   - SiFiveP400VEXQ0
+# CHECK-NEXT: [10]  - SiFiveP400VFloatDiv
+# CHECK-NEXT: [11]  - SiFiveP400VLD
+# CHECK-NEXT: [12]  - SiFiveP400VST
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]
+# CHECK-NEXT:  -      -      -      -     1620.00  -     -      -      -      -      -     124596.00 96850.00
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   Instructions:
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vlseg2e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg2e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg2e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg2e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg2e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg2e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlseg2e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg2e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vlseg2e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg2e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg2e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg2e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg2e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg2e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg2e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg2e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vlseg2e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg2e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg2e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg2e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg2e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg2e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg2e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg2e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vlseg2e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg2e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg2e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg2e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vlseg3e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg3e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlseg3e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg3e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg3e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vlseg3e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vlseg3e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlseg3e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vlseg3e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg3e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlseg3e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg3e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlseg3e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlseg3e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg3e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg3e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vlseg3e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg3e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlseg3e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg3e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg3e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg3e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg3e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg3e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vlseg3e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg3e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg3e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg3e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg4e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg4e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg4e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg4e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg4e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vlseg4e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vlseg4e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlseg4e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg4e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg4e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg4e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg4e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlseg4e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlseg4e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg4e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg4e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg4e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg4e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg4e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg4e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg4e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg4e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg4e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg4e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg4e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg4e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg4e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg4e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vlseg5e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vlseg5e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     52.00   -     vlseg5e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vlseg5e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     652.00  -     vlseg5e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     652.00  -     vlseg5e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     332.00  -     vlseg5e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vlseg5e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vlseg5e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     52.00   -     vlseg5e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     332.00  -     vlseg5e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     332.00  -     vlseg5e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     172.00  -     vlseg5e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     172.00  -     vlseg5e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vlseg5e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vlseg5e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     172.00  -     vlseg5e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     172.00  -     vlseg5e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vlseg5e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vlseg5e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vlseg5e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vlseg5e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vlseg5e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vlseg5e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg6e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlseg6e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg6e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg6e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     780.00  -     vlseg6e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     780.00  -     vlseg6e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vlseg6e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg6e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlseg6e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg6e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vlseg6e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vlseg6e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlseg6e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlseg6e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg6e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlseg6e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlseg6e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlseg6e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg6e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg6e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg6e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg6e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg6e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg6e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vlseg7e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vlseg7e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     68.00   -     vlseg7e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vlseg7e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     908.00  -     vlseg7e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     908.00  -     vlseg7e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     460.00  -     vlseg7e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vlseg7e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vlseg7e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     68.00   -     vlseg7e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     460.00  -     vlseg7e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     460.00  -     vlseg7e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     236.00  -     vlseg7e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     236.00  -     vlseg7e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vlseg7e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vlseg7e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     236.00  -     vlseg7e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     236.00  -     vlseg7e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vlseg7e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vlseg7e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vlseg7e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vlseg7e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vlseg7e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vlseg7e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg8e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg8e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg8e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg8e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1036.00  -    vlseg8e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1036.00  -    vlseg8e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vlseg8e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg8e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg8e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg8e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vlseg8e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vlseg8e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlseg8e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlseg8e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg8e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg8e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlseg8e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlseg8e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg8e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg8e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg8e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg8e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg8e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg8e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsseg2e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsseg2e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsseg2e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsseg2e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsseg2e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsseg2e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vsseg2e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsseg2e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsseg2e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsseg2e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsseg2e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsseg2e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsseg2e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsseg2e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsseg2e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsseg2e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsseg2e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsseg2e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsseg2e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsseg2e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsseg2e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsseg2e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsseg2e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsseg2e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsseg2e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsseg2e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsseg2e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsseg2e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsseg3e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsseg3e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsseg3e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsseg3e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsseg3e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     396.00 vsseg3e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     396.00 vsseg3e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vsseg3e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsseg3e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsseg3e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsseg3e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsseg3e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vsseg3e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vsseg3e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsseg3e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsseg3e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsseg3e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsseg3e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsseg3e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsseg3e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsseg3e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsseg3e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsseg3e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsseg3e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsseg3e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsseg3e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsseg3e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsseg3e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsseg4e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsseg4e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsseg4e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsseg4e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsseg4e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     524.00 vsseg4e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     524.00 vsseg4e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vsseg4e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsseg4e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsseg4e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsseg4e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsseg4e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vsseg4e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vsseg4e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsseg4e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsseg4e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsseg4e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsseg4e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsseg4e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsseg4e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsseg4e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsseg4e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsseg4e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsseg4e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsseg4e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsseg4e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsseg4e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsseg4e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vsseg5e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     32.00  vsseg5e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     52.00  vsseg5e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     92.00  vsseg5e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     652.00 vsseg5e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     652.00 vsseg5e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     332.00 vsseg5e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vsseg5e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     32.00  vsseg5e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     52.00  vsseg5e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     332.00 vsseg5e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     332.00 vsseg5e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     172.00 vsseg5e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     172.00 vsseg5e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vsseg5e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     32.00  vsseg5e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     172.00 vsseg5e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     172.00 vsseg5e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     92.00  vsseg5e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     92.00  vsseg5e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     92.00  vsseg5e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vsseg5e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     92.00  vsseg5e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     92.00  vsseg5e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsseg6e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsseg6e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsseg6e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsseg6e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     780.00 vsseg6e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     780.00 vsseg6e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     396.00 vsseg6e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsseg6e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsseg6e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsseg6e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     396.00 vsseg6e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     396.00 vsseg6e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vsseg6e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vsseg6e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsseg6e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsseg6e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vsseg6e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vsseg6e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsseg6e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsseg6e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsseg6e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsseg6e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsseg6e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsseg6e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vsseg7e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     40.00  vsseg7e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     68.00  vsseg7e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     124.00 vsseg7e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     908.00 vsseg7e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     908.00 vsseg7e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     460.00 vsseg7e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vsseg7e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     40.00  vsseg7e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     68.00  vsseg7e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     460.00 vsseg7e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     460.00 vsseg7e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     236.00 vsseg7e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     236.00 vsseg7e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vsseg7e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     40.00  vsseg7e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     236.00 vsseg7e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     236.00 vsseg7e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     124.00 vsseg7e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     124.00 vsseg7e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     124.00 vsseg7e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vsseg7e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     124.00 vsseg7e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     124.00 vsseg7e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsseg8e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsseg8e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsseg8e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsseg8e8.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1036.00 vsseg8e8.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1036.00 vsseg8e8.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     524.00 vsseg8e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsseg8e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsseg8e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsseg8e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     524.00 vsseg8e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     524.00 vsseg8e16.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vsseg8e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vsseg8e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsseg8e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsseg8e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vsseg8e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vsseg8e32.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsseg8e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsseg8e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsseg8e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsseg8e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsseg8e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsseg8e64.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vlsseg2e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlsseg2e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlsseg2e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlsseg2e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlsseg2e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlsseg2e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlsseg2e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlsseg2e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vlsseg2e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlsseg2e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlsseg2e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlsseg2e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlsseg2e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlsseg2e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlsseg2e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlsseg2e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vlsseg2e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlsseg2e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlsseg2e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlsseg2e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlsseg2e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlsseg2e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlsseg2e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlsseg2e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vlsseg2e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlsseg2e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlsseg2e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlsseg2e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vlsseg3e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlsseg3e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlsseg3e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlsseg3e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlsseg3e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vlsseg3e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vlsseg3e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlsseg3e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vlsseg3e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlsseg3e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlsseg3e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlsseg3e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlsseg3e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlsseg3e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlsseg3e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlsseg3e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vlsseg3e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlsseg3e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlsseg3e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlsseg3e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlsseg3e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlsseg3e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlsseg3e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlsseg3e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vlsseg3e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlsseg3e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlsseg3e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlsseg3e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlsseg4e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlsseg4e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlsseg4e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlsseg4e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlsseg4e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vlsseg4e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vlsseg4e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlsseg4e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlsseg4e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlsseg4e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlsseg4e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlsseg4e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlsseg4e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlsseg4e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlsseg4e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlsseg4e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlsseg4e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlsseg4e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlsseg4e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlsseg4e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlsseg4e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlsseg4e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlsseg4e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlsseg4e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlsseg4e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlsseg4e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlsseg4e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlsseg4e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vlsseg5e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vlsseg5e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     52.00   -     vlsseg5e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vlsseg5e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     652.00  -     vlsseg5e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     652.00  -     vlsseg5e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     332.00  -     vlsseg5e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vlsseg5e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vlsseg5e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     52.00   -     vlsseg5e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     332.00  -     vlsseg5e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     332.00  -     vlsseg5e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     172.00  -     vlsseg5e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     172.00  -     vlsseg5e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vlsseg5e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vlsseg5e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     172.00  -     vlsseg5e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     172.00  -     vlsseg5e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vlsseg5e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vlsseg5e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vlsseg5e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vlsseg5e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vlsseg5e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vlsseg5e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlsseg6e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlsseg6e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlsseg6e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlsseg6e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     780.00  -     vlsseg6e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     780.00  -     vlsseg6e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vlsseg6e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlsseg6e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlsseg6e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlsseg6e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vlsseg6e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vlsseg6e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlsseg6e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlsseg6e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlsseg6e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlsseg6e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlsseg6e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlsseg6e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlsseg6e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlsseg6e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlsseg6e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlsseg6e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlsseg6e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlsseg6e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vlsseg7e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vlsseg7e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     68.00   -     vlsseg7e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vlsseg7e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     908.00  -     vlsseg7e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     908.00  -     vlsseg7e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     460.00  -     vlsseg7e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vlsseg7e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vlsseg7e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     68.00   -     vlsseg7e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     460.00  -     vlsseg7e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     460.00  -     vlsseg7e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     236.00  -     vlsseg7e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     236.00  -     vlsseg7e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vlsseg7e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vlsseg7e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     236.00  -     vlsseg7e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     236.00  -     vlsseg7e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vlsseg7e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vlsseg7e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vlsseg7e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vlsseg7e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vlsseg7e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vlsseg7e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlsseg8e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlsseg8e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlsseg8e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlsseg8e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1036.00  -    vlsseg8e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1036.00  -    vlsseg8e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vlsseg8e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlsseg8e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlsseg8e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlsseg8e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vlsseg8e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vlsseg8e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlsseg8e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlsseg8e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlsseg8e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlsseg8e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlsseg8e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlsseg8e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlsseg8e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlsseg8e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlsseg8e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlsseg8e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlsseg8e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlsseg8e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vssseg2e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vssseg2e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vssseg2e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vssseg2e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vssseg2e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vssseg2e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vssseg2e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vssseg2e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vssseg2e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vssseg2e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vssseg2e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vssseg2e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vssseg2e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vssseg2e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vssseg2e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vssseg2e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vssseg2e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vssseg2e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vssseg2e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vssseg2e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vssseg2e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vssseg2e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vssseg2e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vssseg2e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vssseg2e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vssseg2e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vssseg2e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vssseg2e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vssseg3e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vssseg3e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vssseg3e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vssseg3e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vssseg3e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     396.00 vssseg3e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     396.00 vssseg3e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vssseg3e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vssseg3e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vssseg3e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vssseg3e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vssseg3e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vssseg3e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vssseg3e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vssseg3e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vssseg3e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vssseg3e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vssseg3e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vssseg3e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vssseg3e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vssseg3e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vssseg3e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vssseg3e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vssseg3e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vssseg3e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vssseg3e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vssseg3e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vssseg3e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vssseg4e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vssseg4e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vssseg4e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vssseg4e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vssseg4e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     524.00 vssseg4e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     524.00 vssseg4e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vssseg4e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vssseg4e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vssseg4e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vssseg4e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vssseg4e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vssseg4e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vssseg4e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vssseg4e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vssseg4e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vssseg4e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vssseg4e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vssseg4e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vssseg4e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vssseg4e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vssseg4e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vssseg4e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vssseg4e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vssseg4e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vssseg4e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vssseg4e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vssseg4e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vssseg5e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     32.00  vssseg5e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     52.00  vssseg5e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     92.00  vssseg5e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     652.00 vssseg5e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     652.00 vssseg5e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     332.00 vssseg5e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vssseg5e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     32.00  vssseg5e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     52.00  vssseg5e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     332.00 vssseg5e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     332.00 vssseg5e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     172.00 vssseg5e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     172.00 vssseg5e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vssseg5e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     32.00  vssseg5e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     172.00 vssseg5e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     172.00 vssseg5e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     92.00  vssseg5e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     92.00  vssseg5e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     92.00  vssseg5e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vssseg5e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     92.00  vssseg5e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     92.00  vssseg5e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vssseg6e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vssseg6e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vssseg6e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vssseg6e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     780.00 vssseg6e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     780.00 vssseg6e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     396.00 vssseg6e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vssseg6e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vssseg6e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vssseg6e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     396.00 vssseg6e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     396.00 vssseg6e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vssseg6e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vssseg6e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vssseg6e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vssseg6e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vssseg6e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vssseg6e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vssseg6e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vssseg6e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vssseg6e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vssseg6e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vssseg6e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vssseg6e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vssseg7e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     40.00  vssseg7e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     68.00  vssseg7e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     124.00 vssseg7e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     908.00 vssseg7e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     908.00 vssseg7e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     460.00 vssseg7e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vssseg7e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     40.00  vssseg7e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     68.00  vssseg7e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     460.00 vssseg7e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     460.00 vssseg7e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     236.00 vssseg7e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     236.00 vssseg7e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vssseg7e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     40.00  vssseg7e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     236.00 vssseg7e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     236.00 vssseg7e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     124.00 vssseg7e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     124.00 vssseg7e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     124.00 vssseg7e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vssseg7e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     124.00 vssseg7e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     124.00 vssseg7e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vssseg8e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vssseg8e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vssseg8e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vssseg8e8.v  v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1036.00 vssseg8e8.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1036.00 vssseg8e8.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     524.00 vssseg8e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vssseg8e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vssseg8e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vssseg8e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     524.00 vssseg8e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     524.00 vssseg8e16.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vssseg8e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vssseg8e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vssseg8e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vssseg8e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vssseg8e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vssseg8e32.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vssseg8e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vssseg8e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vssseg8e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vssseg8e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vssseg8e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vssseg8e64.v v8, (a0), a1
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vlseg2e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg2e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg2e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg2e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg2e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg2e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlseg2e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg2e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vlseg2e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg2e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg2e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg2e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg2e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg2e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg2e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg2e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vlseg2e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg2e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg2e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg2e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg2e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg2e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg2e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg2e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vlseg2e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg2e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg2e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg2e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vlseg3e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg3e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlseg3e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg3e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg3e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vlseg3e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vlseg3e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlseg3e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vlseg3e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg3e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlseg3e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg3e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlseg3e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlseg3e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg3e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg3e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vlseg3e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg3e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlseg3e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg3e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg3e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg3e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg3e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg3e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vlseg3e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg3e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg3e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg3e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg4e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg4e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg4e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg4e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg4e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vlseg4e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vlseg4e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlseg4e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg4e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg4e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg4e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg4e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlseg4e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlseg4e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg4e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg4e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg4e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg4e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg4e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg4e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg4e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg4e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg4e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg4e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vlseg4e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg4e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg4e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg4e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vlseg5e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vlseg5e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     52.00   -     vlseg5e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vlseg5e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     652.00  -     vlseg5e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     652.00  -     vlseg5e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     332.00  -     vlseg5e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vlseg5e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vlseg5e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     52.00   -     vlseg5e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     332.00  -     vlseg5e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     332.00  -     vlseg5e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     172.00  -     vlseg5e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     172.00  -     vlseg5e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vlseg5e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vlseg5e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     172.00  -     vlseg5e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     172.00  -     vlseg5e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vlseg5e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vlseg5e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vlseg5e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vlseg5e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vlseg5e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vlseg5e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg6e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlseg6e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg6e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg6e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     780.00  -     vlseg6e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     780.00  -     vlseg6e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vlseg6e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg6e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlseg6e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vlseg6e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vlseg6e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vlseg6e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlseg6e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlseg6e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg6e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vlseg6e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlseg6e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vlseg6e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg6e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg6e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg6e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vlseg6e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg6e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vlseg6e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vlseg7e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vlseg7e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     68.00   -     vlseg7e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vlseg7e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     908.00  -     vlseg7e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     908.00  -     vlseg7e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     460.00  -     vlseg7e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vlseg7e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vlseg7e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     68.00   -     vlseg7e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     460.00  -     vlseg7e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     460.00  -     vlseg7e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     236.00  -     vlseg7e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     236.00  -     vlseg7e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vlseg7e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vlseg7e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     236.00  -     vlseg7e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     236.00  -     vlseg7e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vlseg7e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vlseg7e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vlseg7e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vlseg7e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vlseg7e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vlseg7e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg8e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg8e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg8e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg8e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1036.00  -    vlseg8e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1036.00  -    vlseg8e8ff.v v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vlseg8e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg8e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg8e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vlseg8e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vlseg8e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vlseg8e16ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlseg8e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlseg8e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg8e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vlseg8e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlseg8e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vlseg8e32ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg8e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg8e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg8e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vlseg8e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg8e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vlseg8e64ff.v  v8, (a0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vluxseg2ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vluxseg2ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg2ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vluxseg2ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vluxseg2ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vluxseg2ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vluxseg2ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vluxseg2ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vluxseg2ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg2ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg2ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg2ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg2ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vluxseg2ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vluxseg2ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vluxseg2ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vluxseg2ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vluxseg2ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vluxseg2ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg2ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vluxseg2ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vluxseg2ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vluxseg2ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vluxseg2ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vluxseg2ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vluxseg2ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg2ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vluxseg2ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vluxseg3ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vluxseg3ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vluxseg3ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vluxseg3ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vluxseg3ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vluxseg3ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vluxseg3ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vluxseg3ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vluxseg3ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vluxseg3ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vluxseg3ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vluxseg3ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vluxseg3ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vluxseg3ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vluxseg3ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vluxseg3ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vluxseg3ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vluxseg3ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vluxseg3ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vluxseg3ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vluxseg3ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vluxseg3ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vluxseg3ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vluxseg3ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vluxseg3ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vluxseg3ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vluxseg3ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vluxseg3ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vluxseg4ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg4ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vluxseg4ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vluxseg4ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vluxseg4ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vluxseg4ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vluxseg4ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vluxseg4ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg4ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vluxseg4ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vluxseg4ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vluxseg4ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vluxseg4ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vluxseg4ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vluxseg4ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg4ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg4ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg4ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg4ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vluxseg4ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vluxseg4ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vluxseg4ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vluxseg4ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vluxseg4ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vluxseg4ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg4ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vluxseg4ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vluxseg4ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vluxseg5ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vluxseg5ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     52.00   -     vluxseg5ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vluxseg5ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     652.00  -     vluxseg5ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     652.00  -     vluxseg5ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vluxseg5ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vluxseg5ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     52.00   -     vluxseg5ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     52.00   -     vluxseg5ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     332.00  -     vluxseg5ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     332.00  -     vluxseg5ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vluxseg5ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vluxseg5ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vluxseg5ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vluxseg5ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     172.00  -     vluxseg5ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     172.00  -     vluxseg5ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vluxseg5ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vluxseg5ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vluxseg5ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vluxseg5ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vluxseg5ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vluxseg5ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vluxseg6ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vluxseg6ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vluxseg6ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vluxseg6ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     780.00  -     vluxseg6ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     780.00  -     vluxseg6ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vluxseg6ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vluxseg6ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vluxseg6ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vluxseg6ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vluxseg6ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vluxseg6ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vluxseg6ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vluxseg6ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vluxseg6ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vluxseg6ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vluxseg6ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vluxseg6ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vluxseg6ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vluxseg6ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vluxseg6ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vluxseg6ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vluxseg6ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vluxseg6ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vluxseg7ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vluxseg7ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     68.00   -     vluxseg7ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vluxseg7ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     908.00  -     vluxseg7ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     908.00  -     vluxseg7ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vluxseg7ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vluxseg7ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     68.00   -     vluxseg7ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     68.00   -     vluxseg7ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     460.00  -     vluxseg7ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     460.00  -     vluxseg7ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vluxseg7ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vluxseg7ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vluxseg7ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vluxseg7ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     236.00  -     vluxseg7ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     236.00  -     vluxseg7ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vluxseg7ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vluxseg7ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vluxseg7ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vluxseg7ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vluxseg7ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vluxseg7ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg8ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vluxseg8ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vluxseg8ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vluxseg8ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1036.00  -    vluxseg8ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1036.00  -    vluxseg8ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg8ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vluxseg8ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vluxseg8ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vluxseg8ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vluxseg8ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vluxseg8ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg8ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vluxseg8ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vluxseg8ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vluxseg8ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vluxseg8ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vluxseg8ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg8ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg8ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg8ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vluxseg8ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vluxseg8ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vluxseg8ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vloxseg2ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vloxseg2ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg2ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vloxseg2ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vloxseg2ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vloxseg2ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vloxseg2ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vloxseg2ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vloxseg2ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg2ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg2ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg2ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg2ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vloxseg2ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vloxseg2ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vloxseg2ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vloxseg2ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vloxseg2ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vloxseg2ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg2ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vloxseg2ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vloxseg2ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vloxseg2ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vloxseg2ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     16.00   -     vloxseg2ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vloxseg2ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg2ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vloxseg2ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vloxseg3ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vloxseg3ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vloxseg3ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vloxseg3ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vloxseg3ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vloxseg3ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vloxseg3ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vloxseg3ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vloxseg3ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vloxseg3ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vloxseg3ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vloxseg3ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vloxseg3ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vloxseg3ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vloxseg3ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vloxseg3ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vloxseg3ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vloxseg3ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vloxseg3ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vloxseg3ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vloxseg3ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vloxseg3ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vloxseg3ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vloxseg3ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     18.00   -     vloxseg3ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vloxseg3ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vloxseg3ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vloxseg3ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vloxseg4ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg4ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vloxseg4ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vloxseg4ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vloxseg4ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vloxseg4ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vloxseg4ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vloxseg4ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg4ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vloxseg4ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vloxseg4ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vloxseg4ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vloxseg4ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vloxseg4ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vloxseg4ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg4ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg4ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg4ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg4ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vloxseg4ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vloxseg4ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vloxseg4ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vloxseg4ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vloxseg4ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     20.00   -     vloxseg4ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg4ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vloxseg4ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vloxseg4ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vloxseg5ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vloxseg5ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     52.00   -     vloxseg5ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vloxseg5ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     652.00  -     vloxseg5ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     652.00  -     vloxseg5ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vloxseg5ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vloxseg5ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     52.00   -     vloxseg5ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     52.00   -     vloxseg5ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     332.00  -     vloxseg5ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     332.00  -     vloxseg5ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vloxseg5ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vloxseg5ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vloxseg5ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     32.00   -     vloxseg5ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     172.00  -     vloxseg5ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     172.00  -     vloxseg5ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vloxseg5ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vloxseg5ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vloxseg5ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     22.00   -     vloxseg5ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vloxseg5ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     92.00   -     vloxseg5ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vloxseg6ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vloxseg6ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vloxseg6ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vloxseg6ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     780.00  -     vloxseg6ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     780.00  -     vloxseg6ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vloxseg6ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vloxseg6ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vloxseg6ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     60.00   -     vloxseg6ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vloxseg6ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     396.00  -     vloxseg6ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vloxseg6ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vloxseg6ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vloxseg6ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     36.00   -     vloxseg6ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vloxseg6ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     204.00  -     vloxseg6ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vloxseg6ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vloxseg6ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vloxseg6ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     24.00   -     vloxseg6ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vloxseg6ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     108.00  -     vloxseg6ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vloxseg7ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vloxseg7ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     68.00   -     vloxseg7ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vloxseg7ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     908.00  -     vloxseg7ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     908.00  -     vloxseg7ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vloxseg7ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vloxseg7ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     68.00   -     vloxseg7ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     68.00   -     vloxseg7ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     460.00  -     vloxseg7ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     460.00  -     vloxseg7ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vloxseg7ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vloxseg7ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vloxseg7ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     40.00   -     vloxseg7ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     236.00  -     vloxseg7ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     236.00  -     vloxseg7ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vloxseg7ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vloxseg7ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vloxseg7ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     26.00   -     vloxseg7ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vloxseg7ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     124.00  -     vloxseg7ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg8ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vloxseg8ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vloxseg8ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vloxseg8ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1036.00  -    vloxseg8ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1036.00  -    vloxseg8ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg8ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vloxseg8ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vloxseg8ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     76.00   -     vloxseg8ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vloxseg8ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     524.00  -     vloxseg8ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg8ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vloxseg8ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vloxseg8ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     44.00   -     vloxseg8ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vloxseg8ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     268.00  -     vloxseg8ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg8ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg8ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg8ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     28.00   -     vloxseg8ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vloxseg8ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     140.00  -     vloxseg8ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsuxseg2ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsuxseg2ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsuxseg2ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsuxseg2ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsuxseg2ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsuxseg2ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vsuxseg2ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     14.00  vsuxseg2ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsuxseg2ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsuxseg2ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsuxseg2ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsuxseg2ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsuxseg2ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsuxseg2ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg2ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     14.00  vsuxseg2ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsuxseg2ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsuxseg2ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsuxseg2ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsuxseg2ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsuxseg2ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg2ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg2ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     14.00  vsuxseg2ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsuxseg2ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsuxseg2ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsuxseg2ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsuxseg2ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsuxseg3ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsuxseg3ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsuxseg3ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsuxseg3ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsuxseg3ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     396.00 vsuxseg3ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     396.00 vsuxseg3ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     15.00  vsuxseg3ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsuxseg3ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsuxseg3ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsuxseg3ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsuxseg3ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vsuxseg3ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vsuxseg3ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg3ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     15.00  vsuxseg3ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsuxseg3ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsuxseg3ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsuxseg3ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsuxseg3ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsuxseg3ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg3ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg3ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     15.00  vsuxseg3ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsuxseg3ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsuxseg3ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsuxseg3ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsuxseg3ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsuxseg4ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsuxseg4ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsuxseg4ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsuxseg4ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsuxseg4ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     524.00 vsuxseg4ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     524.00 vsuxseg4ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsuxseg4ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsuxseg4ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsuxseg4ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsuxseg4ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsuxseg4ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vsuxseg4ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vsuxseg4ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg4ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsuxseg4ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsuxseg4ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsuxseg4ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsuxseg4ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsuxseg4ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsuxseg4ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg4ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg4ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsuxseg4ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsuxseg4ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsuxseg4ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsuxseg4ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsuxseg4ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vsuxseg5ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     32.00  vsuxseg5ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     52.00  vsuxseg5ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     92.00  vsuxseg5ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     652.00 vsuxseg5ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     652.00 vsuxseg5ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     17.00  vsuxseg5ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vsuxseg5ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     32.00  vsuxseg5ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     52.00  vsuxseg5ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     332.00 vsuxseg5ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     332.00 vsuxseg5ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg5ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     17.00  vsuxseg5ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vsuxseg5ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     32.00  vsuxseg5ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     172.00 vsuxseg5ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     172.00 vsuxseg5ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg5ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg5ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     17.00  vsuxseg5ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vsuxseg5ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     92.00  vsuxseg5ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     92.00  vsuxseg5ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsuxseg6ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsuxseg6ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsuxseg6ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsuxseg6ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     780.00 vsuxseg6ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     780.00 vsuxseg6ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsuxseg6ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsuxseg6ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsuxseg6ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsuxseg6ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     396.00 vsuxseg6ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     396.00 vsuxseg6ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg6ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsuxseg6ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsuxseg6ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsuxseg6ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vsuxseg6ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vsuxseg6ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg6ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg6ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsuxseg6ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsuxseg6ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsuxseg6ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsuxseg6ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vsuxseg7ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     40.00  vsuxseg7ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     68.00  vsuxseg7ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     124.00 vsuxseg7ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     908.00 vsuxseg7ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     908.00 vsuxseg7ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     19.00  vsuxseg7ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vsuxseg7ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     40.00  vsuxseg7ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     68.00  vsuxseg7ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     460.00 vsuxseg7ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     460.00 vsuxseg7ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg7ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     19.00  vsuxseg7ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vsuxseg7ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     40.00  vsuxseg7ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     236.00 vsuxseg7ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     236.00 vsuxseg7ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg7ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg7ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     19.00  vsuxseg7ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vsuxseg7ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     124.00 vsuxseg7ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     124.00 vsuxseg7ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsuxseg8ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsuxseg8ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsuxseg8ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsuxseg8ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1036.00 vsuxseg8ei8.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1036.00 vsuxseg8ei8.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsuxseg8ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsuxseg8ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsuxseg8ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsuxseg8ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     524.00 vsuxseg8ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     524.00 vsuxseg8ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg8ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsuxseg8ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsuxseg8ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsuxseg8ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vsuxseg8ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vsuxseg8ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg8ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsuxseg8ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsuxseg8ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsuxseg8ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsuxseg8ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsuxseg8ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsoxseg2ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsoxseg2ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsoxseg2ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsoxseg2ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsoxseg2ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsoxseg2ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vsoxseg2ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     14.00  vsoxseg2ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsoxseg2ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsoxseg2ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsoxseg2ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsoxseg2ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsoxseg2ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsoxseg2ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg2ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     14.00  vsoxseg2ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsoxseg2ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsoxseg2ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsoxseg2ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsoxseg2ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsoxseg2ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg2ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg2ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     14.00  vsoxseg2ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsoxseg2ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsoxseg2ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsoxseg2ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsoxseg2ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsoxseg3ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsoxseg3ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsoxseg3ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsoxseg3ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsoxseg3ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     396.00 vsoxseg3ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     396.00 vsoxseg3ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     15.00  vsoxseg3ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsoxseg3ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsoxseg3ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsoxseg3ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsoxseg3ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vsoxseg3ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vsoxseg3ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg3ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     15.00  vsoxseg3ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsoxseg3ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsoxseg3ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsoxseg3ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsoxseg3ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsoxseg3ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg3ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg3ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     15.00  vsoxseg3ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsoxseg3ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsoxseg3ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsoxseg3ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsoxseg3ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsoxseg4ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsoxseg4ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsoxseg4ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsoxseg4ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsoxseg4ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     524.00 vsoxseg4ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     524.00 vsoxseg4ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsoxseg4ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsoxseg4ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsoxseg4ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsoxseg4ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsoxseg4ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vsoxseg4ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vsoxseg4ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg4ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsoxseg4ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsoxseg4ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsoxseg4ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsoxseg4ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsoxseg4ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsoxseg4ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg4ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg4ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     16.00  vsoxseg4ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsoxseg4ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsoxseg4ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsoxseg4ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsoxseg4ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vsoxseg5ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     32.00  vsoxseg5ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     52.00  vsoxseg5ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     92.00  vsoxseg5ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     652.00 vsoxseg5ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     652.00 vsoxseg5ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     17.00  vsoxseg5ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vsoxseg5ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     32.00  vsoxseg5ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     52.00  vsoxseg5ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     332.00 vsoxseg5ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     332.00 vsoxseg5ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg5ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     17.00  vsoxseg5ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vsoxseg5ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     32.00  vsoxseg5ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     172.00 vsoxseg5ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     172.00 vsoxseg5ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg5ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg5ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     17.00  vsoxseg5ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     22.00  vsoxseg5ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     92.00  vsoxseg5ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     92.00  vsoxseg5ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsoxseg6ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsoxseg6ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsoxseg6ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsoxseg6ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     780.00 vsoxseg6ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     780.00 vsoxseg6ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsoxseg6ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsoxseg6ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsoxseg6ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     60.00  vsoxseg6ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     396.00 vsoxseg6ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     396.00 vsoxseg6ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg6ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsoxseg6ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsoxseg6ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     36.00  vsoxseg6ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vsoxseg6ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     204.00 vsoxseg6ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg6ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg6ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     18.00  vsoxseg6ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     24.00  vsoxseg6ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsoxseg6ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     108.00 vsoxseg6ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vsoxseg7ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     40.00  vsoxseg7ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     68.00  vsoxseg7ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     124.00 vsoxseg7ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     908.00 vsoxseg7ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     908.00 vsoxseg7ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     19.00  vsoxseg7ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vsoxseg7ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     40.00  vsoxseg7ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     68.00  vsoxseg7ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     460.00 vsoxseg7ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     460.00 vsoxseg7ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg7ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     19.00  vsoxseg7ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vsoxseg7ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     40.00  vsoxseg7ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     236.00 vsoxseg7ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     236.00 vsoxseg7ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg7ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg7ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     19.00  vsoxseg7ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     26.00  vsoxseg7ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     124.00 vsoxseg7ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     124.00 vsoxseg7ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsoxseg8ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsoxseg8ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsoxseg8ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsoxseg8ei8.v  v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1036.00 vsoxseg8ei8.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1036.00 vsoxseg8ei8.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsoxseg8ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsoxseg8ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsoxseg8ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     76.00  vsoxseg8ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     524.00 vsoxseg8ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     524.00 vsoxseg8ei16.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg8ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsoxseg8ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsoxseg8ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     44.00  vsoxseg8ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vsoxseg8ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     268.00 vsoxseg8ei32.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg8ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     12.00  vsoxseg8ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     20.00  vsoxseg8ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     28.00  vsoxseg8ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsoxseg8ei64.v v8, (a0), v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     140.00 vsoxseg8ei64.v v8, (a0), v16
diff --git a/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/vmv.s b/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/vmv.s
new file mode 100644
index 000000000000000..31178e8e238f1ad
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/vmv.s
@@ -0,0 +1,895 @@
+
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=riscv64 -mcpu=sifive-p470 -iterations=1 < %s | FileCheck %s
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e8, m1, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e8, m1, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e8, m2, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e8, m4, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e8, m8, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e16, m1, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e16, m1, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e16, m2, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e16, m4, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e16, m8, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e32, m1, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e32, m1, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e32, m2, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e32, m4, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e32, m8, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e64, m1, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e64, m1, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e64, m2, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e64, m4, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e64, m8, tu, mu
+vmv1r.v v8, v16
+vsetvli zero, zero, e8, mf8, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e8, m1, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e8, m1, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e8, m2, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e8, m4, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e8, m8, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e16, m1, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e16, m1, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e16, m2, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e16, m4, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e16, m8, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e32, m1, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e32, m1, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e32, m2, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e32, m4, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e32, m8, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e64, m1, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e64, m1, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e64, m2, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e64, m4, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e64, m8, tu, mu
+vmv2r.v v8, v16
+vsetvli zero, zero, e8, mf8, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e8, m1, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e8, m1, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e8, m2, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e8, m4, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e8, m8, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e16, m1, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e16, m1, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e16, m2, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e16, m4, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e16, m8, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e32, m1, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e32, m1, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e32, m2, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e32, m4, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e32, m8, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e64, m1, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e64, m1, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e64, m2, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e64, m4, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e64, m8, tu, mu
+vmv4r.v v8, v16
+vsetvli zero, zero, e8, mf8, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e8, mf4, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e8, mf2, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e8, m1, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e8, m1, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e8, m2, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e8, m4, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e8, m8, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e16, mf8, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e16, mf4, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e16, mf2, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e16, m1, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e16, m1, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e16, m2, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e16, m4, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e16, m8, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e32, mf8, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e32, mf4, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e32, mf2, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e32, m1, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e32, m1, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e32, m2, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e32, m4, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e32, m8, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e64, mf8, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e64, mf4, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e64, mf2, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e64, m1, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e64, m1, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e64, m2, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e64, m4, tu, mu
+vmv8r.v v8, v16
+vsetvli zero, zero, e64, m8, tu, mu
+vmv8r.v v8, v16
+
+vsetvli zero, zero, e64, m1, tu, mu
+vmv.s.x v8, x5
+vmv.x.s x7, v16
+
+vsetvli zero, zero, e64, m2, tu, mu
+vmv.s.x v8, x5
+vmv.x.s x7, v16
+
+vsetvli zero, zero, e64, m4, tu, mu
+vmv.s.x v8, x5
+vmv.x.s x7, v16
+
+vsetvli zero, zero, e64, m8, tu, mu
+vmv.s.x v8, x5
+vmv.x.s x7, v16
+
+vsetvli zero, zero, e64, m1, tu, mu
+vfmv.s.f v8, f5
+vfmv.f.s f7, v16
+
+vsetvli zero, zero, e64, m2, tu, mu
+vfmv.s.f v8, f5
+vfmv.f.s f7, v16
+
+vsetvli zero, zero, e64, m4, tu, mu
+vfmv.s.f v8, f5
+vfmv.f.s f7, v16
+
+vsetvli zero, zero, e64, m8, tu, mu
+vfmv.s.f v8, f5
+vfmv.f.s f7, v16
+
+# CHECK:      Iterations:        1
+# CHECK-NEXT: Instructions:      280
+# CHECK-NEXT: Total Cycles:      523
+# CHECK-NEXT: Total uOps:        280
+
+# CHECK:      Dispatch Width:    3
+# CHECK-NEXT: uOps Per Cycle:    0.54
+# CHECK-NEXT: IPC:               0.54
+# CHECK-NEXT: Block RThroughput: 512.0
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vmv1r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv2r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vmv4r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vmv8r.v v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv.s.x v8, t0
+# CHECK-NEXT:  1      2     2.00                        vmv.x.s t2, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv.s.x v8, t0
+# CHECK-NEXT:  1      2     2.00                        vmv.x.s t2, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv.s.x v8, t0
+# CHECK-NEXT:  1      2     2.00                        vmv.x.s t2, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vmv.s.x v8, t0
+# CHECK-NEXT:  1      2     2.00                        vmv.x.s t2, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      6     2.00                        vfmv.s.f  v8, ft5
+# CHECK-NEXT:  1      6     2.00                        vfmv.f.s  ft7, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      6     2.00                        vfmv.s.f  v8, ft5
+# CHECK-NEXT:  1      6     2.00                        vfmv.f.s  ft7, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      6     2.00                        vfmv.s.f  v8, ft5
+# CHECK-NEXT:  1      6     2.00                        vfmv.f.s  ft7, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      6     2.00                        vfmv.s.f  v8, ft5
+# CHECK-NEXT:  1      6     2.00                        vfmv.f.s  ft7, v16
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - SiFiveP400Div
+# CHECK-NEXT: [1]   - SiFiveP400FEXQ0
+# CHECK-NEXT: [2]   - SiFiveP400FloatDiv
+# CHECK-NEXT: [3]   - SiFiveP400IEXQ0
+# CHECK-NEXT: [4]   - SiFiveP400IEXQ1
+# CHECK-NEXT: [5]   - SiFiveP400IEXQ2
+# CHECK-NEXT: [6]   - SiFiveP400Load
+# CHECK-NEXT: [7]   - SiFiveP400Store
+# CHECK-NEXT: [8]   - SiFiveP400VDiv
+# CHECK-NEXT: [9]   - SiFiveP400VEXQ0
+# CHECK-NEXT: [10]  - SiFiveP400VFloatDiv
+# CHECK-NEXT: [11]  - SiFiveP400VLD
+# CHECK-NEXT: [12]  - SiFiveP400VST
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]
+# CHECK-NEXT:  -      -      -      -     136.00  -      -      -      -     512.00  -      -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   Instructions:
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vmv1r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv2r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vmv4r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmv8r.v  v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv.s.x  v8, t0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv.x.s  t2, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv.s.x  v8, t0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv.x.s  t2, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv.s.x  v8, t0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv.x.s  t2, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv.s.x  v8, t0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vmv.x.s  t2, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vfmv.s.f v8, ft5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vfmv.f.s ft7, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vfmv.s.f v8, ft5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vfmv.f.s ft7, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vfmv.s.f v8, ft5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vfmv.f.s ft7, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vfmv.s.f v8, ft5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vfmv.f.s ft7, v16
+
diff --git a/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/vreduce.s b/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/vreduce.s
new file mode 100644
index 000000000000000..0fc0bf03d7ecb27
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/vreduce.s
@@ -0,0 +1,438 @@
+
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=riscv64 -mcpu=sifive-p470 -iterations=1 < %s | FileCheck %s
+
+# Simple integer reductions: varies by LMUL
+vsetvli zero, zero, e32, m1, tu, mu
+vredsum.vs v5, v7, v8
+
+vsetvli zero, zero, e32, m2, tu, mu
+vredsum.vs v5, v7, v8
+
+vsetvli zero, zero, e32, m4, tu, mu
+vredsum.vs v5, v7, v8
+
+vsetvli zero, zero, e32, m8, tu, mu
+vredsum.vs v5, v7, v8
+
+# Advanced integer reductions: varies by LMUL and SEW
+vsetvli zero, zero, e16, m1, tu, mu
+vredmin.vs v5, v7, v8
+vredmax.vs v5, v7, v8
+
+vsetvli zero, zero, e16, m2, tu, mu
+vredmin.vs v5, v7, v8
+vredmax.vs v5, v7, v8
+
+vsetvli zero, zero, e16, m4, tu, mu
+vredmin.vs v5, v7, v8
+vredmax.vs v5, v7, v8
+
+vsetvli zero, zero, e16, m8, tu, mu
+vredmin.vs v5, v7, v8
+vredmax.vs v5, v7, v8
+
+vsetvli zero, zero, e32, m1, tu, mu
+vredmin.vs v5, v7, v8
+vredmax.vs v5, v7, v8
+
+vsetvli zero, zero, e32, m2, tu, mu
+vredmin.vs v5, v7, v8
+vredmax.vs v5, v7, v8
+
+vsetvli zero, zero, e32, m4, tu, mu
+vredmin.vs v5, v7, v8
+vredmax.vs v5, v7, v8
+
+vsetvli zero, zero, e32, m8, tu, mu
+vredmin.vs v5, v7, v8
+vredmax.vs v5, v7, v8
+
+vsetvli zero, zero, e64, m1, tu, mu
+vredmin.vs v5, v7, v8
+vredmax.vs v5, v7, v8
+
+vsetvli zero, zero, e64, m2, tu, mu
+vredmin.vs v5, v7, v8
+vredmax.vs v5, v7, v8
+
+vsetvli zero, zero, e64, m4, tu, mu
+vredmin.vs v5, v7, v8
+vredmax.vs v5, v7, v8
+
+vsetvli zero, zero, e64, m8, tu, mu
+vredmin.vs v5, v7, v8
+vredmax.vs v5, v7, v8
+
+# Simple floating point reductions: varies by LMUL and SEW
+vsetvli zero, zero, e16, m1, tu, mu
+vfredusum.vs v5, v7, v8
+vfredosum.vs v5, v7, v8
+
+vsetvli zero, zero, e16, m2, tu, mu
+vfredusum.vs v5, v7, v8
+vfredosum.vs v5, v7, v8
+
+vsetvli zero, zero, e16, m4, tu, mu
+vfredusum.vs v5, v7, v8
+vfredosum.vs v5, v7, v8
+
+vsetvli zero, zero, e16, m8, tu, mu
+vfredusum.vs v5, v7, v8
+vfredosum.vs v5, v7, v8
+
+vsetvli zero, zero, e32, m1, tu, mu
+vfredusum.vs v5, v7, v8
+vfredosum.vs v5, v7, v8
+
+vsetvli zero, zero, e32, m2, tu, mu
+vfredusum.vs v5, v7, v8
+vfredosum.vs v5, v7, v8
+
+vsetvli zero, zero, e32, m4, tu, mu
+vfredusum.vs v5, v7, v8
+vfredosum.vs v5, v7, v8
+
+vsetvli zero, zero, e32, m8, tu, mu
+vfredusum.vs v5, v7, v8
+vfredosum.vs v5, v7, v8
+
+vsetvli zero, zero, e64, m1, tu, mu
+vfredusum.vs v5, v7, v8
+vfredosum.vs v5, v7, v8
+
+vsetvli zero, zero, e64, m2, tu, mu
+vfredusum.vs v5, v7, v8
+vfredosum.vs v5, v7, v8
+
+vsetvli zero, zero, e64, m4, tu, mu
+vfredusum.vs v5, v7, v8
+vfredosum.vs v5, v7, v8
+
+vsetvli zero, zero, e64, m8, tu, mu
+vfredusum.vs v5, v7, v8
+vfredosum.vs v5, v7, v8
+
+# Advanced floating point reductions: varies by LMUL and SEW
+vsetvli zero, zero, e16, m1, tu, mu
+vfredmin.vs v5, v7, v8
+vfredmax.vs v5, v7, v8
+
+vsetvli zero, zero, e16, m2, tu, mu
+vfredmin.vs v5, v7, v8
+vfredmax.vs v5, v7, v8
+
+vsetvli zero, zero, e16, m4, tu, mu
+vfredmin.vs v5, v7, v8
+vfredmax.vs v5, v7, v8
+
+vsetvli zero, zero, e16, m8, tu, mu
+vfredmin.vs v5, v7, v8
+vfredmax.vs v5, v7, v8
+
+vsetvli zero, zero, e32, m1, tu, mu
+vfredmin.vs v5, v7, v8
+vfredmax.vs v5, v7, v8
+
+vsetvli zero, zero, e32, m2, tu, mu
+vfredmin.vs v5, v7, v8
+vfredmax.vs v5, v7, v8
+
+vsetvli zero, zero, e32, m4, tu, mu
+vfredmin.vs v5, v7, v8
+vfredmax.vs v5, v7, v8
+
+vsetvli zero, zero, e32, m8, tu, mu
+vfredmin.vs v5, v7, v8
+vfredmax.vs v5, v7, v8
+
+vsetvli zero, zero, e64, m1, tu, mu
+vfredmin.vs v5, v7, v8
+vfredmax.vs v5, v7, v8
+
+vsetvli zero, zero, e64, m2, tu, mu
+vfredmin.vs v5, v7, v8
+vfredmax.vs v5, v7, v8
+
+vsetvli zero, zero, e64, m4, tu, mu
+vfredmin.vs v5, v7, v8
+vfredmax.vs v5, v7, v8
+
+vsetvli zero, zero, e64, m8, tu, mu
+vfredmin.vs v5, v7, v8
+vfredmax.vs v5, v7, v8
+
+# CHECK:      Iterations:        1
+# CHECK-NEXT: Instructions:      116
+# CHECK-NEXT: Total Cycles:      921
+# CHECK-NEXT: Total uOps:        116
+
+# CHECK:      Dispatch Width:    3
+# CHECK-NEXT: uOps Per Cycle:    0.13
+# CHECK-NEXT: IPC:               0.13
+# CHECK-NEXT: Block RThroughput: 916.0
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vredsum.vs  v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      4     2.00                        vredsum.vs  v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      8     4.00                        vredsum.vs  v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      16    9.00                        vredsum.vs  v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      8     4.00                        vredmin.vs  v5, v7, v8
+# CHECK-NEXT:  1      8     4.00                        vredmax.vs  v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      10    5.00                        vredmin.vs  v5, v7, v8
+# CHECK-NEXT:  1      10    5.00                        vredmax.vs  v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      12    8.00                        vredmin.vs  v5, v7, v8
+# CHECK-NEXT:  1      12    8.00                        vredmax.vs  v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      15    14.00                       vredmin.vs  v5, v7, v8
+# CHECK-NEXT:  1      15    14.00                       vredmax.vs  v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      6     3.00                        vredmin.vs  v5, v7, v8
+# CHECK-NEXT:  1      6     3.00                        vredmax.vs  v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      8     4.00                        vredmin.vs  v5, v7, v8
+# CHECK-NEXT:  1      8     4.00                        vredmax.vs  v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      10    6.00                        vredmin.vs  v5, v7, v8
+# CHECK-NEXT:  1      10    6.00                        vredmax.vs  v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      13    12.00                       vredmin.vs  v5, v7, v8
+# CHECK-NEXT:  1      13    12.00                       vredmax.vs  v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      4     2.00                        vredmin.vs  v5, v7, v8
+# CHECK-NEXT:  1      4     2.00                        vredmax.vs  v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      6     3.00                        vredmin.vs  v5, v7, v8
+# CHECK-NEXT:  1      6     3.00                        vredmax.vs  v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      8     5.00                        vredmin.vs  v5, v7, v8
+# CHECK-NEXT:  1      8     5.00                        vredmax.vs  v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      11    10.00                       vredmin.vs  v5, v7, v8
+# CHECK-NEXT:  1      11    10.00                       vredmax.vs  v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      16    16.00                       vfredusum.vs  v5, v7, v8
+# CHECK-NEXT:  1      16    16.00                       vfredosum.vs  v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      22    16.00                       vfredusum.vs  v5, v7, v8
+# CHECK-NEXT:  1      32    32.00                       vfredosum.vs  v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      28    16.00                       vfredusum.vs  v5, v7, v8
+# CHECK-NEXT:  1      64    64.00                       vfredosum.vs  v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      34    16.00                       vfredusum.vs  v5, v7, v8
+# CHECK-NEXT:  1      128   128.00                      vfredosum.vs  v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      10    10.00                       vfredusum.vs  v5, v7, v8
+# CHECK-NEXT:  1      10    10.00                       vfredosum.vs  v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      16    10.00                       vfredusum.vs  v5, v7, v8
+# CHECK-NEXT:  1      20    20.00                       vfredosum.vs  v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      22    10.00                       vfredusum.vs  v5, v7, v8
+# CHECK-NEXT:  1      40    40.00                       vfredosum.vs  v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      28    10.00                       vfredusum.vs  v5, v7, v8
+# CHECK-NEXT:  1      80    80.00                       vfredosum.vs  v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      6     6.00                        vfredusum.vs  v5, v7, v8
+# CHECK-NEXT:  1      6     6.00                        vfredosum.vs  v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      12    6.00                        vfredusum.vs  v5, v7, v8
+# CHECK-NEXT:  1      12    12.00                       vfredosum.vs  v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      18    6.00                        vfredusum.vs  v5, v7, v8
+# CHECK-NEXT:  1      24    24.00                       vfredosum.vs  v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      24    6.00                        vfredusum.vs  v5, v7, v8
+# CHECK-NEXT:  1      48    48.00                       vfredosum.vs  v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  1      6     3.00                        vfredmin.vs v5, v7, v8
+# CHECK-NEXT:  1      6     3.00                        vfredmax.vs v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  1      8     4.00                        vfredmin.vs v5, v7, v8
+# CHECK-NEXT:  1      8     4.00                        vfredmax.vs v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      10    6.00                        vfredmin.vs v5, v7, v8
+# CHECK-NEXT:  1      10    6.00                        vfredmax.vs v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  1      13    12.00                       vfredmin.vs v5, v7, v8
+# CHECK-NEXT:  1      13    12.00                       vfredmax.vs v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      6     3.00                        vfredmin.vs v5, v7, v8
+# CHECK-NEXT:  1      6     3.00                        vfredmax.vs v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      8     4.00                        vfredmin.vs v5, v7, v8
+# CHECK-NEXT:  1      8     4.00                        vfredmax.vs v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      10    6.00                        vfredmin.vs v5, v7, v8
+# CHECK-NEXT:  1      10    6.00                        vfredmax.vs v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      13    12.00                       vfredmin.vs v5, v7, v8
+# CHECK-NEXT:  1      13    12.00                       vfredmax.vs v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      4     2.00                        vfredmin.vs v5, v7, v8
+# CHECK-NEXT:  1      4     2.00                        vfredmax.vs v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      6     3.00                        vfredmin.vs v5, v7, v8
+# CHECK-NEXT:  1      6     3.00                        vfredmax.vs v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      8     5.00                        vfredmin.vs v5, v7, v8
+# CHECK-NEXT:  1      8     5.00                        vfredmax.vs v5, v7, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      11    10.00                       vfredmin.vs v5, v7, v8
+# CHECK-NEXT:  1      11    10.00                       vfredmax.vs v5, v7, v8
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - SiFiveP400Div
+# CHECK-NEXT: [1]   - SiFiveP400FEXQ0
+# CHECK-NEXT: [2]   - SiFiveP400FloatDiv
+# CHECK-NEXT: [3]   - SiFiveP400IEXQ0
+# CHECK-NEXT: [4]   - SiFiveP400IEXQ1
+# CHECK-NEXT: [5]   - SiFiveP400IEXQ2
+# CHECK-NEXT: [6]   - SiFiveP400Load
+# CHECK-NEXT: [7]   - SiFiveP400Store
+# CHECK-NEXT: [8]   - SiFiveP400VDiv
+# CHECK-NEXT: [9]   - SiFiveP400VEXQ0
+# CHECK-NEXT: [10]  - SiFiveP400VFloatDiv
+# CHECK-NEXT: [11]  - SiFiveP400VLD
+# CHECK-NEXT: [12]  - SiFiveP400VST
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]
+# CHECK-NEXT:  -      -      -      -     40.00   -      -      -      -     916.00  -      -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   Instructions:
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vredsum.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vredsum.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vredsum.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     9.00    -      -      -     vredsum.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vredmin.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vredmax.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     5.00    -      -      -     vredmin.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     5.00    -      -      -     vredmax.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vredmin.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vredmax.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     14.00   -      -      -     vredmin.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     14.00   -      -      -     vredmax.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     3.00    -      -      -     vredmin.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     3.00    -      -      -     vredmax.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vredmin.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vredmax.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     6.00    -      -      -     vredmin.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     6.00    -      -      -     vredmax.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     12.00   -      -      -     vredmin.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     12.00   -      -      -     vredmax.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vredmin.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vredmax.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     3.00    -      -      -     vredmin.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     3.00    -      -      -     vredmax.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     5.00    -      -      -     vredmin.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     5.00    -      -      -     vredmax.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     10.00   -      -      -     vredmin.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     10.00   -      -      -     vredmax.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     16.00   -      -      -     vfredusum.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     16.00   -      -      -     vfredosum.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     16.00   -      -      -     vfredusum.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     32.00   -      -      -     vfredosum.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     16.00   -      -      -     vfredusum.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     64.00   -      -      -     vfredosum.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     16.00   -      -      -     vfredusum.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     128.00  -      -      -     vfredosum.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     10.00   -      -      -     vfredusum.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     10.00   -      -      -     vfredosum.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     10.00   -      -      -     vfredusum.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     20.00   -      -      -     vfredosum.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     10.00   -      -      -     vfredusum.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     40.00   -      -      -     vfredosum.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     10.00   -      -      -     vfredusum.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     80.00   -      -      -     vfredosum.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     6.00    -      -      -     vfredusum.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     6.00    -      -      -     vfredosum.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     6.00    -      -      -     vfredusum.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     12.00   -      -      -     vfredosum.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     6.00    -      -      -     vfredusum.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     24.00   -      -      -     vfredosum.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     6.00    -      -      -     vfredusum.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     48.00   -      -      -     vfredosum.vs v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     3.00    -      -      -     vfredmin.vs  v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     3.00    -      -      -     vfredmax.vs  v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vfredmin.vs  v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vfredmax.vs  v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     6.00    -      -      -     vfredmin.vs  v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     6.00    -      -      -     vfredmax.vs  v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     12.00   -      -      -     vfredmin.vs  v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     12.00   -      -      -     vfredmax.vs  v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     3.00    -      -      -     vfredmin.vs  v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     3.00    -      -      -     vfredmax.vs  v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vfredmin.vs  v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vfredmax.vs  v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     6.00    -      -      -     vfredmin.vs  v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     6.00    -      -      -     vfredmax.vs  v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     12.00   -      -      -     vfredmin.vs  v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     12.00   -      -      -     vfredmax.vs  v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vfredmin.vs  v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vfredmax.vs  v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     3.00    -      -      -     vfredmin.vs  v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     3.00    -      -      -     vfredmax.vs  v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     5.00    -      -      -     vfredmin.vs  v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     5.00    -      -      -     vfredmax.vs  v5, v7, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     10.00   -      -      -     vfredmin.vs  v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     10.00   -      -      -     vfredmax.vs  v5, v7, v8
+
diff --git a/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/vrgather.s b/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/vrgather.s
new file mode 100644
index 000000000000000..ef248509571931a
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/vrgather.s
@@ -0,0 +1,86 @@
+
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=riscv64 -mcpu=sifive-p470 -iterations=1 < %s | FileCheck %s
+
+vsetvli zero, zero, e32, m1, tu, mu
+vrgather.vv  v5, v7, v8
+vcompress.vm v4, v9, v0
+
+vsetvli zero, zero, e32, m2, tu, mu
+vrgather.vv  v5, v7, v8
+vcompress.vm v4, v9, v0
+
+vsetvli zero, zero, e32, m4, tu, mu
+vrgather.vv  v5, v7, v8
+vcompress.vm v4, v9, v0
+
+vsetvli zero, zero, e32, m8, tu, mu
+vrgather.vv  v5, v7, v8
+vcompress.vm v4, v9, v0
+
+# CHECK:      Iterations:        1
+# CHECK-NEXT: Instructions:      12
+# CHECK-NEXT: Total Cycles:      92
+# CHECK-NEXT: Total uOps:        12
+
+# CHECK:      Dispatch Width:    3
+# CHECK-NEXT: uOps Per Cycle:    0.13
+# CHECK-NEXT: IPC:               0.13
+# CHECK-NEXT: Block RThroughput: 106.0
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      3     1.00                        vrgather.vv v5, v7, v8
+# CHECK-NEXT:  1      3     1.00                        vcompress.vm  v4, v9, v0
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      6     12.00                       vrgather.vv v5, v7, v8
+# CHECK-NEXT:  1      6     12.00                       vcompress.vm  v4, v9, v0
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      6     16.00                       vrgather.vv v5, v7, v8
+# CHECK-NEXT:  1      6     16.00                       vcompress.vm  v4, v9, v0
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      6     24.00                       vrgather.vv v5, v7, v8
+# CHECK-NEXT:  1      6     24.00                       vcompress.vm  v4, v9, v0
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - SiFiveP400Div
+# CHECK-NEXT: [1]   - SiFiveP400FEXQ0
+# CHECK-NEXT: [2]   - SiFiveP400FloatDiv
+# CHECK-NEXT: [3]   - SiFiveP400IEXQ0
+# CHECK-NEXT: [4]   - SiFiveP400IEXQ1
+# CHECK-NEXT: [5]   - SiFiveP400IEXQ2
+# CHECK-NEXT: [6]   - SiFiveP400Load
+# CHECK-NEXT: [7]   - SiFiveP400Store
+# CHECK-NEXT: [8]   - SiFiveP400VDiv
+# CHECK-NEXT: [9]   - SiFiveP400VEXQ0
+# CHECK-NEXT: [10]  - SiFiveP400VFloatDiv
+# CHECK-NEXT: [11]  - SiFiveP400VLD
+# CHECK-NEXT: [12]  - SiFiveP400VST
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]
+# CHECK-NEXT:  -      -      -      -     4.00    -      -      -      -     106.00  -      -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   Instructions:
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vrgather.vv  v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vcompress.vm v4, v9, v0
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     12.00   -      -      -     vrgather.vv  v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     12.00   -      -      -     vcompress.vm v4, v9, v0
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     16.00   -      -      -     vrgather.vv  v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     16.00   -      -      -     vcompress.vm v4, v9, v0
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     24.00   -      -      -     vrgather.vv  v5, v7, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     24.00   -      -      -     vcompress.vm v4, v9, v0
+
diff --git a/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/vshift-vmul.s b/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/vshift-vmul.s
new file mode 100644
index 000000000000000..7a0cf27f37425df
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/vshift-vmul.s
@@ -0,0 +1,132 @@
+
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=riscv64 -mcpu=sifive-p470 -iterations=1 < %s | FileCheck %s
+
+vsetvli zero, zero, e32, m1, ta, ma
+
+vsll.vv v1, v2, v5
+vsll.vx v1, v2, t0
+vsll.vi v1, v2, 7
+
+vsrl.vv v1, v2, v5
+vsrl.vx v1, v2, t0
+vsrl.vi v1, v2, 7
+
+vsra.vv v1, v2, v5
+vsra.vx v1, v2, t0
+vsra.vi v1, v2, 7
+
+vsetvli zero, zero, e32, mf4, ta, ma
+
+vsll.vv v1, v2, v5
+vsll.vx v1, v2, t0
+vsll.vi v1, v2, 7
+
+vsrl.vv v1, v2, v5
+vsrl.vx v1, v2, t0
+vsrl.vi v1, v2, 7
+
+vsra.vv v1, v2, v5
+vsra.vx v1, v2, t0
+vsra.vi v1, v2, 7
+
+vsetvli zero, zero, e32, m8, ta, ma
+
+vmul.vv v1, v2, v5
+vmul.vx v1, v2, t1
+
+vmadd.vv v1, v2, v5
+vmadd.vx v1, t1, v2
+
+# CHECK:      Iterations:        1
+# CHECK-NEXT: Instructions:      25
+# CHECK-NEXT: Total Cycles:      57
+# CHECK-NEXT: Total uOps:        25
+
+# CHECK:      Dispatch Width:    3
+# CHECK-NEXT: uOps Per Cycle:    0.44
+# CHECK-NEXT: IPC:               0.44
+# CHECK-NEXT: Block RThroughput: 50.0
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, ta, ma
+# CHECK-NEXT:  1      2     1.00                        vsll.vv v1, v2, v5
+# CHECK-NEXT:  1      2     1.00                        vsll.vx v1, v2, t0
+# CHECK-NEXT:  1      2     1.00                        vsll.vi v1, v2, 7
+# CHECK-NEXT:  1      2     1.00                        vsrl.vv v1, v2, v5
+# CHECK-NEXT:  1      2     1.00                        vsrl.vx v1, v2, t0
+# CHECK-NEXT:  1      2     1.00                        vsrl.vi v1, v2, 7
+# CHECK-NEXT:  1      2     1.00                        vsra.vv v1, v2, v5
+# CHECK-NEXT:  1      2     1.00                        vsra.vx v1, v2, t0
+# CHECK-NEXT:  1      2     1.00                        vsra.vi v1, v2, 7
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf4, ta, ma
+# CHECK-NEXT:  1      2     1.00                        vsll.vv v1, v2, v5
+# CHECK-NEXT:  1      2     1.00                        vsll.vx v1, v2, t0
+# CHECK-NEXT:  1      2     1.00                        vsll.vi v1, v2, 7
+# CHECK-NEXT:  1      2     1.00                        vsrl.vv v1, v2, v5
+# CHECK-NEXT:  1      2     1.00                        vsrl.vx v1, v2, t0
+# CHECK-NEXT:  1      2     1.00                        vsrl.vi v1, v2, 7
+# CHECK-NEXT:  1      2     1.00                        vsra.vv v1, v2, v5
+# CHECK-NEXT:  1      2     1.00                        vsra.vx v1, v2, t0
+# CHECK-NEXT:  1      2     1.00                        vsra.vi v1, v2, 7
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, ta, ma
+# CHECK-NEXT:  1      9     8.00                        vmul.vv v1, v2, v5
+# CHECK-NEXT:  1      9     8.00                        vmul.vx v1, v2, t1
+# CHECK-NEXT:  1      9     8.00                        vmadd.vv  v1, v2, v5
+# CHECK-NEXT:  1      9     8.00                        vmadd.vx  v1, t1, v2
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - SiFiveP400Div
+# CHECK-NEXT: [1]   - SiFiveP400FEXQ0
+# CHECK-NEXT: [2]   - SiFiveP400FloatDiv
+# CHECK-NEXT: [3]   - SiFiveP400IEXQ0
+# CHECK-NEXT: [4]   - SiFiveP400IEXQ1
+# CHECK-NEXT: [5]   - SiFiveP400IEXQ2
+# CHECK-NEXT: [6]   - SiFiveP400Load
+# CHECK-NEXT: [7]   - SiFiveP400Store
+# CHECK-NEXT: [8]   - SiFiveP400VDiv
+# CHECK-NEXT: [9]   - SiFiveP400VEXQ0
+# CHECK-NEXT: [10]  - SiFiveP400VFloatDiv
+# CHECK-NEXT: [11]  - SiFiveP400VLD
+# CHECK-NEXT: [12]  - SiFiveP400VST
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]
+# CHECK-NEXT:  -      -      -      -     3.00    -      -      -      -     50.00   -      -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   Instructions:
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, ta, ma
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsll.vv  v1, v2, v5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsll.vx  v1, v2, t0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsll.vi  v1, v2, 7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsrl.vv  v1, v2, v5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsrl.vx  v1, v2, t0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsrl.vi  v1, v2, 7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsra.vv  v1, v2, v5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsra.vx  v1, v2, t0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsra.vi  v1, v2, 7
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf4, ta, ma
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsll.vv  v1, v2, v5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsll.vx  v1, v2, t0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsll.vi  v1, v2, 7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsrl.vv  v1, v2, v5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsrl.vx  v1, v2, t0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsrl.vi  v1, v2, 7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsra.vv  v1, v2, v5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsra.vx  v1, v2, t0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsra.vi  v1, v2, 7
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, ta, ma
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmul.vv  v1, v2, v5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmul.vx  v1, v2, t1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmadd.vv v1, v2, v5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vmadd.vx v1, t1, v2
+
diff --git a/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/zvbb.s b/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/zvbb.s
new file mode 100644
index 000000000000000..236c8f0199f6633
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/zvbb.s
@@ -0,0 +1,460 @@
+
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=riscv64 -mcpu=sifive-p470 -iterations=1 < %s | FileCheck %s
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vandn.vv v4, v8, v12
+vandn.vx v4, v8, a0
+vbrev.v v4, v8
+vbrev8.v v4, v8
+vrev8.v v4, v8
+vclz.v v4, v8
+vctz.v v4, v8
+vcpop.v v4, v8
+vrol.vv v4, v8, v12
+vrol.vx v4, v8, a0
+vror.vv v4, v8, v12
+vror.vx v4, v8, a0
+vror.vi v4, v8, 8
+
+vwsll.vv v4, v8, v12
+vwsll.vx v4, v8, a0
+vwsll.vi v4, v8, 8
+
+vsetvli zero, zero, e16, mf4, tu, mu
+vandn.vv v4, v8, v12
+vandn.vx v4, v8, a0
+vbrev.v v4, v8
+vbrev8.v v4, v8
+vrev8.v v4, v8
+vclz.v v4, v8
+vctz.v v4, v8
+vcpop.v v4, v8
+vrol.vv v4, v8, v12
+vrol.vx v4, v8, a0
+vror.vv v4, v8, v12
+vror.vx v4, v8, a0
+vror.vi v4, v8, 8
+
+vwsll.vv v4, v8, v12
+vwsll.vx v4, v8, a0
+vwsll.vi v4, v8, 8
+
+vsetvli zero, zero, e32, mf2, tu, mu
+vandn.vv v4, v8, v12
+vandn.vx v4, v8, a0
+vbrev.v v4, v8
+vbrev8.v v4, v8
+vrev8.v v4, v8
+vclz.v v4, v8
+vctz.v v4, v8
+vcpop.v v4, v8
+vrol.vv v4, v8, v12
+vrol.vx v4, v8, a0
+vror.vv v4, v8, v12
+vror.vx v4, v8, a0
+vror.vi v4, v8, 8
+
+vwsll.vv v4, v8, v12
+vwsll.vx v4, v8, a0
+vwsll.vi v4, v8, 8
+
+vsetvli zero, zero, e32, m1, tu, mu
+vandn.vv v4, v8, v12
+vandn.vx v4, v8, a0
+vbrev.v v4, v8
+vbrev8.v v4, v8
+vrev8.v v4, v8
+vclz.v v4, v8
+vctz.v v4, v8
+vcpop.v v4, v8
+vrol.vv v4, v8, v12
+vrol.vx v4, v8, a0
+vror.vv v4, v8, v12
+vror.vx v4, v8, a0
+vror.vi v4, v8, 8
+
+vwsll.vv v4, v8, v12
+vwsll.vx v4, v8, a0
+vwsll.vi v4, v8, 8
+
+vsetvli zero, zero, e32, m2, tu, mu
+vandn.vv v4, v8, v12
+vandn.vx v4, v8, a0
+vbrev.v v4, v8
+vbrev8.v v4, v8
+vrev8.v v4, v8
+vclz.v v4, v8
+vctz.v v4, v8
+vcpop.v v4, v8
+vrol.vv v4, v8, v12
+vrol.vx v4, v8, a0
+vror.vv v4, v8, v12
+vror.vx v4, v8, a0
+vror.vi v4, v8, 8
+
+vwsll.vv v4, v8, v12
+vwsll.vx v4, v8, a0
+vwsll.vi v4, v8, 8
+
+vsetvli zero, zero, e32, m4, tu, mu
+vandn.vv v4, v8, v12
+vandn.vx v4, v8, a0
+vbrev.v v4, v8
+vbrev8.v v4, v8
+vrev8.v v4, v8
+vclz.v v4, v8
+vctz.v v4, v8
+vcpop.v v4, v8
+vrol.vv v4, v8, v12
+vrol.vx v4, v8, a0
+vror.vv v4, v8, v12
+vror.vx v4, v8, a0
+vror.vi v4, v8, 8
+
+vwsll.vv v8, v4, v12
+vwsll.vx v8, v4, a0
+vwsll.vi v8, v4, 8
+
+vsetvli zero, zero, e32, m8, tu, mu
+vandn.vv v8, v16, v24
+vandn.vx v8, v16, a0
+vbrev.v  v8, v16
+vbrev8.v v8, v16
+vrev8.v  v8, v16
+vclz.v   v8, v16
+vctz.v   v8, v16
+vcpop.v  v8, v16
+vrol.vv  v8, v16, v24
+vrol.vx  v8, v16, a0
+vror.vv  v8, v16, v24
+vror.vx  v8, v16, a0
+vror.vi  v8, v16, 8
+
+# Show SEW does not matter
+vsetvli zero, zero, e16, m4, tu, mu
+vandn.vv v4, v8, v12
+vandn.vx v4, v8, a0
+vbrev.v  v4, v8
+vbrev8.v v4, v8
+vrev8.v  v4, v8
+vclz.v   v4, v8
+vctz.v   v4, v8
+vcpop.v  v4, v8
+vrol.vv  v4, v8, v12
+vrol.vx  v4, v8, a0
+vror.vv  v4, v8, v12
+vror.vx  v4, v8, a0
+vror.vi  v4, v8, 8
+vwsll.vv v8, v4, v12
+vwsll.vx v8, v4, a0
+vwsll.vi v8, v4, 8
+
+# CHECK:      Iterations:        1
+# CHECK-NEXT: Instructions:      133
+# CHECK-NEXT: Total Cycles:      330
+# CHECK-NEXT: Total uOps:        133
+
+# CHECK:      Dispatch Width:    3
+# CHECK-NEXT: uOps Per Cycle:    0.40
+# CHECK-NEXT: IPC:               0.40
+# CHECK-NEXT: Block RThroughput: 328.0
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vandn.vv  v4, v8, v12
+# CHECK-NEXT:  1      2     1.00                        vandn.vx  v4, v8, a0
+# CHECK-NEXT:  1      2     1.00                        vbrev.v v4, v8
+# CHECK-NEXT:  1      2     1.00                        vbrev8.v  v4, v8
+# CHECK-NEXT:  1      2     1.00                        vrev8.v v4, v8
+# CHECK-NEXT:  1      2     1.00                        vclz.v  v4, v8
+# CHECK-NEXT:  1      2     1.00                        vctz.v  v4, v8
+# CHECK-NEXT:  1      2     1.00                        vcpop.v v4, v8
+# CHECK-NEXT:  1      2     1.00                        vrol.vv v4, v8, v12
+# CHECK-NEXT:  1      2     1.00                        vrol.vx v4, v8, a0
+# CHECK-NEXT:  1      2     1.00                        vror.vv v4, v8, v12
+# CHECK-NEXT:  1      2     1.00                        vror.vx v4, v8, a0
+# CHECK-NEXT:  1      2     1.00                        vror.vi v4, v8, 8
+# CHECK-NEXT:  1      2     1.00                        vwsll.vv  v4, v8, v12
+# CHECK-NEXT:  1      2     1.00                        vwsll.vx  v4, v8, a0
+# CHECK-NEXT:  1      2     1.00                        vwsll.vi  v4, v8, 8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vandn.vv  v4, v8, v12
+# CHECK-NEXT:  1      2     1.00                        vandn.vx  v4, v8, a0
+# CHECK-NEXT:  1      2     1.00                        vbrev.v v4, v8
+# CHECK-NEXT:  1      2     1.00                        vbrev8.v  v4, v8
+# CHECK-NEXT:  1      2     1.00                        vrev8.v v4, v8
+# CHECK-NEXT:  1      2     1.00                        vclz.v  v4, v8
+# CHECK-NEXT:  1      2     1.00                        vctz.v  v4, v8
+# CHECK-NEXT:  1      2     1.00                        vcpop.v v4, v8
+# CHECK-NEXT:  1      2     1.00                        vrol.vv v4, v8, v12
+# CHECK-NEXT:  1      2     1.00                        vrol.vx v4, v8, a0
+# CHECK-NEXT:  1      2     1.00                        vror.vv v4, v8, v12
+# CHECK-NEXT:  1      2     1.00                        vror.vx v4, v8, a0
+# CHECK-NEXT:  1      2     1.00                        vror.vi v4, v8, 8
+# CHECK-NEXT:  1      2     1.00                        vwsll.vv  v4, v8, v12
+# CHECK-NEXT:  1      2     1.00                        vwsll.vx  v4, v8, a0
+# CHECK-NEXT:  1      2     1.00                        vwsll.vi  v4, v8, 8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vandn.vv  v4, v8, v12
+# CHECK-NEXT:  1      2     1.00                        vandn.vx  v4, v8, a0
+# CHECK-NEXT:  1      2     1.00                        vbrev.v v4, v8
+# CHECK-NEXT:  1      2     1.00                        vbrev8.v  v4, v8
+# CHECK-NEXT:  1      2     1.00                        vrev8.v v4, v8
+# CHECK-NEXT:  1      2     1.00                        vclz.v  v4, v8
+# CHECK-NEXT:  1      2     1.00                        vctz.v  v4, v8
+# CHECK-NEXT:  1      2     1.00                        vcpop.v v4, v8
+# CHECK-NEXT:  1      2     1.00                        vrol.vv v4, v8, v12
+# CHECK-NEXT:  1      2     1.00                        vrol.vx v4, v8, a0
+# CHECK-NEXT:  1      2     1.00                        vror.vv v4, v8, v12
+# CHECK-NEXT:  1      2     1.00                        vror.vx v4, v8, a0
+# CHECK-NEXT:  1      2     1.00                        vror.vi v4, v8, 8
+# CHECK-NEXT:  1      2     1.00                        vwsll.vv  v4, v8, v12
+# CHECK-NEXT:  1      2     1.00                        vwsll.vx  v4, v8, a0
+# CHECK-NEXT:  1      2     1.00                        vwsll.vi  v4, v8, 8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vandn.vv  v4, v8, v12
+# CHECK-NEXT:  1      2     1.00                        vandn.vx  v4, v8, a0
+# CHECK-NEXT:  1      2     1.00                        vbrev.v v4, v8
+# CHECK-NEXT:  1      2     1.00                        vbrev8.v  v4, v8
+# CHECK-NEXT:  1      2     1.00                        vrev8.v v4, v8
+# CHECK-NEXT:  1      2     1.00                        vclz.v  v4, v8
+# CHECK-NEXT:  1      2     1.00                        vctz.v  v4, v8
+# CHECK-NEXT:  1      2     1.00                        vcpop.v v4, v8
+# CHECK-NEXT:  1      2     1.00                        vrol.vv v4, v8, v12
+# CHECK-NEXT:  1      2     1.00                        vrol.vx v4, v8, a0
+# CHECK-NEXT:  1      2     1.00                        vror.vv v4, v8, v12
+# CHECK-NEXT:  1      2     1.00                        vror.vx v4, v8, a0
+# CHECK-NEXT:  1      2     1.00                        vror.vi v4, v8, 8
+# CHECK-NEXT:  1      2     1.00                        vwsll.vv  v4, v8, v12
+# CHECK-NEXT:  1      2     1.00                        vwsll.vx  v4, v8, a0
+# CHECK-NEXT:  1      2     1.00                        vwsll.vi  v4, v8, 8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vandn.vv  v4, v8, v12
+# CHECK-NEXT:  1      2     2.00                        vandn.vx  v4, v8, a0
+# CHECK-NEXT:  1      2     2.00                        vbrev.v v4, v8
+# CHECK-NEXT:  1      2     2.00                        vbrev8.v  v4, v8
+# CHECK-NEXT:  1      2     2.00                        vrev8.v v4, v8
+# CHECK-NEXT:  1      2     2.00                        vclz.v  v4, v8
+# CHECK-NEXT:  1      2     2.00                        vctz.v  v4, v8
+# CHECK-NEXT:  1      2     2.00                        vcpop.v v4, v8
+# CHECK-NEXT:  1      2     2.00                        vrol.vv v4, v8, v12
+# CHECK-NEXT:  1      2     2.00                        vrol.vx v4, v8, a0
+# CHECK-NEXT:  1      2     2.00                        vror.vv v4, v8, v12
+# CHECK-NEXT:  1      2     2.00                        vror.vx v4, v8, a0
+# CHECK-NEXT:  1      2     2.00                        vror.vi v4, v8, 8
+# CHECK-NEXT:  1      2     2.00                        vwsll.vv  v4, v8, v12
+# CHECK-NEXT:  1      2     2.00                        vwsll.vx  v4, v8, a0
+# CHECK-NEXT:  1      2     2.00                        vwsll.vi  v4, v8, 8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vandn.vv  v4, v8, v12
+# CHECK-NEXT:  1      2     4.00                        vandn.vx  v4, v8, a0
+# CHECK-NEXT:  1      2     4.00                        vbrev.v v4, v8
+# CHECK-NEXT:  1      2     4.00                        vbrev8.v  v4, v8
+# CHECK-NEXT:  1      2     4.00                        vrev8.v v4, v8
+# CHECK-NEXT:  1      2     4.00                        vclz.v  v4, v8
+# CHECK-NEXT:  1      2     4.00                        vctz.v  v4, v8
+# CHECK-NEXT:  1      2     4.00                        vcpop.v v4, v8
+# CHECK-NEXT:  1      2     4.00                        vrol.vv v4, v8, v12
+# CHECK-NEXT:  1      2     4.00                        vrol.vx v4, v8, a0
+# CHECK-NEXT:  1      2     4.00                        vror.vv v4, v8, v12
+# CHECK-NEXT:  1      2     4.00                        vror.vx v4, v8, a0
+# CHECK-NEXT:  1      2     4.00                        vror.vi v4, v8, 8
+# CHECK-NEXT:  1      2     4.00                        vwsll.vv  v8, v4, v12
+# CHECK-NEXT:  1      2     4.00                        vwsll.vx  v8, v4, a0
+# CHECK-NEXT:  1      2     4.00                        vwsll.vi  v8, v4, 8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vandn.vv  v8, v16, v24
+# CHECK-NEXT:  1      2     8.00                        vandn.vx  v8, v16, a0
+# CHECK-NEXT:  1      2     8.00                        vbrev.v v8, v16
+# CHECK-NEXT:  1      2     8.00                        vbrev8.v  v8, v16
+# CHECK-NEXT:  1      2     8.00                        vrev8.v v8, v16
+# CHECK-NEXT:  1      2     8.00                        vclz.v  v8, v16
+# CHECK-NEXT:  1      2     8.00                        vctz.v  v8, v16
+# CHECK-NEXT:  1      2     8.00                        vcpop.v v8, v16
+# CHECK-NEXT:  1      2     8.00                        vrol.vv v8, v16, v24
+# CHECK-NEXT:  1      2     8.00                        vrol.vx v8, v16, a0
+# CHECK-NEXT:  1      2     8.00                        vror.vv v8, v16, v24
+# CHECK-NEXT:  1      2     8.00                        vror.vx v8, v16, a0
+# CHECK-NEXT:  1      2     8.00                        vror.vi v8, v16, 8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vandn.vv  v4, v8, v12
+# CHECK-NEXT:  1      2     4.00                        vandn.vx  v4, v8, a0
+# CHECK-NEXT:  1      2     4.00                        vbrev.v v4, v8
+# CHECK-NEXT:  1      2     4.00                        vbrev8.v  v4, v8
+# CHECK-NEXT:  1      2     4.00                        vrev8.v v4, v8
+# CHECK-NEXT:  1      2     4.00                        vclz.v  v4, v8
+# CHECK-NEXT:  1      2     4.00                        vctz.v  v4, v8
+# CHECK-NEXT:  1      2     4.00                        vcpop.v v4, v8
+# CHECK-NEXT:  1      2     4.00                        vrol.vv v4, v8, v12
+# CHECK-NEXT:  1      2     4.00                        vrol.vx v4, v8, a0
+# CHECK-NEXT:  1      2     4.00                        vror.vv v4, v8, v12
+# CHECK-NEXT:  1      2     4.00                        vror.vx v4, v8, a0
+# CHECK-NEXT:  1      2     4.00                        vror.vi v4, v8, 8
+# CHECK-NEXT:  1      2     4.00                        vwsll.vv  v8, v4, v12
+# CHECK-NEXT:  1      2     4.00                        vwsll.vx  v8, v4, a0
+# CHECK-NEXT:  1      2     4.00                        vwsll.vi  v8, v4, 8
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - SiFiveP400Div
+# CHECK-NEXT: [1]   - SiFiveP400FEXQ0
+# CHECK-NEXT: [2]   - SiFiveP400FloatDiv
+# CHECK-NEXT: [3]   - SiFiveP400IEXQ0
+# CHECK-NEXT: [4]   - SiFiveP400IEXQ1
+# CHECK-NEXT: [5]   - SiFiveP400IEXQ2
+# CHECK-NEXT: [6]   - SiFiveP400Load
+# CHECK-NEXT: [7]   - SiFiveP400Store
+# CHECK-NEXT: [8]   - SiFiveP400VDiv
+# CHECK-NEXT: [9]   - SiFiveP400VEXQ0
+# CHECK-NEXT: [10]  - SiFiveP400VFloatDiv
+# CHECK-NEXT: [11]  - SiFiveP400VLD
+# CHECK-NEXT: [12]  - SiFiveP400VST
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]
+# CHECK-NEXT:  -      -      -      -     8.00    -      -      -      -     328.00  -      -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   Instructions:
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vandn.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vandn.vx v4, v8, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vbrev.v  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vbrev8.v v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vrev8.v  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vclz.v v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vctz.v v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vcpop.v  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vrol.vv  v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vrol.vx  v4, v8, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vror.vv  v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vror.vx  v4, v8, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vror.vi  v4, v8, 8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vwsll.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vwsll.vx v4, v8, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vwsll.vi v4, v8, 8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vandn.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vandn.vx v4, v8, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vbrev.v  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vbrev8.v v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vrev8.v  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vclz.v v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vctz.v v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vcpop.v  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vrol.vv  v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vrol.vx  v4, v8, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vror.vv  v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vror.vx  v4, v8, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vror.vi  v4, v8, 8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vwsll.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vwsll.vx v4, v8, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vwsll.vi v4, v8, 8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vandn.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vandn.vx v4, v8, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vbrev.v  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vbrev8.v v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vrev8.v  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vclz.v v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vctz.v v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vcpop.v  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vrol.vv  v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vrol.vx  v4, v8, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vror.vv  v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vror.vx  v4, v8, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vror.vi  v4, v8, 8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vwsll.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vwsll.vx v4, v8, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vwsll.vi v4, v8, 8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vandn.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vandn.vx v4, v8, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vbrev.v  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vbrev8.v v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vrev8.v  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vclz.v v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vctz.v v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vcpop.v  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vrol.vv  v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vrol.vx  v4, v8, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vror.vv  v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vror.vx  v4, v8, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vror.vi  v4, v8, 8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vwsll.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vwsll.vx v4, v8, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vwsll.vi v4, v8, 8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vandn.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vandn.vx v4, v8, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vbrev.v  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vbrev8.v v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vrev8.v  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vclz.v v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vctz.v v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vcpop.v  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vrol.vv  v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vrol.vx  v4, v8, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vror.vv  v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vror.vx  v4, v8, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vror.vi  v4, v8, 8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vwsll.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vwsll.vx v4, v8, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vwsll.vi v4, v8, 8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vandn.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vandn.vx v4, v8, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vbrev.v  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vbrev8.v v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vrev8.v  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vclz.v v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vctz.v v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vcpop.v  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vrol.vv  v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vrol.vx  v4, v8, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vror.vv  v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vror.vx  v4, v8, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vror.vi  v4, v8, 8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vwsll.vv v8, v4, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vwsll.vx v8, v4, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vwsll.vi v8, v4, 8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vandn.vv v8, v16, v24
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vandn.vx v8, v16, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vbrev.v  v8, v16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vbrev8.v v8, v16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vrev8.v  v8, v16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vclz.v v8, v16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vctz.v v8, v16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vcpop.v  v8, v16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vrol.vv  v8, v16, v24
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vrol.vx  v8, v16, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vror.vv  v8, v16, v24
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vror.vx  v8, v16, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vror.vi  v8, v16, 8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vandn.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vandn.vx v4, v8, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vbrev.v  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vbrev8.v v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vrev8.v  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vclz.v v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vctz.v v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vcpop.v  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vrol.vv  v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vrol.vx  v4, v8, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vror.vv  v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vror.vx  v4, v8, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vror.vi  v4, v8, 8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vwsll.vv v8, v4, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vwsll.vx v8, v4, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vwsll.vi v8, v4, 8
+
diff --git a/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/zvbc.s b/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/zvbc.s
new file mode 100644
index 000000000000000..1e304f6b75a140f
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/zvbc.s
@@ -0,0 +1,112 @@
+
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=riscv64 -mcpu=sifive-p470 -iterations=1 < %s | FileCheck %s
+
+# These instructions only work with e64
+
+vsetvli zero, zero, e64, m1, tu, mu
+vclmul.vv v4, v8, v12
+vclmul.vx v4, v8, a0
+vclmulh.vv v4, v8, v12
+vclmulh.vx v4, v8, a0
+
+vsetvli zero, zero, e64, m2, tu, mu
+vclmul.vv v4, v8, v12
+vclmul.vx v4, v8, a0
+vclmulh.vv v4, v8, v12
+vclmulh.vx v4, v8, a0
+
+vsetvli zero, zero, e64, m4, tu, mu
+vclmul.vv v4, v8, v12
+vclmul.vx v4, v8, a0
+vclmulh.vv v4, v8, v12
+vclmulh.vx v4, v8, a0
+
+vsetvli zero, zero, e64, m8, tu, mu
+vclmul.vv  v8, v12, v24
+vclmul.vx  v8, v12, a0
+vclmulh.vv v8, v12, v24
+vclmulh.vx v8, v12, a0
+
+# CHECK:      Iterations:        1
+# CHECK-NEXT: Instructions:      20
+# CHECK-NEXT: Total Cycles:      58
+# CHECK-NEXT: Total uOps:        20
+
+# CHECK:      Dispatch Width:    3
+# CHECK-NEXT: uOps Per Cycle:    0.34
+# CHECK-NEXT: IPC:               0.34
+# CHECK-NEXT: Block RThroughput: 60.0
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vclmul.vv v4, v8, v12
+# CHECK-NEXT:  1      2     1.00                        vclmul.vx v4, v8, a0
+# CHECK-NEXT:  1      2     1.00                        vclmulh.vv  v4, v8, v12
+# CHECK-NEXT:  1      2     1.00                        vclmulh.vx  v4, v8, a0
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vclmul.vv v4, v8, v12
+# CHECK-NEXT:  1      2     2.00                        vclmul.vx v4, v8, a0
+# CHECK-NEXT:  1      2     2.00                        vclmulh.vv  v4, v8, v12
+# CHECK-NEXT:  1      2     2.00                        vclmulh.vx  v4, v8, a0
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vclmul.vv v4, v8, v12
+# CHECK-NEXT:  1      2     4.00                        vclmul.vx v4, v8, a0
+# CHECK-NEXT:  1      2     4.00                        vclmulh.vv  v4, v8, v12
+# CHECK-NEXT:  1      2     4.00                        vclmulh.vx  v4, v8, a0
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vclmul.vv v8, v12, v24
+# CHECK-NEXT:  1      2     8.00                        vclmul.vx v8, v12, a0
+# CHECK-NEXT:  1      2     8.00                        vclmulh.vv  v8, v12, v24
+# CHECK-NEXT:  1      2     8.00                        vclmulh.vx  v8, v12, a0
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - SiFiveP400Div
+# CHECK-NEXT: [1]   - SiFiveP400FEXQ0
+# CHECK-NEXT: [2]   - SiFiveP400FloatDiv
+# CHECK-NEXT: [3]   - SiFiveP400IEXQ0
+# CHECK-NEXT: [4]   - SiFiveP400IEXQ1
+# CHECK-NEXT: [5]   - SiFiveP400IEXQ2
+# CHECK-NEXT: [6]   - SiFiveP400Load
+# CHECK-NEXT: [7]   - SiFiveP400Store
+# CHECK-NEXT: [8]   - SiFiveP400VDiv
+# CHECK-NEXT: [9]   - SiFiveP400VEXQ0
+# CHECK-NEXT: [10]  - SiFiveP400VFloatDiv
+# CHECK-NEXT: [11]  - SiFiveP400VLD
+# CHECK-NEXT: [12]  - SiFiveP400VST
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]
+# CHECK-NEXT:  -      -      -      -     4.00    -      -      -      -     60.00   -      -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   Instructions:
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vclmul.vv  v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vclmul.vx  v4, v8, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vclmulh.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vclmulh.vx v4, v8, a0
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vclmul.vv  v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vclmul.vx  v4, v8, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vclmulh.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vclmulh.vx v4, v8, a0
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vclmul.vv  v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vclmul.vx  v4, v8, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vclmulh.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vclmulh.vx v4, v8, a0
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vclmul.vv  v8, v12, v24
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vclmul.vx  v8, v12, a0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vclmulh.vv v8, v12, v24
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vclmulh.vx v8, v12, a0
+
diff --git a/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/zvkg.s b/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/zvkg.s
new file mode 100644
index 000000000000000..8e498a678138825
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/zvkg.s
@@ -0,0 +1,127 @@
+
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=riscv64 -mcpu=sifive-p470 -iterations=1 < %s | FileCheck %s
+
+vsetvli zero, zero, e8, mf8, tu, mu
+vghsh.vv v4, v8, v12
+vgmul.vv v4, v8
+
+vsetvli zero, zero, e16, mf4, tu, mu
+vghsh.vv v4, v8, v12
+vgmul.vv v4, v8
+
+vsetvli zero, zero, e32, mf2, tu, mu
+vghsh.vv v4, v8, v12
+vgmul.vv v4, v8
+
+vsetvli zero, zero, e32, m1, tu, mu
+vghsh.vv v4, v8, v12
+vgmul.vv v4, v8
+
+vsetvli zero, zero, e32, m2, tu, mu
+vghsh.vv v4, v8, v12
+vgmul.vv v4, v8
+
+vsetvli zero, zero, e32, m4, tu, mu
+vghsh.vv v4, v8, v12
+vgmul.vv v4, v8
+
+vsetvli zero, zero, e32, m8, tu, mu
+vghsh.vv v8, v16, v24
+vgmul.vv v8, v16
+
+# Show SEW does not matter
+vsetvli zero, zero, e64, m4, tu, mu
+vghsh.vv v4, v8, v12
+vgmul.vv v4, v8
+
+# CHECK:      Iterations:        1
+# CHECK-NEXT: Instructions:      24
+# CHECK-NEXT: Total Cycles:      76
+# CHECK-NEXT: Total uOps:        24
+
+# CHECK:      Dispatch Width:    3
+# CHECK-NEXT: uOps Per Cycle:    0.32
+# CHECK-NEXT: IPC:               0.32
+# CHECK-NEXT: Block RThroughput: 72.0
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vghsh.vv  v4, v8, v12
+# CHECK-NEXT:  1      2     8.00                        vgmul.vv  v4, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vghsh.vv  v4, v8, v12
+# CHECK-NEXT:  1      2     8.00                        vgmul.vv  v4, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vghsh.vv  v4, v8, v12
+# CHECK-NEXT:  1      2     1.00                        vgmul.vv  v4, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vghsh.vv  v4, v8, v12
+# CHECK-NEXT:  1      2     1.00                        vgmul.vv  v4, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vghsh.vv  v4, v8, v12
+# CHECK-NEXT:  1      2     2.00                        vgmul.vv  v4, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vghsh.vv  v4, v8, v12
+# CHECK-NEXT:  1      2     4.00                        vgmul.vv  v4, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vghsh.vv  v8, v16, v24
+# CHECK-NEXT:  1      2     8.00                        vgmul.vv  v8, v16
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vghsh.vv  v4, v8, v12
+# CHECK-NEXT:  1      2     4.00                        vgmul.vv  v4, v8
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - SiFiveP400Div
+# CHECK-NEXT: [1]   - SiFiveP400FEXQ0
+# CHECK-NEXT: [2]   - SiFiveP400FloatDiv
+# CHECK-NEXT: [3]   - SiFiveP400IEXQ0
+# CHECK-NEXT: [4]   - SiFiveP400IEXQ1
+# CHECK-NEXT: [5]   - SiFiveP400IEXQ2
+# CHECK-NEXT: [6]   - SiFiveP400Load
+# CHECK-NEXT: [7]   - SiFiveP400Store
+# CHECK-NEXT: [8]   - SiFiveP400VDiv
+# CHECK-NEXT: [9]   - SiFiveP400VEXQ0
+# CHECK-NEXT: [10]  - SiFiveP400VFloatDiv
+# CHECK-NEXT: [11]  - SiFiveP400VLD
+# CHECK-NEXT: [12]  - SiFiveP400VST
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]
+# CHECK-NEXT:  -      -      -      -     8.00    -      -      -      -     72.00   -      -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   Instructions:
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e8, mf8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vghsh.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vgmul.vv v4, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e16, mf4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vghsh.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vgmul.vv v4, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vghsh.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vgmul.vv v4, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vghsh.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vgmul.vv v4, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vghsh.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vgmul.vv v4, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vghsh.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vgmul.vv v4, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vghsh.vv v8, v16, v24
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vgmul.vv v8, v16
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vghsh.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vgmul.vv v4, v8
+
diff --git a/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/zvkned.s b/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/zvkned.s
new file mode 100644
index 000000000000000..791069b87eee15c
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/zvkned.s
@@ -0,0 +1,203 @@
+
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=riscv64 -mcpu=sifive-p470 -iterations=1 < %s | FileCheck %s
+
+# These instructions only support e32
+
+vsetvli zero, zero, e32, mf2, tu, mu
+vaesef.vv v4, v8
+vaesef.vs v4, v8
+vaesem.vv v4, v8
+vaesem.vs v4, v8
+vaesdm.vv v4, v8
+vaesdm.vs v4, v8
+vaeskf1.vi v4, v8, 8
+vaeskf2.vi v4, v8, 8
+vaesz.vs v4, v8
+
+vsetvli zero, zero, e32, m1, tu, mu
+vaesef.vv v4, v8
+vaesef.vs v4, v8
+vaesem.vv v4, v8
+vaesem.vs v4, v8
+vaesdm.vv v4, v8
+vaesdm.vs v4, v8
+vaeskf1.vi v4, v8, 8
+vaeskf2.vi v4, v8, 8
+vaesz.vs v4, v8
+
+vsetvli zero, zero, e32, m2, tu, mu
+vaesef.vv v4, v8
+vaesef.vs v4, v8
+vaesem.vv v4, v8
+vaesem.vs v4, v8
+vaesdm.vv v4, v8
+vaesdm.vs v4, v8
+vaeskf1.vi v4, v8, 8
+vaeskf2.vi v4, v8, 8
+vaesz.vs v4, v8
+
+vsetvli zero, zero, e32, m4, tu, mu
+vaesef.vv v4, v8
+vaesef.vs v4, v8
+vaesem.vv v4, v8
+vaesem.vs v4, v8
+vaesdm.vv v4, v8
+vaesdm.vs v4, v8
+vaeskf1.vi v4, v8, 8
+vaeskf2.vi v4, v8, 8
+vaesz.vs v4, v8
+
+vsetvli zero, zero, e32, m8, tu, mu
+vaesef.vv  v8, v16
+vaesef.vs  v8, v16
+vaesem.vv  v8, v16
+vaesem.vs  v8, v16
+vaesdm.vv  v8, v16
+vaesdm.vs  v8, v16
+vaeskf1.vi v8, v16, 8
+vaeskf2.vi v8, v16, 8
+vaesz.vs   v8, v16
+
+# CHECK:      Iterations:        1
+# CHECK-NEXT: Instructions:      50
+# CHECK-NEXT: Total Cycles:      142
+# CHECK-NEXT: Total uOps:        50
+
+# CHECK:      Dispatch Width:    3
+# CHECK-NEXT: uOps Per Cycle:    0.35
+# CHECK-NEXT: IPC:               0.35
+# CHECK-NEXT: Block RThroughput: 144.0
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vaesef.vv v4, v8
+# CHECK-NEXT:  1      2     1.00                        vaesef.vs v4, v8
+# CHECK-NEXT:  1      2     1.00                        vaesem.vv v4, v8
+# CHECK-NEXT:  1      2     1.00                        vaesem.vs v4, v8
+# CHECK-NEXT:  1      2     1.00                        vaesdm.vv v4, v8
+# CHECK-NEXT:  1      2     1.00                        vaesdm.vs v4, v8
+# CHECK-NEXT:  1      2     1.00                        vaeskf1.vi  v4, v8, 8
+# CHECK-NEXT:  1      2     1.00                        vaeskf2.vi  v4, v8, 8
+# CHECK-NEXT:  1      2     1.00                        vaesz.vs  v4, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      2     1.00                        vaesef.vv v4, v8
+# CHECK-NEXT:  1      2     1.00                        vaesef.vs v4, v8
+# CHECK-NEXT:  1      2     1.00                        vaesem.vv v4, v8
+# CHECK-NEXT:  1      2     1.00                        vaesem.vs v4, v8
+# CHECK-NEXT:  1      2     1.00                        vaesdm.vv v4, v8
+# CHECK-NEXT:  1      2     1.00                        vaesdm.vs v4, v8
+# CHECK-NEXT:  1      2     1.00                        vaeskf1.vi  v4, v8, 8
+# CHECK-NEXT:  1      2     1.00                        vaeskf2.vi  v4, v8, 8
+# CHECK-NEXT:  1      2     1.00                        vaesz.vs  v4, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      2     2.00                        vaesef.vv v4, v8
+# CHECK-NEXT:  1      2     2.00                        vaesef.vs v4, v8
+# CHECK-NEXT:  1      2     2.00                        vaesem.vv v4, v8
+# CHECK-NEXT:  1      2     2.00                        vaesem.vs v4, v8
+# CHECK-NEXT:  1      2     2.00                        vaesdm.vv v4, v8
+# CHECK-NEXT:  1      2     2.00                        vaesdm.vs v4, v8
+# CHECK-NEXT:  1      2     2.00                        vaeskf1.vi  v4, v8, 8
+# CHECK-NEXT:  1      2     2.00                        vaeskf2.vi  v4, v8, 8
+# CHECK-NEXT:  1      2     2.00                        vaesz.vs  v4, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      2     4.00                        vaesef.vv v4, v8
+# CHECK-NEXT:  1      2     4.00                        vaesef.vs v4, v8
+# CHECK-NEXT:  1      2     4.00                        vaesem.vv v4, v8
+# CHECK-NEXT:  1      2     4.00                        vaesem.vs v4, v8
+# CHECK-NEXT:  1      2     4.00                        vaesdm.vv v4, v8
+# CHECK-NEXT:  1      2     4.00                        vaesdm.vs v4, v8
+# CHECK-NEXT:  1      2     4.00                        vaeskf1.vi  v4, v8, 8
+# CHECK-NEXT:  1      2     4.00                        vaeskf2.vi  v4, v8, 8
+# CHECK-NEXT:  1      2     4.00                        vaesz.vs  v4, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      2     8.00                        vaesef.vv v8, v16
+# CHECK-NEXT:  1      2     8.00                        vaesef.vs v8, v16
+# CHECK-NEXT:  1      2     8.00                        vaesem.vv v8, v16
+# CHECK-NEXT:  1      2     8.00                        vaesem.vs v8, v16
+# CHECK-NEXT:  1      2     8.00                        vaesdm.vv v8, v16
+# CHECK-NEXT:  1      2     8.00                        vaesdm.vs v8, v16
+# CHECK-NEXT:  1      2     8.00                        vaeskf1.vi  v8, v16, 8
+# CHECK-NEXT:  1      2     8.00                        vaeskf2.vi  v8, v16, 8
+# CHECK-NEXT:  1      2     8.00                        vaesz.vs  v8, v16
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - SiFiveP400Div
+# CHECK-NEXT: [1]   - SiFiveP400FEXQ0
+# CHECK-NEXT: [2]   - SiFiveP400FloatDiv
+# CHECK-NEXT: [3]   - SiFiveP400IEXQ0
+# CHECK-NEXT: [4]   - SiFiveP400IEXQ1
+# CHECK-NEXT: [5]   - SiFiveP400IEXQ2
+# CHECK-NEXT: [6]   - SiFiveP400Load
+# CHECK-NEXT: [7]   - SiFiveP400Store
+# CHECK-NEXT: [8]   - SiFiveP400VDiv
+# CHECK-NEXT: [9]   - SiFiveP400VEXQ0
+# CHECK-NEXT: [10]  - SiFiveP400VFloatDiv
+# CHECK-NEXT: [11]  - SiFiveP400VLD
+# CHECK-NEXT: [12]  - SiFiveP400VST
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]
+# CHECK-NEXT:  -      -      -      -     5.00    -      -      -      -     144.00  -      -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   Instructions:
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vaesef.vv  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vaesef.vs  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vaesem.vv  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vaesem.vs  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vaesdm.vv  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vaesdm.vs  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vaeskf1.vi v4, v8, 8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vaeskf2.vi v4, v8, 8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vaesz.vs v4, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vaesef.vv  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vaesef.vs  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vaesem.vv  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vaesem.vs  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vaesdm.vv  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vaesdm.vs  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vaeskf1.vi v4, v8, 8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vaeskf2.vi v4, v8, 8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vaesz.vs v4, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vaesef.vv  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vaesef.vs  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vaesem.vv  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vaesem.vs  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vaesdm.vv  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vaesdm.vs  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vaeskf1.vi v4, v8, 8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vaeskf2.vi v4, v8, 8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vaesz.vs v4, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vaesef.vv  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vaesef.vs  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vaesem.vv  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vaesem.vs  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vaesdm.vv  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vaesdm.vs  v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vaeskf1.vi v4, v8, 8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vaeskf2.vi v4, v8, 8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vaesz.vs v4, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vaesef.vv  v8, v16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vaesef.vs  v8, v16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vaesem.vv  v8, v16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vaesem.vs  v8, v16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vaesdm.vv  v8, v16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vaesdm.vs  v8, v16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vaeskf1.vi v8, v16, 8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vaeskf2.vi v8, v16, 8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vaesz.vs v8, v16
+
diff --git a/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/zvknhb.s b/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/zvknhb.s
new file mode 100644
index 000000000000000..285d21cbd33d53f
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/zvknhb.s
@@ -0,0 +1,152 @@
+
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=riscv64 -mcpu=sifive-p470 -iterations=1 < %s | FileCheck %s
+
+# SEW is only e32 or e64
+
+vsetvli zero, zero, e32, m1, tu, mu
+vsha2ms.vv v4, v8, v12
+vsha2ch.vv v4, v8, v12
+vsha2cl.vv v4, v8, v12
+
+vsetvli zero, zero, e32, m2, tu, mu
+vsha2ms.vv v4, v8, v12
+vsha2ch.vv v4, v8, v12
+vsha2cl.vv v4, v8, v12
+
+vsetvli zero, zero, e32, m4, tu, mu
+vsha2ms.vv v4, v8, v12
+vsha2ch.vv v4, v8, v12
+vsha2cl.vv v4, v8, v12
+
+vsetvli zero, zero, e32, m8, tu, mu
+vsha2ms.vv v8, v16, v24
+vsha2ch.vv v8, v16, v24
+vsha2cl.vv v8, v16, v24
+
+vsetvli zero, zero, e64, m1, tu, mu
+vsha2ms.vv v4, v8, v12
+vsha2ch.vv v4, v8, v12
+vsha2cl.vv v4, v8, v12
+
+vsetvli zero, zero, e64, m2, tu, mu
+vsha2ms.vv v4, v8, v12
+vsha2ch.vv v4, v8, v12
+vsha2cl.vv v4, v8, v12
+
+vsetvli zero, zero, e64, m4, tu, mu
+vsha2ms.vv v4, v8, v12
+vsha2ch.vv v4, v8, v12
+vsha2cl.vv v4, v8, v12
+
+vsetvli zero, zero, e64, m8, tu, mu
+vsha2ms.vv v8, v16, v24
+vsha2ch.vv v8, v16, v24
+vsha2cl.vv v8, v16, v24
+
+# CHECK:      Iterations:        1
+# CHECK-NEXT: Instructions:      32
+# CHECK-NEXT: Total Cycles:      102
+# CHECK-NEXT: Total uOps:        32
+
+# CHECK:      Dispatch Width:    3
+# CHECK-NEXT: uOps Per Cycle:    0.31
+# CHECK-NEXT: IPC:               0.31
+# CHECK-NEXT: Block RThroughput: 90.0
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      3     1.00                        vsha2ms.vv  v4, v8, v12
+# CHECK-NEXT:  1      3     1.00                        vsha2ch.vv  v4, v8, v12
+# CHECK-NEXT:  1      3     1.00                        vsha2cl.vv  v4, v8, v12
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      3     2.00                        vsha2ms.vv  v4, v8, v12
+# CHECK-NEXT:  1      3     2.00                        vsha2ch.vv  v4, v8, v12
+# CHECK-NEXT:  1      3     2.00                        vsha2cl.vv  v4, v8, v12
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      3     4.00                        vsha2ms.vv  v4, v8, v12
+# CHECK-NEXT:  1      3     4.00                        vsha2ch.vv  v4, v8, v12
+# CHECK-NEXT:  1      3     4.00                        vsha2cl.vv  v4, v8, v12
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      3     8.00                        vsha2ms.vv  v8, v16, v24
+# CHECK-NEXT:  1      3     8.00                        vsha2ch.vv  v8, v16, v24
+# CHECK-NEXT:  1      3     8.00                        vsha2cl.vv  v8, v16, v24
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  1      3     1.00                        vsha2ms.vv  v4, v8, v12
+# CHECK-NEXT:  1      3     1.00                        vsha2ch.vv  v4, v8, v12
+# CHECK-NEXT:  1      3     1.00                        vsha2cl.vv  v4, v8, v12
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  1      3     2.00                        vsha2ms.vv  v4, v8, v12
+# CHECK-NEXT:  1      3     2.00                        vsha2ch.vv  v4, v8, v12
+# CHECK-NEXT:  1      3     2.00                        vsha2cl.vv  v4, v8, v12
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  1      3     4.00                        vsha2ms.vv  v4, v8, v12
+# CHECK-NEXT:  1      3     4.00                        vsha2ch.vv  v4, v8, v12
+# CHECK-NEXT:  1      3     4.00                        vsha2cl.vv  v4, v8, v12
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  1      3     8.00                        vsha2ms.vv  v8, v16, v24
+# CHECK-NEXT:  1      3     8.00                        vsha2ch.vv  v8, v16, v24
+# CHECK-NEXT:  1      3     8.00                        vsha2cl.vv  v8, v16, v24
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - SiFiveP400Div
+# CHECK-NEXT: [1]   - SiFiveP400FEXQ0
+# CHECK-NEXT: [2]   - SiFiveP400FloatDiv
+# CHECK-NEXT: [3]   - SiFiveP400IEXQ0
+# CHECK-NEXT: [4]   - SiFiveP400IEXQ1
+# CHECK-NEXT: [5]   - SiFiveP400IEXQ2
+# CHECK-NEXT: [6]   - SiFiveP400Load
+# CHECK-NEXT: [7]   - SiFiveP400Store
+# CHECK-NEXT: [8]   - SiFiveP400VDiv
+# CHECK-NEXT: [9]   - SiFiveP400VEXQ0
+# CHECK-NEXT: [10]  - SiFiveP400VFloatDiv
+# CHECK-NEXT: [11]  - SiFiveP400VLD
+# CHECK-NEXT: [12]  - SiFiveP400VST
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]
+# CHECK-NEXT:  -      -      -      -     8.00    -      -      -      -     90.00   -      -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   Instructions:
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsha2ms.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsha2ch.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsha2cl.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vsha2ms.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vsha2ch.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vsha2cl.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vsha2ms.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vsha2ch.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vsha2cl.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vsha2ms.vv v8, v16, v24
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vsha2ch.vv v8, v16, v24
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vsha2cl.vv v8, v16, v24
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsha2ms.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsha2ch.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsha2cl.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vsha2ms.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vsha2ch.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vsha2cl.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vsha2ms.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vsha2ch.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vsha2cl.vv v4, v8, v12
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e64, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vsha2ms.vv v8, v16, v24
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vsha2ch.vv v8, v16, v24
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vsha2cl.vv v8, v16, v24
+
diff --git a/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/zvksed.s b/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/zvksed.s
new file mode 100644
index 000000000000000..127998f34aa38a7
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/zvksed.s
@@ -0,0 +1,113 @@
+
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=riscv64 -mcpu=sifive-p470 -iterations=1 < %s | FileCheck %s
+
+# These instructions only support e32
+
+vsetvli zero, zero, e32, mf2, tu, mu
+vsm4k.vi v4, v8, 8
+vsm4r.vv v4, v8
+vsm4r.vs v4, v8
+
+vsetvli zero, zero, e32, m1, tu, mu
+vsm4k.vi v4, v8, 8
+vsm4r.vv v4, v8
+vsm4r.vs v4, v8
+
+vsetvli zero, zero, e32, m2, tu, mu
+vsm4k.vi v4, v8, 8
+vsm4r.vv v4, v8
+vsm4r.vs v4, v8
+
+vsetvli zero, zero, e32, m4, tu, mu
+vsm4k.vi v4, v8, 8
+vsm4r.vv v4, v8
+vsm4r.vs v4, v8
+
+vsetvli zero, zero, e32, m8, tu, mu
+vsm4k.vi v8, v16, 8
+vsm4r.vv v8, v16
+vsm4r.vs v8, v16
+
+# CHECK:      Iterations:        1
+# CHECK-NEXT: Instructions:      20
+# CHECK-NEXT: Total Cycles:      47
+# CHECK-NEXT: Total uOps:        20
+
+# CHECK:      Dispatch Width:    3
+# CHECK-NEXT: uOps Per Cycle:    0.43
+# CHECK-NEXT: IPC:               0.43
+# CHECK-NEXT: Block RThroughput: 48.0
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      3     1.00                        vsm4k.vi  v4, v8, 8
+# CHECK-NEXT:  1      3     1.00                        vsm4r.vv  v4, v8
+# CHECK-NEXT:  1      3     1.00                        vsm4r.vs  v4, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      3     1.00                        vsm4k.vi  v4, v8, 8
+# CHECK-NEXT:  1      3     1.00                        vsm4r.vv  v4, v8
+# CHECK-NEXT:  1      3     1.00                        vsm4r.vs  v4, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      3     2.00                        vsm4k.vi  v4, v8, 8
+# CHECK-NEXT:  1      3     2.00                        vsm4r.vv  v4, v8
+# CHECK-NEXT:  1      3     2.00                        vsm4r.vs  v4, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      3     4.00                        vsm4k.vi  v4, v8, 8
+# CHECK-NEXT:  1      3     4.00                        vsm4r.vv  v4, v8
+# CHECK-NEXT:  1      3     4.00                        vsm4r.vs  v4, v8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      3     8.00                        vsm4k.vi  v8, v16, 8
+# CHECK-NEXT:  1      3     8.00                        vsm4r.vv  v8, v16
+# CHECK-NEXT:  1      3     8.00                        vsm4r.vs  v8, v16
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - SiFiveP400Div
+# CHECK-NEXT: [1]   - SiFiveP400FEXQ0
+# CHECK-NEXT: [2]   - SiFiveP400FloatDiv
+# CHECK-NEXT: [3]   - SiFiveP400IEXQ0
+# CHECK-NEXT: [4]   - SiFiveP400IEXQ1
+# CHECK-NEXT: [5]   - SiFiveP400IEXQ2
+# CHECK-NEXT: [6]   - SiFiveP400Load
+# CHECK-NEXT: [7]   - SiFiveP400Store
+# CHECK-NEXT: [8]   - SiFiveP400VDiv
+# CHECK-NEXT: [9]   - SiFiveP400VEXQ0
+# CHECK-NEXT: [10]  - SiFiveP400VFloatDiv
+# CHECK-NEXT: [11]  - SiFiveP400VLD
+# CHECK-NEXT: [12]  - SiFiveP400VST
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]
+# CHECK-NEXT:  -      -      -      -     5.00    -      -      -      -     48.00   -      -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   Instructions:
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsm4k.vi v4, v8, 8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsm4r.vv v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsm4r.vs v4, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsm4k.vi v4, v8, 8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsm4r.vv v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsm4r.vs v4, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vsm4k.vi v4, v8, 8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vsm4r.vv v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vsm4r.vs v4, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vsm4k.vi v4, v8, 8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vsm4r.vv v4, v8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vsm4r.vs v4, v8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vsm4k.vi v8, v16, 8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vsm4r.vv v8, v16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vsm4r.vs v8, v16
+
diff --git a/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/zvksh.s b/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/zvksh.s
new file mode 100644
index 000000000000000..4c7b4dce8b6f486
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/RISCV/SiFiveP400/zvksh.s
@@ -0,0 +1,98 @@
+
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=riscv64 -mcpu=sifive-p470 -iterations=1 < %s | FileCheck %s
+
+# These instructions only support e32
+
+vsetvli zero, zero, e32, mf2, tu, mu
+vsm3me.vv v4, v8, v12
+vsm3c.vi v4, v8, 8
+
+vsetvli zero, zero, e32, m1, tu, mu
+vsm3me.vv v4, v8, v12
+vsm3c.vi v4, v8, 8
+
+vsetvli zero, zero, e32, m2, tu, mu
+vsm3me.vv v4, v8, v12
+vsm3c.vi v4, v8, 8
+
+vsetvli zero, zero, e32, m4, tu, mu
+vsm3me.vv v4, v8, v12
+vsm3c.vi v4, v8, 8
+
+vsetvli zero, zero, e32, m8, tu, mu
+vsm3me.vv v8, v16, v24
+vsm3c.vi  v8, v16, 8
+
+# CHECK:      Iterations:        1
+# CHECK-NEXT: Instructions:      15
+# CHECK-NEXT: Total Cycles:      28
+# CHECK-NEXT: Total uOps:        15
+
+# CHECK:      Dispatch Width:    3
+# CHECK-NEXT: uOps Per Cycle:    0.54
+# CHECK-NEXT: IPC:               0.54
+# CHECK-NEXT: Block RThroughput: 25.0
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  1      6     1.00                        vsm3me.vv v4, v8, v12
+# CHECK-NEXT:  1      3     1.00                        vsm3c.vi  v4, v8, 8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  1      6     1.00                        vsm3me.vv v4, v8, v12
+# CHECK-NEXT:  1      3     1.00                        vsm3c.vi  v4, v8, 8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  1      6     2.00                        vsm3me.vv v4, v8, v12
+# CHECK-NEXT:  1      3     1.00                        vsm3c.vi  v4, v8, 8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  1      6     4.00                        vsm3me.vv v4, v8, v12
+# CHECK-NEXT:  1      3     2.00                        vsm3c.vi  v4, v8, 8
+# CHECK-NEXT:  1      1     1.00                  U     vsetvli zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  1      6     8.00                        vsm3me.vv v8, v16, v24
+# CHECK-NEXT:  1      3     4.00                        vsm3c.vi  v8, v16, 8
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - SiFiveP400Div
+# CHECK-NEXT: [1]   - SiFiveP400FEXQ0
+# CHECK-NEXT: [2]   - SiFiveP400FloatDiv
+# CHECK-NEXT: [3]   - SiFiveP400IEXQ0
+# CHECK-NEXT: [4]   - SiFiveP400IEXQ1
+# CHECK-NEXT: [5]   - SiFiveP400IEXQ2
+# CHECK-NEXT: [6]   - SiFiveP400Load
+# CHECK-NEXT: [7]   - SiFiveP400Store
+# CHECK-NEXT: [8]   - SiFiveP400VDiv
+# CHECK-NEXT: [9]   - SiFiveP400VEXQ0
+# CHECK-NEXT: [10]  - SiFiveP400VFloatDiv
+# CHECK-NEXT: [11]  - SiFiveP400VLD
+# CHECK-NEXT: [12]  - SiFiveP400VST
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]
+# CHECK-NEXT:  -      -      -      -     5.00    -      -      -      -     25.00   -      -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   Instructions:
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, mf2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsm3me.vv  v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsm3c.vi v4, v8, 8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m1, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsm3me.vv  v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsm3c.vi v4, v8, 8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m2, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vsm3me.vv  v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -     vsm3c.vi v4, v8, 8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m4, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vsm3me.vv  v4, v8, v12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -     vsm3c.vi v4, v8, 8
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -      -      -      -      -     vsetvli  zero, zero, e32, m8, tu, mu
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     8.00    -      -      -     vsm3me.vv  v8, v16, v24
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     4.00    -      -      -     vsm3c.vi v8, v16, 8
+

From ea28668f8c980c9987b0e2f305b0a8eff9797401 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell@arm.com>
Date: Mon, 19 Aug 2024 14:55:09 +0100
Subject: [PATCH 360/441] [mlir][ArmSME] Remove XFAILs (#104758)

Buildbots have been updated to QEMU 9.0.2:
https://github.com/llvm/llvm-project/pull/104758#issuecomment-2296592845
---
 .../CPU/ArmSME/multi-tile-matmul-mixed-types.mlir      | 10 ++--------
 .../Vector/CPU/ArmSME/outerproduct-i8i8i32.mlir        | 10 ++--------
 2 files changed, 4 insertions(+), 16 deletions(-)

diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/multi-tile-matmul-mixed-types.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/multi-tile-matmul-mixed-types.mlir
index 8a4c8688d647482..78815a38612e960 100644
--- a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/multi-tile-matmul-mixed-types.mlir
+++ b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/multi-tile-matmul-mixed-types.mlir
@@ -13,14 +13,8 @@
 /// tests a mixed i8 to i32 matmul and outer product fusion which fuses 16
 /// outer products (four per tile) into four 4-way outer products.
 
-/// NOTE: QEMU gives incorrect result for SME SMOPA 4-way outer product
-/// instruction (version <= 8.2.0, latest version at time of writing), see:
-/// https://gitlab.com/qemu-project/qemu/-/issues/2083
-/// This test is expected to fail until a fixed version of QEMU can be used.
-
-/// FIXME: Remove the 'XFAIL' below once a fixed QEMU version is available
-/// (and installed on CI buildbot).
-/// XFAIL: *
+// NOTE: QEMU <= 8.2.0 gives incorrect result for SME SMOPA 4-way outer product
+// instruction see: https://gitlab.com/qemu-project/qemu/-/issues/2083.
 
 func.func @matmul_i8_to_i32(%A : tensor<?x?xi8>, %B : tensor<?x?xi8>, %C : tensor<?x?xi32>) {
   %res = linalg.matmul ins(%A, %B: tensor<?x?xi8>, tensor<?x?xi8>)
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/outerproduct-i8i8i32.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/outerproduct-i8i8i32.mlir
index 653b0d7beb2c78e..74ddbc63380b70e 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/outerproduct-i8i8i32.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/outerproduct-i8i8i32.mlir
@@ -7,14 +7,8 @@
 
 // RUN: %{compile} | %{run} | FileCheck %s
 
-// NOTE: QEMU gives incorrect result for SME SMOPA 4-way outer product
-// instruction (version <= 8.2.0, latest version at time of writing), see:
-// https://gitlab.com/qemu-project/qemu/-/issues/2083 This test is expected to
-// fail (CHECK lines are correct) until a fixed version of QEMU can be used.
-
-// FIXME: Remove the 'XFAIL' below once a fixed QEMU version is available
-// (and installed on CI buildbot).
-// XFAIL: *
+// NOTE: QEMU <= 8.2.0 gives incorrect result for SME SMOPA 4-way outer product
+// instruction see: https://gitlab.com/qemu-project/qemu/-/issues/2083.
 
 // NOTE: there is no non-widening variant for these types and this test can't
 // be lowered without the widening pass, therefore we can't check if the result

From 4f083c0a2cb1d19942406f7e6e97ae47e8ac91b1 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Mon, 19 Aug 2024 15:38:24 +0200
Subject: [PATCH 361/441] [DebugInfo] Make tests SimplifyCFG-independent (NFC)

Run SimplifyCFG over the test input and disable the SimplifyCFG
run in the backend, so that these tests are not affected by changes
to SimplifyCFG.
---
 .../COFF/AArch64/arm64-register-variables.ll  |  78 ++++++------
 .../COFF/ARMNT/arm-register-variables.ll      | 111 +++++++++---------
 2 files changed, 89 insertions(+), 100 deletions(-)

diff --git a/llvm/test/DebugInfo/COFF/AArch64/arm64-register-variables.ll b/llvm/test/DebugInfo/COFF/AArch64/arm64-register-variables.ll
index 834e8f4ce7fed16..950594ed105d631 100644
--- a/llvm/test/DebugInfo/COFF/AArch64/arm64-register-variables.ll
+++ b/llvm/test/DebugInfo/COFF/AArch64/arm64-register-variables.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -mtriple=arm64-windows -filetype=obj | llvm-readobj --codeview - | FileCheck %s --check-prefix=OBJ
+; RUN: llc < %s -mtriple=arm64-windows -filetype=obj -aarch64-enable-atomic-cfg-tidy=0 | llvm-readobj --codeview - | FileCheck %s --check-prefix=OBJ
 
 ; Generated from:
 ; volatile int x;
@@ -41,56 +41,50 @@ target triple = "arm64-unknown-windows-msvc19.16.27023"
 
 @x = common dso_local global i32 0, align 4, !dbg !0
 
-; Function Attrs: noinline nounwind optnone uwtable
-define dso_local void @f(i32 %p) #0 !dbg !12 {
+define dso_local void @f(i32 %p) !dbg !11 {
 entry:
   %p.addr = alloca i32, align 4
   %a = alloca i32, align 4
   %b = alloca i32, align 4
   %c = alloca i32, align 4
   store i32 %p, ptr %p.addr, align 4
-  call void @llvm.dbg.declare(metadata ptr %p.addr, metadata !15, metadata !DIExpression()), !dbg !16
-  %0 = load i32, ptr %p.addr, align 4, !dbg !17
-  %tobool = icmp ne i32 %0, 0, !dbg !17
-  br i1 %tobool, label %if.then, label %if.else, !dbg !17
+    #dbg_declare(ptr %p.addr, !14, !DIExpression(), !15)
+  %0 = load i32, ptr %p.addr, align 4, !dbg !16
+  %tobool = icmp ne i32 %0, 0, !dbg !16
+  %call = call i32 @getint(), !dbg !17
+  br i1 %tobool, label %if.then, label %if.else, !dbg !16
 
 if.then:                                          ; preds = %entry
-  call void @llvm.dbg.declare(metadata ptr %a, metadata !18, metadata !DIExpression()), !dbg !21
-  %call = call i32 @getint(), !dbg !21
+    #dbg_declare(ptr %a, !19, !DIExpression(), !21)
   store i32 %call, ptr %a, align 4, !dbg !21
-  call void @llvm.dbg.declare(metadata ptr %b, metadata !22, metadata !DIExpression()), !dbg !23
+    #dbg_declare(ptr %b, !22, !DIExpression(), !23)
   %1 = load i32, ptr %a, align 4, !dbg !23
   %call1 = call i32 @inlineinc(i32 %1), !dbg !23
   store i32 %call1, ptr %b, align 4, !dbg !23
   %2 = load i32, ptr %b, align 4, !dbg !24
-  call void @putint(i32 %2), !dbg !24
   br label %if.end, !dbg !25
 
 if.else:                                          ; preds = %entry
-  call void @llvm.dbg.declare(metadata ptr %c, metadata !26, metadata !DIExpression()), !dbg !28
-  %call2 = call i32 @getint(), !dbg !28
-  store i32 %call2, ptr %c, align 4, !dbg !28
+    #dbg_declare(ptr %c, !26, !DIExpression(), !28)
+  store i32 %call, ptr %c, align 4, !dbg !28
   %3 = load i32, ptr %c, align 4, !dbg !29
-  call void @putint(i32 %3), !dbg !29
   br label %if.end, !dbg !30
 
 if.end:                                           ; preds = %if.else, %if.then
+  %.sink = phi i32 [ %3, %if.else ], [ %2, %if.then ]
+  call void @putint(i32 %.sink), !dbg !17
   ret void, !dbg !31
 }
 
-; Function Attrs: nounwind readnone speculatable
-declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+declare dso_local i32 @getint()
 
-declare dso_local i32 @getint() #2
-
-; Function Attrs: noinline nounwind optnone uwtable
-define internal i32 @inlineinc(i32 %a) #0 !dbg !32 {
+define internal i32 @inlineinc(i32 %a) !dbg !32 {
 entry:
   %a.addr = alloca i32, align 4
   %b = alloca i32, align 4
   store i32 %a, ptr %a.addr, align 4
-  call void @llvm.dbg.declare(metadata ptr %a.addr, metadata !35, metadata !DIExpression()), !dbg !36
-  call void @llvm.dbg.declare(metadata ptr %b, metadata !37, metadata !DIExpression()), !dbg !38
+    #dbg_declare(ptr %a.addr, !35, !DIExpression(), !36)
+    #dbg_declare(ptr %b, !37, !DIExpression(), !38)
   %0 = load i32, ptr %a.addr, align 4, !dbg !38
   %add = add nsw i32 %0, 1, !dbg !38
   store i32 %add, ptr %b, align 4, !dbg !38
@@ -101,7 +95,7 @@ entry:
   ret i32 %2, !dbg !40
 }
 
-declare dso_local void @putint(i32) #2
+declare dso_local void @putint(i32)
 
 !llvm.dbg.cu = !{!2}
 !llvm.module.flags = !{!8, !9, !10}
@@ -109,7 +103,7 @@ declare dso_local void @putint(i32) #2
 !0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
 !1 = distinct !DIGlobalVariable(name: "x", scope: !2, file: !3, line: 1, type: !6, isLocal: false, isDefinition: true)
 !2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 9.0.0 (trunk 361867) (llvm/trunk 361866)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5, nameTableKind: None)
-!3 = !DIFile(filename: "t.c", directory: "S:\5CLLVM\5Csvn\5Csbr\5Cbin", checksumkind: CSK_MD5, checksum: "734c448e95a6204a439a847ed063e5ce")
+!3 = !DIFile(filename: "t.c", directory: "S:\\LLVM\\svn\\sbr\\bin", checksumkind: CSK_MD5, checksum: "734c448e95a6204a439a847ed063e5ce")
 !4 = !{}
 !5 = !{!0}
 !6 = !DIDerivedType(tag: DW_TAG_volatile_type, baseType: !7)
@@ -117,27 +111,27 @@ declare dso_local void @putint(i32) #2
 !8 = !{i32 2, !"CodeView", i32 1}
 !9 = !{i32 2, !"Debug Info Version", i32 3}
 !10 = !{i32 1, !"wchar_size", i32 2}
-!11 = !{!"clang version 9.0.0 (trunk 361867) (llvm/trunk 361866)"}
-!12 = distinct !DISubprogram(name: "f", scope: !3, file: !3, line: 9, type: !13, scopeLine: 9, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !2, retainedNodes: !4)
-!13 = !DISubroutineType(types: !14)
-!14 = !{null, !7}
-!15 = !DILocalVariable(name: "p", arg: 1, scope: !12, file: !3, line: 9, type: !7)
-!16 = !DILocation(line: 9, scope: !12)
-!17 = !DILocation(line: 10, scope: !12)
-!18 = !DILocalVariable(name: "a", scope: !19, file: !3, line: 11, type: !7)
-!19 = distinct !DILexicalBlock(scope: !20, file: !3, line: 10)
-!20 = distinct !DILexicalBlock(scope: !12, file: !3, line: 10)
-!21 = !DILocation(line: 11, scope: !19)
-!22 = !DILocalVariable(name: "b", scope: !19, file: !3, line: 12, type: !7)
-!23 = !DILocation(line: 12, scope: !19)
-!24 = !DILocation(line: 13, scope: !19)
-!25 = !DILocation(line: 14, scope: !19)
+!11 = distinct !DISubprogram(name: "f", scope: !3, file: !3, line: 9, type: !12, scopeLine: 9, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !2, retainedNodes: !4)
+!12 = !DISubroutineType(types: !13)
+!13 = !{null, !7}
+!14 = !DILocalVariable(name: "p", arg: 1, scope: !11, file: !3, line: 9, type: !7)
+!15 = !DILocation(line: 9, scope: !11)
+!16 = !DILocation(line: 10, scope: !11)
+!17 = !DILocation(line: 0, scope: !18)
+!18 = distinct !DILexicalBlock(scope: !11, file: !3, line: 10)
+!19 = !DILocalVariable(name: "a", scope: !20, file: !3, line: 11, type: !7)
+!20 = distinct !DILexicalBlock(scope: !18, file: !3, line: 10)
+!21 = !DILocation(line: 11, scope: !20)
+!22 = !DILocalVariable(name: "b", scope: !20, file: !3, line: 12, type: !7)
+!23 = !DILocation(line: 12, scope: !20)
+!24 = !DILocation(line: 13, scope: !20)
+!25 = !DILocation(line: 14, scope: !20)
 !26 = !DILocalVariable(name: "c", scope: !27, file: !3, line: 15, type: !7)
-!27 = distinct !DILexicalBlock(scope: !20, file: !3, line: 14)
+!27 = distinct !DILexicalBlock(scope: !18, file: !3, line: 14)
 !28 = !DILocation(line: 15, scope: !27)
 !29 = !DILocation(line: 16, scope: !27)
 !30 = !DILocation(line: 17, scope: !27)
-!31 = !DILocation(line: 18, scope: !12)
+!31 = !DILocation(line: 18, scope: !11)
 !32 = distinct !DISubprogram(name: "inlineinc", scope: !3, file: !3, line: 4, type: !33, scopeLine: 4, flags: DIFlagPrototyped, spFlags: DISPFlagLocalToUnit | DISPFlagDefinition, unit: !2, retainedNodes: !4)
 !33 = !DISubroutineType(types: !34)
 !34 = !{!7, !7}
diff --git a/llvm/test/DebugInfo/COFF/ARMNT/arm-register-variables.ll b/llvm/test/DebugInfo/COFF/ARMNT/arm-register-variables.ll
index 1eb8dff00e58938..079c14eeb3abf3e 100644
--- a/llvm/test/DebugInfo/COFF/ARMNT/arm-register-variables.ll
+++ b/llvm/test/DebugInfo/COFF/ARMNT/arm-register-variables.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -filetype=obj | llvm-readobj --codeview - | FileCheck %s --check-prefix=OBJ
+; RUN: llc < %s -filetype=obj -arm-atomic-cfg-tidy=0 | llvm-readobj --codeview - | FileCheck %s --check-prefix=OBJ
 
 ; Generated from:
 
@@ -57,7 +57,6 @@ target triple = "thumbv7-pc-windows-msvc19.11.0"
 
 @x = dso_local global i32 0, align 4, !dbg !0
 
-; Function Attrs: noinline nounwind optnone
 define dso_local arm_aapcs_vfpcc void @f(i32 %p) !dbg !14 {
 entry:
   %p.addr = alloca i32, align 4
@@ -65,56 +64,51 @@ entry:
   %b = alloca i32, align 4
   %c = alloca i32, align 4
   store i32 %p, ptr %p.addr, align 4
-  call void @llvm.dbg.declare(metadata ptr %p.addr, metadata !17, metadata !DIExpression()), !dbg !18
+    #dbg_declare(ptr %p.addr, !17, !DIExpression(), !18)
   %0 = load i32, ptr %p.addr, align 4, !dbg !19
   %tobool = icmp ne i32 %0, 0, !dbg !19
+  %call = call arm_aapcs_vfpcc i32 @getint(), !dbg !20
   br i1 %tobool, label %if.then, label %if.else, !dbg !19
 
 if.then:                                          ; preds = %entry
-  call void @llvm.dbg.declare(metadata ptr %a, metadata !20, metadata !DIExpression()), !dbg !23
-  %call = call arm_aapcs_vfpcc i32 @getint(), !dbg !23
-  store i32 %call, ptr %a, align 4, !dbg !23
-  call void @llvm.dbg.declare(metadata ptr %b, metadata !24, metadata !DIExpression()), !dbg !25
-  %1 = load i32, ptr %a, align 4, !dbg !25
-  %call1 = call arm_aapcs_vfpcc i32 @inlineinc(i32 %1), !dbg !25
-  store i32 %call1, ptr %b, align 4, !dbg !25
-  %2 = load i32, ptr %b, align 4, !dbg !26
-  call arm_aapcs_vfpcc void @putint(i32 %2), !dbg !26
-  br label %if.end, !dbg !27
+    #dbg_declare(ptr %a, !22, !DIExpression(), !24)
+  store i32 %call, ptr %a, align 4, !dbg !24
+    #dbg_declare(ptr %b, !25, !DIExpression(), !26)
+  %1 = load i32, ptr %a, align 4, !dbg !26
+  %call1 = call arm_aapcs_vfpcc i32 @inlineinc(i32 %1), !dbg !26
+  store i32 %call1, ptr %b, align 4, !dbg !26
+  %2 = load i32, ptr %b, align 4, !dbg !27
+  br label %if.end, !dbg !28
 
 if.else:                                          ; preds = %entry
-  call void @llvm.dbg.declare(metadata ptr %c, metadata !28, metadata !DIExpression()), !dbg !30
-  %call2 = call arm_aapcs_vfpcc i32 @getint(), !dbg !30
-  store i32 %call2, ptr %c, align 4, !dbg !30
-  %3 = load i32, ptr %c, align 4, !dbg !31
-  call arm_aapcs_vfpcc void @putint(i32 %3), !dbg !31
-  br label %if.end, !dbg !32
+    #dbg_declare(ptr %c, !29, !DIExpression(), !31)
+  store i32 %call, ptr %c, align 4, !dbg !31
+  %3 = load i32, ptr %c, align 4, !dbg !32
+  br label %if.end, !dbg !33
 
 if.end:                                           ; preds = %if.else, %if.then
-  ret void, !dbg !33
+  %.sink = phi i32 [ %3, %if.else ], [ %2, %if.then ]
+  call arm_aapcs_vfpcc void @putint(i32 %.sink), !dbg !20
+  ret void, !dbg !34
 }
 
-; Function Attrs: nounwind readnone speculatable willreturn
-declare void @llvm.dbg.declare(metadata, metadata, metadata)
-
 declare dso_local arm_aapcs_vfpcc i32 @getint()
 
-; Function Attrs: noinline nounwind optnone
-define internal arm_aapcs_vfpcc i32 @inlineinc(i32 %a) !dbg !34 {
+define internal arm_aapcs_vfpcc i32 @inlineinc(i32 %a) !dbg !35 {
 entry:
   %a.addr = alloca i32, align 4
   %b = alloca i32, align 4
   store i32 %a, ptr %a.addr, align 4
-  call void @llvm.dbg.declare(metadata ptr %a.addr, metadata !37, metadata !DIExpression()), !dbg !38
-  call void @llvm.dbg.declare(metadata ptr %b, metadata !39, metadata !DIExpression()), !dbg !40
-  %0 = load i32, ptr %a.addr, align 4, !dbg !40
-  %add = add nsw i32 %0, 1, !dbg !40
-  store i32 %add, ptr %b, align 4, !dbg !40
-  %1 = load volatile i32, ptr @x, align 4, !dbg !41
-  %inc = add nsw i32 %1, 1, !dbg !41
-  store volatile i32 %inc, ptr @x, align 4, !dbg !41
-  %2 = load i32, ptr %b, align 4, !dbg !42
-  ret i32 %2, !dbg !42
+    #dbg_declare(ptr %a.addr, !38, !DIExpression(), !39)
+    #dbg_declare(ptr %b, !40, !DIExpression(), !41)
+  %0 = load i32, ptr %a.addr, align 4, !dbg !41
+  %add = add nsw i32 %0, 1, !dbg !41
+  store i32 %add, ptr %b, align 4, !dbg !41
+  %1 = load volatile i32, ptr @x, align 4, !dbg !42
+  %inc = add nsw i32 %1, 1, !dbg !42
+  store volatile i32 %inc, ptr @x, align 4, !dbg !42
+  %2 = load i32, ptr %b, align 4, !dbg !43
+  ret i32 %2, !dbg !43
 }
 
 declare dso_local arm_aapcs_vfpcc void @putint(i32)
@@ -143,26 +137,27 @@ declare dso_local arm_aapcs_vfpcc void @putint(i32)
 !17 = !DILocalVariable(name: "p", arg: 1, scope: !14, file: !6, line: 9, type: !8)
 !18 = !DILocation(line: 9, scope: !14)
 !19 = !DILocation(line: 10, scope: !14)
-!20 = !DILocalVariable(name: "a", scope: !21, file: !6, line: 11, type: !8)
-!21 = distinct !DILexicalBlock(scope: !22, file: !6, line: 10)
-!22 = distinct !DILexicalBlock(scope: !14, file: !6, line: 10)
-!23 = !DILocation(line: 11, scope: !21)
-!24 = !DILocalVariable(name: "b", scope: !21, file: !6, line: 12, type: !8)
-!25 = !DILocation(line: 12, scope: !21)
-!26 = !DILocation(line: 13, scope: !21)
-!27 = !DILocation(line: 14, scope: !21)
-!28 = !DILocalVariable(name: "c", scope: !29, file: !6, line: 15, type: !8)
-!29 = distinct !DILexicalBlock(scope: !22, file: !6, line: 14)
-!30 = !DILocation(line: 15, scope: !29)
-!31 = !DILocation(line: 16, scope: !29)
-!32 = !DILocation(line: 17, scope: !29)
-!33 = !DILocation(line: 18, scope: !14)
-!34 = distinct !DISubprogram(name: "inlineinc", scope: !6, file: !6, line: 4, type: !35, scopeLine: 4, flags: DIFlagPrototyped, spFlags: DISPFlagLocalToUnit | DISPFlagDefinition, unit: !2, retainedNodes: !4)
-!35 = !DISubroutineType(types: !36)
-!36 = !{!8, !8}
-!37 = !DILocalVariable(name: "a", arg: 1, scope: !34, file: !6, line: 4, type: !8)
-!38 = !DILocation(line: 4, scope: !34)
-!39 = !DILocalVariable(name: "b", scope: !34, file: !6, line: 5, type: !8)
-!40 = !DILocation(line: 5, scope: !34)
-!41 = !DILocation(line: 6, scope: !34)
-!42 = !DILocation(line: 7, scope: !34)
+!20 = !DILocation(line: 0, scope: !21)
+!21 = distinct !DILexicalBlock(scope: !14, file: !6, line: 10)
+!22 = !DILocalVariable(name: "a", scope: !23, file: !6, line: 11, type: !8)
+!23 = distinct !DILexicalBlock(scope: !21, file: !6, line: 10)
+!24 = !DILocation(line: 11, scope: !23)
+!25 = !DILocalVariable(name: "b", scope: !23, file: !6, line: 12, type: !8)
+!26 = !DILocation(line: 12, scope: !23)
+!27 = !DILocation(line: 13, scope: !23)
+!28 = !DILocation(line: 14, scope: !23)
+!29 = !DILocalVariable(name: "c", scope: !30, file: !6, line: 15, type: !8)
+!30 = distinct !DILexicalBlock(scope: !21, file: !6, line: 14)
+!31 = !DILocation(line: 15, scope: !30)
+!32 = !DILocation(line: 16, scope: !30)
+!33 = !DILocation(line: 17, scope: !30)
+!34 = !DILocation(line: 18, scope: !14)
+!35 = distinct !DISubprogram(name: "inlineinc", scope: !6, file: !6, line: 4, type: !36, scopeLine: 4, flags: DIFlagPrototyped, spFlags: DISPFlagLocalToUnit | DISPFlagDefinition, unit: !2, retainedNodes: !4)
+!36 = !DISubroutineType(types: !37)
+!37 = !{!8, !8}
+!38 = !DILocalVariable(name: "a", arg: 1, scope: !35, file: !6, line: 4, type: !8)
+!39 = !DILocation(line: 4, scope: !35)
+!40 = !DILocalVariable(name: "b", scope: !35, file: !6, line: 5, type: !8)
+!41 = !DILocation(line: 5, scope: !35)
+!42 = !DILocation(line: 6, scope: !35)
+!43 = !DILocation(line: 7, scope: !35)

From 79f6ae05c139d3d5b6446f8a265a3c6e3f5b18f8 Mon Sep 17 00:00:00 2001
From: smanna12 <soumi.manna@intel.com>
Date: Mon, 19 Aug 2024 07:37:06 -0700
Subject: [PATCH 362/441] [Clang][NFC] Fix potential null dereference in
 encodeTypeForFunctionPointerAuth (#104737)

This patch replaces getAs with castAs in
encodeTypeForFunctionPointerAuth to prevent dereferencing a potential
null pointer, enhancing type safety as reported by static analyzer tool.
---
 clang/lib/AST/ASTContext.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index f35a025fc8edfc9..b201d201e1ea6ad 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -3279,7 +3279,7 @@ static void encodeTypeForFunctionPointerAuth(const ASTContext &Ctx,
 
   case Type::MemberPointer: {
     OS << "M";
-    const auto *MPT = T->getAs<MemberPointerType>();
+    const auto *MPT = T->castAs<MemberPointerType>();
     encodeTypeForFunctionPointerAuth(Ctx, OS, QualType(MPT->getClass(), 0));
     encodeTypeForFunctionPointerAuth(Ctx, OS, MPT->getPointeeType());
     return;

From c60da1a271a6bb271e7703b2f7c71fbece67ab78 Mon Sep 17 00:00:00 2001
From: Daniel Grumberg <dgrumberg@apple.com>
Date: Mon, 19 Aug 2024 15:57:43 +0100
Subject: [PATCH 363/441] [clang][ExtractAPI] Stop dropping fields of nested
 anonymous record types when they aren't attached to variable declaration
 (#104600)

- Introduce primitives for removing records from `APISet` and managing
the record chain of `RecordContext`
- Detect nested anonymous record types and remove them from the `APISet`
after they have been fully traversed and transfer ownership of child
records to the parent context (if any)
---
 clang/include/clang/ExtractAPI/API.h          | 25 +++++----
 .../clang/ExtractAPI/ExtractAPIVisitor.h      | 39 ++++++++++++--
 clang/lib/ExtractAPI/API.cpp                  | 53 ++++++++++++++++++-
 .../Serialization/SymbolGraphSerializer.cpp   |  8 ---
 .../ExtractAPI/anonymous_record_no_typedef.c  | 12 +++++
 5 files changed, 110 insertions(+), 27 deletions(-)

diff --git a/clang/include/clang/ExtractAPI/API.h b/clang/include/clang/ExtractAPI/API.h
index bf291074fd06143..188e35b72117b50 100644
--- a/clang/include/clang/ExtractAPI/API.h
+++ b/clang/include/clang/ExtractAPI/API.h
@@ -19,21 +19,13 @@
 #define LLVM_CLANG_EXTRACTAPI_API_H
 
 #include "clang/AST/Availability.h"
-#include "clang/AST/Decl.h"
 #include "clang/AST/DeclBase.h"
-#include "clang/AST/DeclObjC.h"
 #include "clang/AST/RawCommentList.h"
 #include "clang/Basic/SourceLocation.h"
-#include "clang/Basic/Specifiers.h"
 #include "clang/ExtractAPI/DeclarationFragments.h"
-#include "llvm/ADT/ArrayRef.h"
-#include "llvm/ADT/MapVector.h"
-#include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/Support/Allocator.h"
 #include "llvm/Support/Casting.h"
-#include "llvm/Support/Compiler.h"
-#include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/raw_ostream.h"
 #include "llvm/TargetParser/Triple.h"
 #include <cstddef>
 #include <iterator>
@@ -328,6 +320,8 @@ class RecordContext {
   /// chain.
   void stealRecordChain(RecordContext &Other);
 
+  void removeFromRecordChain(APIRecord *Record);
+
   APIRecord::RecordKind getKind() const { return Kind; }
 
   struct record_iterator {
@@ -1426,10 +1420,15 @@ class APISet {
   typename std::enable_if_t<std::is_base_of_v<APIRecord, RecordTy>, RecordTy> *
   createRecord(StringRef USR, StringRef Name, CtorArgsContTy &&...CtorArgs);
 
-  ArrayRef<const APIRecord *> getTopLevelRecords() const {
-    return TopLevelRecords;
+  auto getTopLevelRecords() const {
+    return llvm::iterator_range<decltype(TopLevelRecords)::iterator>(
+        TopLevelRecords);
   }
 
+  void removeRecord(StringRef USR);
+
+  void removeRecord(APIRecord *Record);
+
   APISet(const llvm::Triple &Target, Language Lang,
          const std::string &ProductName)
       : Target(Target), Lang(Lang), ProductName(ProductName) {}
@@ -1456,7 +1455,7 @@ class APISet {
   // lives in the BumpPtrAllocator.
   using APIRecordStoredPtr = std::unique_ptr<APIRecord, APIRecordDeleter>;
   llvm::DenseMap<StringRef, APIRecordStoredPtr> USRBasedLookupTable;
-  std::vector<const APIRecord *> TopLevelRecords;
+  llvm::SmallPtrSet<const APIRecord *, 32> TopLevelRecords;
 
 public:
   const std::string ProductName;
@@ -1482,7 +1481,7 @@ APISet::createRecord(StringRef USR, StringRef Name,
             dyn_cast_if_present<RecordContext>(Record->Parent.Record))
       ParentContext->addToRecordChain(Record);
     else
-      TopLevelRecords.push_back(Record);
+      TopLevelRecords.insert(Record);
   } else {
     Record = dyn_cast<RecordTy>(Result.first->second.get());
   }
diff --git a/clang/include/clang/ExtractAPI/ExtractAPIVisitor.h b/clang/include/clang/ExtractAPI/ExtractAPIVisitor.h
index 1b27027621666ae..8d9ac0620345117 100644
--- a/clang/include/clang/ExtractAPI/ExtractAPIVisitor.h
+++ b/clang/include/clang/ExtractAPI/ExtractAPIVisitor.h
@@ -23,13 +23,11 @@
 #include "clang/AST/RecursiveASTVisitor.h"
 #include "clang/Basic/LLVM.h"
 #include "clang/Basic/Module.h"
-#include "clang/Basic/SourceManager.h"
 #include "clang/Basic/Specifiers.h"
 #include "clang/ExtractAPI/API.h"
 #include "clang/ExtractAPI/DeclarationFragments.h"
 #include "clang/ExtractAPI/TypedefUnderlyingTypeResolver.h"
 #include "clang/Index/USRGeneration.h"
-#include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Casting.h"
 #include <type_traits>
@@ -40,6 +38,8 @@ namespace impl {
 
 template <typename Derived>
 class ExtractAPIVisitorBase : public RecursiveASTVisitor<Derived> {
+  using Base = RecursiveASTVisitor<Derived>;
+
 protected:
   ExtractAPIVisitorBase(ASTContext &Context, APISet &API)
       : Context(Context), API(API) {}
@@ -81,8 +81,10 @@ class ExtractAPIVisitorBase : public RecursiveASTVisitor<Derived> {
 
   bool VisitNamespaceDecl(const NamespaceDecl *Decl);
 
+  bool TraverseRecordDecl(RecordDecl *Decl);
   bool VisitRecordDecl(const RecordDecl *Decl);
 
+  bool TraverseCXXRecordDecl(CXXRecordDecl *Decl);
   bool VisitCXXRecordDecl(const CXXRecordDecl *Decl);
 
   bool VisitCXXMethodDecl(const CXXMethodDecl *Decl);
@@ -240,7 +242,7 @@ class ExtractAPIVisitorBase : public RecursiveASTVisitor<Derived> {
 
   bool isEmbeddedInVarDeclarator(const TagDecl &D) {
     return D.getName().empty() && getTypedefName(&D).empty() &&
-           D.isEmbeddedInDeclarator();
+           D.isEmbeddedInDeclarator() && !D.isFreeStanding();
   }
 
   void maybeMergeWithAnonymousTag(const DeclaratorDecl &D,
@@ -252,8 +254,10 @@ class ExtractAPIVisitorBase : public RecursiveASTVisitor<Derived> {
     clang::index::generateUSRForDecl(Tag, TagUSR);
     if (auto *Record = llvm::dyn_cast_if_present<TagRecord>(
             API.findRecordForUSR(TagUSR))) {
-      if (Record->IsEmbeddedInVarDeclarator)
+      if (Record->IsEmbeddedInVarDeclarator) {
         NewRecordContext->stealRecordChain(*Record);
+        API.removeRecord(Record);
+      }
     }
   }
 };
@@ -548,6 +552,19 @@ bool ExtractAPIVisitorBase<Derived>::VisitNamespaceDecl(
   return true;
 }
 
+template <typename Derived>
+bool ExtractAPIVisitorBase<Derived>::TraverseRecordDecl(RecordDecl *Decl) {
+  bool Ret = Base::TraverseRecordDecl(Decl);
+
+  if (!isEmbeddedInVarDeclarator(*Decl) && Decl->isAnonymousStructOrUnion()) {
+    SmallString<128> USR;
+    index::generateUSRForDecl(Decl, USR);
+    API.removeRecord(USR);
+  }
+
+  return Ret;
+}
+
 template <typename Derived>
 bool ExtractAPIVisitorBase<Derived>::VisitRecordDecl(const RecordDecl *Decl) {
   if (!getDerivedExtractAPIVisitor().shouldDeclBeIncluded(Decl))
@@ -588,6 +605,20 @@ bool ExtractAPIVisitorBase<Derived>::VisitRecordDecl(const RecordDecl *Decl) {
   return true;
 }
 
+template <typename Derived>
+bool ExtractAPIVisitorBase<Derived>::TraverseCXXRecordDecl(
+    CXXRecordDecl *Decl) {
+  bool Ret = Base::TraverseCXXRecordDecl(Decl);
+
+  if (!isEmbeddedInVarDeclarator(*Decl) && Decl->isAnonymousStructOrUnion()) {
+    SmallString<128> USR;
+    index::generateUSRForDecl(Decl, USR);
+    API.removeRecord(USR);
+  }
+
+  return Ret;
+}
+
 template <typename Derived>
 bool ExtractAPIVisitorBase<Derived>::VisitCXXRecordDecl(
     const CXXRecordDecl *Decl) {
diff --git a/clang/lib/ExtractAPI/API.cpp b/clang/lib/ExtractAPI/API.cpp
index ab1108f663deacc..9dbc023885c37f1 100644
--- a/clang/lib/ExtractAPI/API.cpp
+++ b/clang/lib/ExtractAPI/API.cpp
@@ -13,8 +13,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/ExtractAPI/API.h"
-#include "clang/AST/RawCommentList.h"
-#include "clang/Index/USRGeneration.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/ErrorHandling.h"
 #include <memory>
@@ -60,6 +58,10 @@ bool RecordContext::IsWellFormed() const {
 
 void RecordContext::stealRecordChain(RecordContext &Other) {
   assert(IsWellFormed());
+  // Other's record chain is empty, nothing to do
+  if (Other.First == nullptr && Other.Last == nullptr)
+    return;
+
   // If we don't have an empty chain append Other's chain into ours.
   if (First)
     Last->NextInContext = Other.First;
@@ -68,6 +70,10 @@ void RecordContext::stealRecordChain(RecordContext &Other) {
 
   Last = Other.Last;
 
+  for (auto *StolenRecord = Other.First; StolenRecord != nullptr;
+       StolenRecord = StolenRecord->getNextInContext())
+    StolenRecord->Parent = SymbolReference(cast<APIRecord>(this));
+
   // Delete Other's chain to ensure we don't accidentally traverse it.
   Other.First = nullptr;
   Other.Last = nullptr;
@@ -85,6 +91,22 @@ void RecordContext::addToRecordChain(APIRecord *Record) const {
   Last = Record;
 }
 
+void RecordContext::removeFromRecordChain(APIRecord *Record) {
+  APIRecord *Prev = nullptr;
+  for (APIRecord *Curr = First; Curr != Record; Curr = Curr->NextInContext)
+    Prev = Curr;
+
+  if (Prev)
+    Prev->NextInContext = Record->NextInContext;
+  else
+    First = Record->NextInContext;
+
+  if (Last == Record)
+    Last = Prev;
+
+  Record->NextInContext = nullptr;
+}
+
 APIRecord *APISet::findRecordForUSR(StringRef USR) const {
   if (USR.empty())
     return nullptr;
@@ -114,6 +136,33 @@ SymbolReference APISet::createSymbolReference(StringRef Name, StringRef USR,
   return SymbolReference(copyString(Name), copyString(USR), copyString(Source));
 }
 
+void APISet::removeRecord(StringRef USR) {
+  auto Result = USRBasedLookupTable.find(USR);
+  if (Result != USRBasedLookupTable.end()) {
+    auto *Record = Result->getSecond().get();
+    auto &ParentReference = Record->Parent;
+    auto *ParentRecord = const_cast<APIRecord *>(ParentReference.Record);
+    if (!ParentRecord)
+      ParentRecord = findRecordForUSR(ParentReference.USR);
+
+    if (auto *ParentCtx = llvm::cast_if_present<RecordContext>(ParentRecord)) {
+      ParentCtx->removeFromRecordChain(Record);
+      if (auto *RecordAsCtx = llvm::dyn_cast<RecordContext>(Record))
+        ParentCtx->stealRecordChain(*RecordAsCtx);
+    } else {
+      TopLevelRecords.erase(Record);
+      if (auto *RecordAsCtx = llvm::dyn_cast<RecordContext>(Record)) {
+        for (const auto *Child = RecordAsCtx->First; Child != nullptr;
+             Child = Child->getNextInContext())
+          TopLevelRecords.insert(Child);
+      }
+    }
+    USRBasedLookupTable.erase(Result);
+  }
+}
+
+void APISet::removeRecord(APIRecord *Record) { removeRecord(Record->USR); }
+
 APIRecord::~APIRecord() {}
 TagRecord::~TagRecord() {}
 RecordRecord::~RecordRecord() {}
diff --git a/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp b/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp
index 1f8029cbd39ad25..1bce9c59b19791f 100644
--- a/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp
+++ b/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp
@@ -673,14 +673,6 @@ bool SymbolGraphSerializer::shouldSkip(const APIRecord *Record) const {
   if (Record->Availability.isUnconditionallyUnavailable())
     return true;
 
-  // Filter out symbols without a name as we can generate correct symbol graphs
-  // for them. In practice these are anonymous record types that aren't attached
-  // to a declaration.
-  if (auto *Tag = dyn_cast<TagRecord>(Record)) {
-    if (Tag->IsEmbeddedInVarDeclarator)
-      return true;
-  }
-
   // Filter out symbols prefixed with an underscored as they are understood to
   // be symbols clients should not use.
   if (Record->Name.starts_with("_"))
diff --git a/clang/test/ExtractAPI/anonymous_record_no_typedef.c b/clang/test/ExtractAPI/anonymous_record_no_typedef.c
index 789316ca8930b88..064c223ad56e73f 100644
--- a/clang/test/ExtractAPI/anonymous_record_no_typedef.c
+++ b/clang/test/ExtractAPI/anonymous_record_no_typedef.c
@@ -163,4 +163,16 @@ enum {
 // GLOBALOTHERCASE-NEXT:   "GlobalOtherCase"
 // GLOBALOTHERCASE-NEXT: ]
 
+// RUN: FileCheck %s --input-file %t/output.symbols.json --check-prefix VEC
+union Vector {
+  struct {
+    float X;
+    float Y;
+  };
+  float Data[2];
+};
+// VEC-DAG: "!testRelLabel": "memberOf $ c:@U@Vector@FI@Data $ c:@U@Vector"
+// VEC-DAG: "!testRelLabel": "memberOf $ c:@U@Vector@Sa@FI@X $ c:@U@Vector"
+// VEC-DAG: "!testRelLabel": "memberOf $ c:@U@Vector@Sa@FI@Y $ c:@U@Vector"
+
 // expected-no-diagnostics

From a566635915730e6720e1b341f014c7c62af748bf Mon Sep 17 00:00:00 2001
From: Christudasan Devadasan <christudasan.devadasan@amd.com>
Date: Mon, 19 Aug 2024 20:32:55 +0530
Subject: [PATCH 364/441] [AMDGPU] Move AMDGPUCodeGenPassBuilder into
 AMDGPUTargetMachine(NFC) (#103720)

This will allow us to reuse the existing flags and the static
functions while building the pipeline for new pass manager.
---
 .../AMDGPU/AMDGPUCodeGenPassBuilder.cpp       | 97 -------------------
 .../Target/AMDGPU/AMDGPUCodeGenPassBuilder.h  | 33 -------
 .../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 95 +++++++++++++++++-
 llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h  | 27 +++++-
 llvm/lib/Target/AMDGPU/CMakeLists.txt         |  1 -
 llvm/lib/Target/AMDGPU/R600TargetMachine.cpp  |  1 -
 .../Target/AMDGPU/SIMachineFunctionInfo.cpp   |  1 -
 7 files changed, 112 insertions(+), 143 deletions(-)
 delete mode 100644 llvm/lib/Target/AMDGPU/AMDGPUCodeGenPassBuilder.cpp
 delete mode 100644 llvm/lib/Target/AMDGPU/AMDGPUCodeGenPassBuilder.h

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPassBuilder.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPassBuilder.cpp
deleted file mode 100644
index 36f44a20d955320..000000000000000
--- a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPassBuilder.cpp
+++ /dev/null
@@ -1,97 +0,0 @@
-//===- lib/Target/AMDGPU/AMDGPUCodeGenPassBuilder.cpp ---------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "AMDGPUCodeGenPassBuilder.h"
-#include "AMDGPU.h"
-#include "AMDGPUISelDAGToDAG.h"
-#include "AMDGPUPerfHintAnalysis.h"
-#include "AMDGPUTargetMachine.h"
-#include "AMDGPUUnifyDivergentExitNodes.h"
-#include "SIFixSGPRCopies.h"
-#include "llvm/Analysis/UniformityAnalysis.h"
-#include "llvm/Transforms/Scalar/FlattenCFG.h"
-#include "llvm/Transforms/Scalar/Sink.h"
-#include "llvm/Transforms/Scalar/StructurizeCFG.h"
-#include "llvm/Transforms/Utils/FixIrreducible.h"
-#include "llvm/Transforms/Utils/LCSSA.h"
-#include "llvm/Transforms/Utils/UnifyLoopExits.h"
-
-using namespace llvm;
-
-AMDGPUCodeGenPassBuilder::AMDGPUCodeGenPassBuilder(
-    GCNTargetMachine &TM, const CGPassBuilderOption &Opts,
-    PassInstrumentationCallbacks *PIC)
-    : CodeGenPassBuilder(TM, Opts, PIC) {
-  Opt.RequiresCodeGenSCCOrder = true;
-  // Exceptions and StackMaps are not supported, so these passes will never do
-  // anything.
-  // Garbage collection is not supported.
-  disablePass<StackMapLivenessPass, FuncletLayoutPass,
-              ShadowStackGCLoweringPass>();
-}
-
-void AMDGPUCodeGenPassBuilder::addPreISel(AddIRPass &addPass) const {
-  const bool LateCFGStructurize = AMDGPUTargetMachine::EnableLateStructurizeCFG;
-  const bool DisableStructurizer = AMDGPUTargetMachine::DisableStructurizer;
-  const bool EnableStructurizerWorkarounds =
-      AMDGPUTargetMachine::EnableStructurizerWorkarounds;
-
-  if (TM.getOptLevel() > CodeGenOptLevel::None)
-    addPass(FlattenCFGPass());
-
-  if (TM.getOptLevel() > CodeGenOptLevel::None)
-    addPass(SinkingPass());
-
-  addPass(AMDGPULateCodeGenPreparePass(TM));
-
-  // Merge divergent exit nodes. StructurizeCFG won't recognize the multi-exit
-  // regions formed by them.
-
-  addPass(AMDGPUUnifyDivergentExitNodesPass());
-
-  if (!LateCFGStructurize && !DisableStructurizer) {
-    if (EnableStructurizerWorkarounds) {
-      addPass(FixIrreduciblePass());
-      addPass(UnifyLoopExitsPass());
-    }
-
-    addPass(StructurizeCFGPass(/*SkipUniformRegions=*/false));
-  }
-
-  addPass(AMDGPUAnnotateUniformValuesPass());
-
-  if (!LateCFGStructurize && !DisableStructurizer) {
-    addPass(SIAnnotateControlFlowPass(TM));
-
-    // TODO: Move this right after structurizeCFG to avoid extra divergence
-    // analysis. This depends on stopping SIAnnotateControlFlow from making
-    // control flow modifications.
-    addPass(AMDGPURewriteUndefForPHIPass());
-  }
-
-  addPass(LCSSAPass());
-
-  if (TM.getOptLevel() > CodeGenOptLevel::Less)
-    addPass(AMDGPUPerfHintAnalysisPass(TM));
-
-  // FIXME: Why isn't this queried as required from AMDGPUISelDAGToDAG, and why
-  // isn't this in addInstSelector?
-  addPass(RequireAnalysisPass<UniformityInfoAnalysis, Function>());
-}
-
-void AMDGPUCodeGenPassBuilder::addAsmPrinter(AddMachinePass &addPass,
-                                             CreateMCStreamer) const {
-  // TODO: Add AsmPrinter.
-}
-
-Error AMDGPUCodeGenPassBuilder::addInstSelector(AddMachinePass &addPass) const {
-  addPass(AMDGPUISelDAGToDAGPass(TM));
-  addPass(SIFixSGPRCopiesPass());
-  addPass(SILowerI1CopiesPass());
-  return Error::success();
-}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPassBuilder.h b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPassBuilder.h
deleted file mode 100644
index e656e166b3eb2e2..000000000000000
--- a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPassBuilder.h
+++ /dev/null
@@ -1,33 +0,0 @@
-//===- lib/Target/AMDGPU/AMDGPUCodeGenPassBuilder.h -----------*- C++ -*---===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUCODEGENPASSBUILDER_H
-#define LLVM_LIB_TARGET_AMDGPU_AMDGPUCODEGENPASSBUILDER_H
-
-#include "llvm/MC/MCStreamer.h"
-#include "llvm/Passes/CodeGenPassBuilder.h"
-
-namespace llvm {
-
-class GCNTargetMachine;
-
-class AMDGPUCodeGenPassBuilder
-    : public CodeGenPassBuilder<AMDGPUCodeGenPassBuilder, GCNTargetMachine> {
-public:
-  AMDGPUCodeGenPassBuilder(GCNTargetMachine &TM,
-                           const CGPassBuilderOption &Opts,
-                           PassInstrumentationCallbacks *PIC);
-
-  void addPreISel(AddIRPass &addPass) const;
-  void addAsmPrinter(AddMachinePass &, CreateMCStreamer) const;
-  Error addInstSelector(AddMachinePass &) const;
-};
-
-} // namespace llvm
-
-#endif // LLVM_LIB_TARGET_AMDGPU_AMDGPUCODEGENPASSBUILDER_H
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index b8aa93285ad849e..60d6b04f819841b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -7,15 +7,16 @@
 //===----------------------------------------------------------------------===//
 //
 /// \file
-/// The AMDGPU target machine contains all of the hardware specific
-/// information  needed to emit code for SI+ GPUs.
+/// This file contains both AMDGPU target machine and the CodeGen pass builder.
+/// The AMDGPU target machine contains all of the hardware specific information
+/// needed to emit code for SI+ GPUs in the legacy pass manager pipeline. The
+/// CodeGen pass builder handles the pass pipeline for new pass manager.
 //
 //===----------------------------------------------------------------------===//
 
 #include "AMDGPUTargetMachine.h"
 #include "AMDGPU.h"
 #include "AMDGPUAliasAnalysis.h"
-#include "AMDGPUCodeGenPassBuilder.h"
 #include "AMDGPUCtorDtorLowering.h"
 #include "AMDGPUExportClustering.h"
 #include "AMDGPUIGroupLP.h"
@@ -40,6 +41,7 @@
 #include "Utils/AMDGPUBaseInfo.h"
 #include "llvm/Analysis/CGSCCPassManager.h"
 #include "llvm/Analysis/CallGraphSCCPass.h"
+#include "llvm/Analysis/UniformityAnalysis.h"
 #include "llvm/CodeGen/GlobalISel/CSEInfo.h"
 #include "llvm/CodeGen/GlobalISel/IRTranslator.h"
 #include "llvm/CodeGen/GlobalISel/InstructionSelect.h"
@@ -64,10 +66,16 @@
 #include "llvm/Transforms/IPO/GlobalDCE.h"
 #include "llvm/Transforms/IPO/Internalize.h"
 #include "llvm/Transforms/Scalar.h"
+#include "llvm/Transforms/Scalar/FlattenCFG.h"
 #include "llvm/Transforms/Scalar/GVN.h"
 #include "llvm/Transforms/Scalar/InferAddressSpaces.h"
+#include "llvm/Transforms/Scalar/Sink.h"
+#include "llvm/Transforms/Scalar/StructurizeCFG.h"
 #include "llvm/Transforms/Utils.h"
+#include "llvm/Transforms/Utils/FixIrreducible.h"
+#include "llvm/Transforms/Utils/LCSSA.h"
 #include "llvm/Transforms/Utils/SimplifyLibCalls.h"
+#include "llvm/Transforms/Utils/UnifyLoopExits.h"
 #include "llvm/Transforms/Vectorize/LoadStoreVectorizer.h"
 #include <optional>
 
@@ -930,7 +938,7 @@ Error GCNTargetMachine::buildCodeGenPipeline(
 }
 
 //===----------------------------------------------------------------------===//
-// AMDGPU Pass Setup
+// AMDGPU Legacy Pass Setup
 //===----------------------------------------------------------------------===//
 
 std::unique_ptr<CSEConfigBase> llvm::AMDGPUPassConfig::getCSEConfig() const {
@@ -1214,7 +1222,7 @@ MachineFunctionInfo *R600TargetMachine::createMachineFunctionInfo(
 }
 
 //===----------------------------------------------------------------------===//
-// GCN Pass Setup
+// GCN Legacy Pass Setup
 //===----------------------------------------------------------------------===//
 
 ScheduleDAGInstrs *GCNPassConfig::createMachineScheduler(
@@ -1751,3 +1759,80 @@ bool GCNTargetMachine::parseMachineFunctionInfo(
 
   return false;
 }
+
+//===----------------------------------------------------------------------===//
+// AMDGPU CodeGen Pass Builder interface.
+//===----------------------------------------------------------------------===//
+
+AMDGPUCodeGenPassBuilder::AMDGPUCodeGenPassBuilder(
+    GCNTargetMachine &TM, const CGPassBuilderOption &Opts,
+    PassInstrumentationCallbacks *PIC)
+    : CodeGenPassBuilder(TM, Opts, PIC) {
+  Opt.RequiresCodeGenSCCOrder = true;
+  // Exceptions and StackMaps are not supported, so these passes will never do
+  // anything.
+  // Garbage collection is not supported.
+  disablePass<StackMapLivenessPass, FuncletLayoutPass,
+              ShadowStackGCLoweringPass>();
+}
+
+void AMDGPUCodeGenPassBuilder::addPreISel(AddIRPass &addPass) const {
+  const bool LateCFGStructurize = AMDGPUTargetMachine::EnableLateStructurizeCFG;
+  const bool DisableStructurizer = AMDGPUTargetMachine::DisableStructurizer;
+  const bool EnableStructurizerWorkarounds =
+      AMDGPUTargetMachine::EnableStructurizerWorkarounds;
+
+  if (TM.getOptLevel() > CodeGenOptLevel::None)
+    addPass(FlattenCFGPass());
+
+  if (TM.getOptLevel() > CodeGenOptLevel::None)
+    addPass(SinkingPass());
+
+  addPass(AMDGPULateCodeGenPreparePass(TM));
+
+  // Merge divergent exit nodes. StructurizeCFG won't recognize the multi-exit
+  // regions formed by them.
+
+  addPass(AMDGPUUnifyDivergentExitNodesPass());
+
+  if (!LateCFGStructurize && !DisableStructurizer) {
+    if (EnableStructurizerWorkarounds) {
+      addPass(FixIrreduciblePass());
+      addPass(UnifyLoopExitsPass());
+    }
+
+    addPass(StructurizeCFGPass(/*SkipUniformRegions=*/false));
+  }
+
+  addPass(AMDGPUAnnotateUniformValuesPass());
+
+  if (!LateCFGStructurize && !DisableStructurizer) {
+    addPass(SIAnnotateControlFlowPass(TM));
+
+    // TODO: Move this right after structurizeCFG to avoid extra divergence
+    // analysis. This depends on stopping SIAnnotateControlFlow from making
+    // control flow modifications.
+    addPass(AMDGPURewriteUndefForPHIPass());
+  }
+
+  addPass(LCSSAPass());
+
+  if (TM.getOptLevel() > CodeGenOptLevel::Less)
+    addPass(AMDGPUPerfHintAnalysisPass(TM));
+
+  // FIXME: Why isn't this queried as required from AMDGPUISelDAGToDAG, and why
+  // isn't this in addInstSelector?
+  addPass(RequireAnalysisPass<UniformityInfoAnalysis, Function>());
+}
+
+void AMDGPUCodeGenPassBuilder::addAsmPrinter(AddMachinePass &addPass,
+                                             CreateMCStreamer) const {
+  // TODO: Add AsmPrinter.
+}
+
+Error AMDGPUCodeGenPassBuilder::addInstSelector(AddMachinePass &addPass) const {
+  addPass(AMDGPUISelDAGToDAGPass(TM));
+  addPass(SIFixSGPRCopiesPass());
+  addPass(SILowerI1CopiesPass());
+  return Error::success();
+}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h
index 4d39ad2b4150521..4dffddd2d44ccf2 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h
@@ -16,6 +16,8 @@
 
 #include "GCNSubtarget.h"
 #include "llvm/CodeGen/TargetPassConfig.h"
+#include "llvm/MC/MCStreamer.h"
+#include "llvm/Passes/CodeGenPassBuilder.h"
 #include "llvm/Target/TargetMachine.h"
 #include <optional>
 #include <utility>
@@ -47,7 +49,8 @@ class AMDGPUTargetMachine : public LLVMTargetMachine {
   ~AMDGPUTargetMachine() override;
 
   const TargetSubtargetInfo *getSubtargetImpl() const;
-  const TargetSubtargetInfo *getSubtargetImpl(const Function &) const override = 0;
+  const TargetSubtargetInfo *
+  getSubtargetImpl(const Function &) const override = 0;
 
   TargetLoweringObjectFile *getObjFileLowering() const override {
     return TLOF.get();
@@ -94,9 +97,7 @@ class GCNTargetMachine final : public AMDGPUTargetMachine {
 
   TargetTransformInfo getTargetTransformInfo(const Function &F) const override;
 
-  bool useIPRA() const override {
-    return true;
-  }
+  bool useIPRA() const override { return true; }
 
   Error buildCodeGenPipeline(ModulePassManager &MPM, raw_pwrite_stream &Out,
                              raw_pwrite_stream *DwoOut,
@@ -120,7 +121,7 @@ class GCNTargetMachine final : public AMDGPUTargetMachine {
 };
 
 //===----------------------------------------------------------------------===//
-// AMDGPU Pass Setup
+// AMDGPU Pass Setup - For Legacy Pass Manager.
 //===----------------------------------------------------------------------===//
 
 class AMDGPUPassConfig : public TargetPassConfig {
@@ -158,6 +159,22 @@ class AMDGPUPassConfig : public TargetPassConfig {
   }
 };
 
+//===----------------------------------------------------------------------===//
+// AMDGPU CodeGen Pass Builder interface.
+//===----------------------------------------------------------------------===//
+
+class AMDGPUCodeGenPassBuilder
+    : public CodeGenPassBuilder<AMDGPUCodeGenPassBuilder, GCNTargetMachine> {
+public:
+  AMDGPUCodeGenPassBuilder(GCNTargetMachine &TM,
+                           const CGPassBuilderOption &Opts,
+                           PassInstrumentationCallbacks *PIC);
+
+  void addPreISel(AddIRPass &addPass) const;
+  void addAsmPrinter(AddMachinePass &, CreateMCStreamer) const;
+  Error addInstSelector(AddMachinePass &) const;
+};
+
 } // end namespace llvm
 
 #endif // LLVM_LIB_TARGET_AMDGPU_AMDGPUTARGETMACHINE_H
diff --git a/llvm/lib/Target/AMDGPU/CMakeLists.txt b/llvm/lib/Target/AMDGPU/CMakeLists.txt
index 671caf8484cd971..f493076f5bb8a3f 100644
--- a/llvm/lib/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/CMakeLists.txt
@@ -51,7 +51,6 @@ add_llvm_target(AMDGPUCodeGen
   AMDGPUAtomicOptimizer.cpp
   AMDGPUAttributor.cpp
   AMDGPUCallLowering.cpp
-  AMDGPUCodeGenPassBuilder.cpp
   AMDGPUCodeGenPrepare.cpp
   AMDGPUCombinerHelper.cpp
   AMDGPUCtorDtorLowering.cpp
diff --git a/llvm/lib/Target/AMDGPU/R600TargetMachine.cpp b/llvm/lib/Target/AMDGPU/R600TargetMachine.cpp
index c550cfaf06c1002..fe1927e5bdf74a9 100644
--- a/llvm/lib/Target/AMDGPU/R600TargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/R600TargetMachine.cpp
@@ -13,7 +13,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "R600TargetMachine.h"
-#include "AMDGPUTargetMachine.h"
 #include "R600.h"
 #include "R600CodeGenPassBuilder.h"
 #include "R600MachineScheduler.h"
diff --git a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp
index c5251826b117cb8..4818f1b13caa6ae 100644
--- a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp
@@ -8,7 +8,6 @@
 
 #include "SIMachineFunctionInfo.h"
 #include "AMDGPUSubtarget.h"
-#include "AMDGPUTargetMachine.h"
 #include "GCNSubtarget.h"
 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
 #include "SIRegisterInfo.h"

From 7dd6340bdadf86bd0facdea89d1876a5c36dc33b Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <ramkumar.ramachandra@codasip.com>
Date: Mon, 19 Aug 2024 16:03:42 +0100
Subject: [PATCH 365/441] MathExtras: template'ize alignToPowerOf2 (#97814)

Follow up on 5627794 (MathExtras: avoid unnecessarily widening types) to
change the overflow behavior of alignToPowerOf2 to only overflow if the
result is not representable in the return type. This allows us to
template'ize it, and avoid unnecessarily widening the types of
arguments.
---
 llvm/include/llvm/Support/MathExtras.h    | 16 ++++++++++++----
 llvm/unittests/Support/MathExtrasTest.cpp |  7 +++++--
 2 files changed, 17 insertions(+), 6 deletions(-)

diff --git a/llvm/include/llvm/Support/MathExtras.h b/llvm/include/llvm/Support/MathExtras.h
index e568e42afcf4d20..a52a9f07bacd4ba 100644
--- a/llvm/include/llvm/Support/MathExtras.h
+++ b/llvm/include/llvm/Support/MathExtras.h
@@ -497,13 +497,21 @@ constexpr uint64_t alignTo(uint64_t Value, uint64_t Align) {
   return CeilDiv * Align;
 }
 
+/// Will overflow only if result is not representable in T.
+template <typename U, typename V, typename T = common_uint<U, V>>
+constexpr T alignToPowerOf2(U Value, V Align) {
+  assert(Align != 0 && (Align & (Align - 1)) == 0 &&
+         "Align must be a power of 2");
+  T NegAlign = static_cast<T>(0) - Align;
+  return (Value + (Align - 1)) & NegAlign;
+}
+
+/// Fallback when arguments aren't integral.
 constexpr uint64_t alignToPowerOf2(uint64_t Value, uint64_t Align) {
   assert(Align != 0 && (Align & (Align - 1)) == 0 &&
          "Align must be a power of 2");
-  // Replace unary minus to avoid compilation error on Windows:
-  // "unary minus operator applied to unsigned type, result still unsigned"
-  uint64_t NegAlign = (~Align) + 1;
-  return (Value + Align - 1) & NegAlign;
+  uint64_t NegAlign = 0 - Align;
+  return (Value + (Align - 1)) & NegAlign;
 }
 
 /// If non-zero \p Skew is specified, the return value will be a minimal integer
diff --git a/llvm/unittests/Support/MathExtrasTest.cpp b/llvm/unittests/Support/MathExtrasTest.cpp
index a1cc609033d972a..984185fece4aa3c 100644
--- a/llvm/unittests/Support/MathExtrasTest.cpp
+++ b/llvm/unittests/Support/MathExtrasTest.cpp
@@ -218,8 +218,11 @@ TEST(MathExtras, AlignToPowerOf2) {
   EXPECT_EQ(24u, alignToPowerOf2(17, 8));
   EXPECT_EQ(0u, alignToPowerOf2(~0LL, 8));
   EXPECT_EQ(240u, alignToPowerOf2(240, 16));
-  EXPECT_EQ(static_cast<uint64_t>(std::numeric_limits<uint32_t>::max()) + 1,
-            alignToPowerOf2(std::numeric_limits<uint32_t>::max(), 2));
+
+  // Overflow.
+  EXPECT_EQ(0u, alignToPowerOf2(static_cast<uint8_t>(200),
+                                static_cast<uint8_t>(128)));
+  EXPECT_EQ(0u, alignToPowerOf2(std::numeric_limits<uint32_t>::max(), 2));
 }
 
 TEST(MathExtras, AlignDown) {

From b18b4547f1bfaf6da37b29440a96176e807c2e6c Mon Sep 17 00:00:00 2001
From: Daniel Grumberg <dgrumberg@apple.com>
Date: Mon, 19 Aug 2024 16:06:25 +0100
Subject: [PATCH 366/441] Revert "[clang][ExtractAPI] Stop dropping fields of
 nested anonymous record types when they aren't attached to variable
 declaration (#104600)"

This reverts commit c60da1a271a6bb271e7703b2f7c71fbece67ab78.
---
 clang/include/clang/ExtractAPI/API.h          | 25 ++++-----
 .../clang/ExtractAPI/ExtractAPIVisitor.h      | 39 ++------------
 clang/lib/ExtractAPI/API.cpp                  | 53 +------------------
 .../Serialization/SymbolGraphSerializer.cpp   |  8 +++
 .../ExtractAPI/anonymous_record_no_typedef.c  | 12 -----
 5 files changed, 27 insertions(+), 110 deletions(-)

diff --git a/clang/include/clang/ExtractAPI/API.h b/clang/include/clang/ExtractAPI/API.h
index 188e35b72117b50..bf291074fd06143 100644
--- a/clang/include/clang/ExtractAPI/API.h
+++ b/clang/include/clang/ExtractAPI/API.h
@@ -19,13 +19,21 @@
 #define LLVM_CLANG_EXTRACTAPI_API_H
 
 #include "clang/AST/Availability.h"
+#include "clang/AST/Decl.h"
 #include "clang/AST/DeclBase.h"
+#include "clang/AST/DeclObjC.h"
 #include "clang/AST/RawCommentList.h"
 #include "clang/Basic/SourceLocation.h"
+#include "clang/Basic/Specifiers.h"
 #include "clang/ExtractAPI/DeclarationFragments.h"
-#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Allocator.h"
 #include "llvm/Support/Casting.h"
+#include "llvm/Support/Compiler.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/raw_ostream.h"
 #include "llvm/TargetParser/Triple.h"
 #include <cstddef>
 #include <iterator>
@@ -320,8 +328,6 @@ class RecordContext {
   /// chain.
   void stealRecordChain(RecordContext &Other);
 
-  void removeFromRecordChain(APIRecord *Record);
-
   APIRecord::RecordKind getKind() const { return Kind; }
 
   struct record_iterator {
@@ -1420,15 +1426,10 @@ class APISet {
   typename std::enable_if_t<std::is_base_of_v<APIRecord, RecordTy>, RecordTy> *
   createRecord(StringRef USR, StringRef Name, CtorArgsContTy &&...CtorArgs);
 
-  auto getTopLevelRecords() const {
-    return llvm::iterator_range<decltype(TopLevelRecords)::iterator>(
-        TopLevelRecords);
+  ArrayRef<const APIRecord *> getTopLevelRecords() const {
+    return TopLevelRecords;
   }
 
-  void removeRecord(StringRef USR);
-
-  void removeRecord(APIRecord *Record);
-
   APISet(const llvm::Triple &Target, Language Lang,
          const std::string &ProductName)
       : Target(Target), Lang(Lang), ProductName(ProductName) {}
@@ -1455,7 +1456,7 @@ class APISet {
   // lives in the BumpPtrAllocator.
   using APIRecordStoredPtr = std::unique_ptr<APIRecord, APIRecordDeleter>;
   llvm::DenseMap<StringRef, APIRecordStoredPtr> USRBasedLookupTable;
-  llvm::SmallPtrSet<const APIRecord *, 32> TopLevelRecords;
+  std::vector<const APIRecord *> TopLevelRecords;
 
 public:
   const std::string ProductName;
@@ -1481,7 +1482,7 @@ APISet::createRecord(StringRef USR, StringRef Name,
             dyn_cast_if_present<RecordContext>(Record->Parent.Record))
       ParentContext->addToRecordChain(Record);
     else
-      TopLevelRecords.insert(Record);
+      TopLevelRecords.push_back(Record);
   } else {
     Record = dyn_cast<RecordTy>(Result.first->second.get());
   }
diff --git a/clang/include/clang/ExtractAPI/ExtractAPIVisitor.h b/clang/include/clang/ExtractAPI/ExtractAPIVisitor.h
index 8d9ac0620345117..1b27027621666ae 100644
--- a/clang/include/clang/ExtractAPI/ExtractAPIVisitor.h
+++ b/clang/include/clang/ExtractAPI/ExtractAPIVisitor.h
@@ -23,11 +23,13 @@
 #include "clang/AST/RecursiveASTVisitor.h"
 #include "clang/Basic/LLVM.h"
 #include "clang/Basic/Module.h"
+#include "clang/Basic/SourceManager.h"
 #include "clang/Basic/Specifiers.h"
 #include "clang/ExtractAPI/API.h"
 #include "clang/ExtractAPI/DeclarationFragments.h"
 #include "clang/ExtractAPI/TypedefUnderlyingTypeResolver.h"
 #include "clang/Index/USRGeneration.h"
+#include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Casting.h"
 #include <type_traits>
@@ -38,8 +40,6 @@ namespace impl {
 
 template <typename Derived>
 class ExtractAPIVisitorBase : public RecursiveASTVisitor<Derived> {
-  using Base = RecursiveASTVisitor<Derived>;
-
 protected:
   ExtractAPIVisitorBase(ASTContext &Context, APISet &API)
       : Context(Context), API(API) {}
@@ -81,10 +81,8 @@ class ExtractAPIVisitorBase : public RecursiveASTVisitor<Derived> {
 
   bool VisitNamespaceDecl(const NamespaceDecl *Decl);
 
-  bool TraverseRecordDecl(RecordDecl *Decl);
   bool VisitRecordDecl(const RecordDecl *Decl);
 
-  bool TraverseCXXRecordDecl(CXXRecordDecl *Decl);
   bool VisitCXXRecordDecl(const CXXRecordDecl *Decl);
 
   bool VisitCXXMethodDecl(const CXXMethodDecl *Decl);
@@ -242,7 +240,7 @@ class ExtractAPIVisitorBase : public RecursiveASTVisitor<Derived> {
 
   bool isEmbeddedInVarDeclarator(const TagDecl &D) {
     return D.getName().empty() && getTypedefName(&D).empty() &&
-           D.isEmbeddedInDeclarator() && !D.isFreeStanding();
+           D.isEmbeddedInDeclarator();
   }
 
   void maybeMergeWithAnonymousTag(const DeclaratorDecl &D,
@@ -254,10 +252,8 @@ class ExtractAPIVisitorBase : public RecursiveASTVisitor<Derived> {
     clang::index::generateUSRForDecl(Tag, TagUSR);
     if (auto *Record = llvm::dyn_cast_if_present<TagRecord>(
             API.findRecordForUSR(TagUSR))) {
-      if (Record->IsEmbeddedInVarDeclarator) {
+      if (Record->IsEmbeddedInVarDeclarator)
         NewRecordContext->stealRecordChain(*Record);
-        API.removeRecord(Record);
-      }
     }
   }
 };
@@ -552,19 +548,6 @@ bool ExtractAPIVisitorBase<Derived>::VisitNamespaceDecl(
   return true;
 }
 
-template <typename Derived>
-bool ExtractAPIVisitorBase<Derived>::TraverseRecordDecl(RecordDecl *Decl) {
-  bool Ret = Base::TraverseRecordDecl(Decl);
-
-  if (!isEmbeddedInVarDeclarator(*Decl) && Decl->isAnonymousStructOrUnion()) {
-    SmallString<128> USR;
-    index::generateUSRForDecl(Decl, USR);
-    API.removeRecord(USR);
-  }
-
-  return Ret;
-}
-
 template <typename Derived>
 bool ExtractAPIVisitorBase<Derived>::VisitRecordDecl(const RecordDecl *Decl) {
   if (!getDerivedExtractAPIVisitor().shouldDeclBeIncluded(Decl))
@@ -605,20 +588,6 @@ bool ExtractAPIVisitorBase<Derived>::VisitRecordDecl(const RecordDecl *Decl) {
   return true;
 }
 
-template <typename Derived>
-bool ExtractAPIVisitorBase<Derived>::TraverseCXXRecordDecl(
-    CXXRecordDecl *Decl) {
-  bool Ret = Base::TraverseCXXRecordDecl(Decl);
-
-  if (!isEmbeddedInVarDeclarator(*Decl) && Decl->isAnonymousStructOrUnion()) {
-    SmallString<128> USR;
-    index::generateUSRForDecl(Decl, USR);
-    API.removeRecord(USR);
-  }
-
-  return Ret;
-}
-
 template <typename Derived>
 bool ExtractAPIVisitorBase<Derived>::VisitCXXRecordDecl(
     const CXXRecordDecl *Decl) {
diff --git a/clang/lib/ExtractAPI/API.cpp b/clang/lib/ExtractAPI/API.cpp
index 9dbc023885c37f1..ab1108f663deacc 100644
--- a/clang/lib/ExtractAPI/API.cpp
+++ b/clang/lib/ExtractAPI/API.cpp
@@ -13,6 +13,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/ExtractAPI/API.h"
+#include "clang/AST/RawCommentList.h"
+#include "clang/Index/USRGeneration.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/ErrorHandling.h"
 #include <memory>
@@ -58,10 +60,6 @@ bool RecordContext::IsWellFormed() const {
 
 void RecordContext::stealRecordChain(RecordContext &Other) {
   assert(IsWellFormed());
-  // Other's record chain is empty, nothing to do
-  if (Other.First == nullptr && Other.Last == nullptr)
-    return;
-
   // If we don't have an empty chain append Other's chain into ours.
   if (First)
     Last->NextInContext = Other.First;
@@ -70,10 +68,6 @@ void RecordContext::stealRecordChain(RecordContext &Other) {
 
   Last = Other.Last;
 
-  for (auto *StolenRecord = Other.First; StolenRecord != nullptr;
-       StolenRecord = StolenRecord->getNextInContext())
-    StolenRecord->Parent = SymbolReference(cast<APIRecord>(this));
-
   // Delete Other's chain to ensure we don't accidentally traverse it.
   Other.First = nullptr;
   Other.Last = nullptr;
@@ -91,22 +85,6 @@ void RecordContext::addToRecordChain(APIRecord *Record) const {
   Last = Record;
 }
 
-void RecordContext::removeFromRecordChain(APIRecord *Record) {
-  APIRecord *Prev = nullptr;
-  for (APIRecord *Curr = First; Curr != Record; Curr = Curr->NextInContext)
-    Prev = Curr;
-
-  if (Prev)
-    Prev->NextInContext = Record->NextInContext;
-  else
-    First = Record->NextInContext;
-
-  if (Last == Record)
-    Last = Prev;
-
-  Record->NextInContext = nullptr;
-}
-
 APIRecord *APISet::findRecordForUSR(StringRef USR) const {
   if (USR.empty())
     return nullptr;
@@ -136,33 +114,6 @@ SymbolReference APISet::createSymbolReference(StringRef Name, StringRef USR,
   return SymbolReference(copyString(Name), copyString(USR), copyString(Source));
 }
 
-void APISet::removeRecord(StringRef USR) {
-  auto Result = USRBasedLookupTable.find(USR);
-  if (Result != USRBasedLookupTable.end()) {
-    auto *Record = Result->getSecond().get();
-    auto &ParentReference = Record->Parent;
-    auto *ParentRecord = const_cast<APIRecord *>(ParentReference.Record);
-    if (!ParentRecord)
-      ParentRecord = findRecordForUSR(ParentReference.USR);
-
-    if (auto *ParentCtx = llvm::cast_if_present<RecordContext>(ParentRecord)) {
-      ParentCtx->removeFromRecordChain(Record);
-      if (auto *RecordAsCtx = llvm::dyn_cast<RecordContext>(Record))
-        ParentCtx->stealRecordChain(*RecordAsCtx);
-    } else {
-      TopLevelRecords.erase(Record);
-      if (auto *RecordAsCtx = llvm::dyn_cast<RecordContext>(Record)) {
-        for (const auto *Child = RecordAsCtx->First; Child != nullptr;
-             Child = Child->getNextInContext())
-          TopLevelRecords.insert(Child);
-      }
-    }
-    USRBasedLookupTable.erase(Result);
-  }
-}
-
-void APISet::removeRecord(APIRecord *Record) { removeRecord(Record->USR); }
-
 APIRecord::~APIRecord() {}
 TagRecord::~TagRecord() {}
 RecordRecord::~RecordRecord() {}
diff --git a/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp b/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp
index 1bce9c59b19791f..1f8029cbd39ad25 100644
--- a/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp
+++ b/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp
@@ -673,6 +673,14 @@ bool SymbolGraphSerializer::shouldSkip(const APIRecord *Record) const {
   if (Record->Availability.isUnconditionallyUnavailable())
     return true;
 
+  // Filter out symbols without a name as we can generate correct symbol graphs
+  // for them. In practice these are anonymous record types that aren't attached
+  // to a declaration.
+  if (auto *Tag = dyn_cast<TagRecord>(Record)) {
+    if (Tag->IsEmbeddedInVarDeclarator)
+      return true;
+  }
+
   // Filter out symbols prefixed with an underscored as they are understood to
   // be symbols clients should not use.
   if (Record->Name.starts_with("_"))
diff --git a/clang/test/ExtractAPI/anonymous_record_no_typedef.c b/clang/test/ExtractAPI/anonymous_record_no_typedef.c
index 064c223ad56e73f..789316ca8930b88 100644
--- a/clang/test/ExtractAPI/anonymous_record_no_typedef.c
+++ b/clang/test/ExtractAPI/anonymous_record_no_typedef.c
@@ -163,16 +163,4 @@ enum {
 // GLOBALOTHERCASE-NEXT:   "GlobalOtherCase"
 // GLOBALOTHERCASE-NEXT: ]
 
-// RUN: FileCheck %s --input-file %t/output.symbols.json --check-prefix VEC
-union Vector {
-  struct {
-    float X;
-    float Y;
-  };
-  float Data[2];
-};
-// VEC-DAG: "!testRelLabel": "memberOf $ c:@U@Vector@FI@Data $ c:@U@Vector"
-// VEC-DAG: "!testRelLabel": "memberOf $ c:@U@Vector@Sa@FI@X $ c:@U@Vector"
-// VEC-DAG: "!testRelLabel": "memberOf $ c:@U@Vector@Sa@FI@Y $ c:@U@Vector"
-
 // expected-no-diagnostics

From a449b857241dd29a1164c483d3ee73612e45cbb4 Mon Sep 17 00:00:00 2001
From: Christudasan Devadasan <christudasan.devadasan@amd.com>
Date: Mon, 19 Aug 2024 20:40:12 +0530
Subject: [PATCH 367/441] [AMDGPU][R600] Move R600CodeGenPassBuilder into
 R600TargetMachine(NFC). (#103721)

---
 .../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp |  8 ----
 llvm/lib/Target/AMDGPU/CMakeLists.txt         |  1 -
 .../Target/AMDGPU/R600CodeGenPassBuilder.cpp  | 33 ---------------
 .../Target/AMDGPU/R600CodeGenPassBuilder.h    | 32 ---------------
 llvm/lib/Target/AMDGPU/R600ISelLowering.cpp   |  1 -
 llvm/lib/Target/AMDGPU/R600TargetMachine.cpp  | 40 +++++++++++++++++--
 llvm/lib/Target/AMDGPU/R600TargetMachine.h    | 15 +++++++
 7 files changed, 52 insertions(+), 78 deletions(-)
 delete mode 100644 llvm/lib/Target/AMDGPU/R600CodeGenPassBuilder.cpp
 delete mode 100644 llvm/lib/Target/AMDGPU/R600CodeGenPassBuilder.h

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 60d6b04f819841b..f15d02ac9301ef1 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -32,7 +32,6 @@
 #include "GCNSchedStrategy.h"
 #include "GCNVOPDUtils.h"
 #include "R600.h"
-#include "R600MachineFunctionInfo.h"
 #include "R600TargetMachine.h"
 #include "SIFixSGPRCopies.h"
 #include "SIMachineFunctionInfo.h"
@@ -1214,13 +1213,6 @@ AMDGPUPassConfig::createMachineScheduler(MachineSchedContext *C) const {
   return DAG;
 }
 
-MachineFunctionInfo *R600TargetMachine::createMachineFunctionInfo(
-    BumpPtrAllocator &Allocator, const Function &F,
-    const TargetSubtargetInfo *STI) const {
-  return R600MachineFunctionInfo::create<R600MachineFunctionInfo>(
-      Allocator, F, static_cast<const R600Subtarget *>(STI));
-}
-
 //===----------------------------------------------------------------------===//
 // GCN Legacy Pass Setup
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/AMDGPU/CMakeLists.txt b/llvm/lib/Target/AMDGPU/CMakeLists.txt
index f493076f5bb8a3f..c91584ae8ce59cd 100644
--- a/llvm/lib/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/CMakeLists.txt
@@ -121,7 +121,6 @@ add_llvm_target(AMDGPUCodeGen
   GCNVOPDUtils.cpp
   R600AsmPrinter.cpp
   R600ClauseMergePass.cpp
-  R600CodeGenPassBuilder.cpp
   R600ControlFlowFinalizer.cpp
   R600EmitClauseMarkers.cpp
   R600ExpandSpecialInstrs.cpp
diff --git a/llvm/lib/Target/AMDGPU/R600CodeGenPassBuilder.cpp b/llvm/lib/Target/AMDGPU/R600CodeGenPassBuilder.cpp
deleted file mode 100644
index a57b3aa0adb158a..000000000000000
--- a/llvm/lib/Target/AMDGPU/R600CodeGenPassBuilder.cpp
+++ /dev/null
@@ -1,33 +0,0 @@
-//===-- R600CodeGenPassBuilder.cpp ------ Build R600 CodeGen pipeline -----===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "R600CodeGenPassBuilder.h"
-#include "R600TargetMachine.h"
-
-using namespace llvm;
-
-R600CodeGenPassBuilder::R600CodeGenPassBuilder(
-    R600TargetMachine &TM, const CGPassBuilderOption &Opts,
-    PassInstrumentationCallbacks *PIC)
-    : CodeGenPassBuilder(TM, Opts, PIC) {
-  Opt.RequiresCodeGenSCCOrder = true;
-}
-
-void R600CodeGenPassBuilder::addPreISel(AddIRPass &addPass) const {
-  // TODO: Add passes pre instruction selection.
-}
-
-void R600CodeGenPassBuilder::addAsmPrinter(AddMachinePass &addPass,
-                                           CreateMCStreamer) const {
-  // TODO: Add AsmPrinter.
-}
-
-Error R600CodeGenPassBuilder::addInstSelector(AddMachinePass &) const {
-  // TODO: Add instruction selector.
-  return Error::success();
-}
diff --git a/llvm/lib/Target/AMDGPU/R600CodeGenPassBuilder.h b/llvm/lib/Target/AMDGPU/R600CodeGenPassBuilder.h
deleted file mode 100644
index be7c935c094d9f0..000000000000000
--- a/llvm/lib/Target/AMDGPU/R600CodeGenPassBuilder.h
+++ /dev/null
@@ -1,32 +0,0 @@
-//===-- R600CodeGenPassBuilder.h -- Build R600 CodeGen pipeline -*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIB_TARGET_AMDGPU_R600CODEGENPASSBUILDER_H
-#define LLVM_LIB_TARGET_AMDGPU_R600CODEGENPASSBUILDER_H
-
-#include "llvm/MC/MCStreamer.h"
-#include "llvm/Passes/CodeGenPassBuilder.h"
-
-namespace llvm {
-
-class R600TargetMachine;
-
-class R600CodeGenPassBuilder
-    : public CodeGenPassBuilder<R600CodeGenPassBuilder, R600TargetMachine> {
-public:
-  R600CodeGenPassBuilder(R600TargetMachine &TM, const CGPassBuilderOption &Opts,
-                         PassInstrumentationCallbacks *PIC);
-
-  void addPreISel(AddIRPass &addPass) const;
-  void addAsmPrinter(AddMachinePass &, CreateMCStreamer) const;
-  Error addInstSelector(AddMachinePass &) const;
-};
-
-} // namespace llvm
-
-#endif // LLVM_LIB_TARGET_AMDGPU_R600CODEGENPASSBUILDER_H
diff --git a/llvm/lib/Target/AMDGPU/R600ISelLowering.cpp b/llvm/lib/Target/AMDGPU/R600ISelLowering.cpp
index c79688bf038be30..7e4d9d21a0b3972 100644
--- a/llvm/lib/Target/AMDGPU/R600ISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/R600ISelLowering.cpp
@@ -15,7 +15,6 @@
 #include "AMDGPU.h"
 #include "MCTargetDesc/R600MCTargetDesc.h"
 #include "R600Defines.h"
-#include "R600InstrInfo.h"
 #include "R600MachineFunctionInfo.h"
 #include "R600Subtarget.h"
 #include "R600TargetMachine.h"
diff --git a/llvm/lib/Target/AMDGPU/R600TargetMachine.cpp b/llvm/lib/Target/AMDGPU/R600TargetMachine.cpp
index fe1927e5bdf74a9..a1a60b8bdfa9ee1 100644
--- a/llvm/lib/Target/AMDGPU/R600TargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/R600TargetMachine.cpp
@@ -7,14 +7,16 @@
 //===----------------------------------------------------------------------===//
 //
 /// \file
-/// The AMDGPU-R600 target machine contains all of the hardware specific
-/// information  needed to emit code for R600 GPUs.
+/// This file contains both AMDGPU-R600 target machine and the CodeGen pass
+/// builder. The target machine contains all of the hardware specific
+/// information needed to emit code for R600 GPUs and the CodeGen pass builder
+/// handles the pass pipeline for new pass manager.
 //
 //===----------------------------------------------------------------------===//
 
 #include "R600TargetMachine.h"
 #include "R600.h"
-#include "R600CodeGenPassBuilder.h"
+#include "R600MachineFunctionInfo.h"
 #include "R600MachineScheduler.h"
 #include "R600TargetTransformInfo.h"
 #include "llvm/Transforms/Scalar.h"
@@ -152,3 +154,35 @@ Error R600TargetMachine::buildCodeGenPipeline(
   R600CodeGenPassBuilder CGPB(*this, Opts, PIC);
   return CGPB.buildPipeline(MPM, Out, DwoOut, FileType);
 }
+
+MachineFunctionInfo *R600TargetMachine::createMachineFunctionInfo(
+    BumpPtrAllocator &Allocator, const Function &F,
+    const TargetSubtargetInfo *STI) const {
+  return R600MachineFunctionInfo::create<R600MachineFunctionInfo>(
+      Allocator, F, static_cast<const R600Subtarget *>(STI));
+}
+
+//===----------------------------------------------------------------------===//
+// R600 CodeGen Pass Builder interface.
+//===----------------------------------------------------------------------===//
+
+R600CodeGenPassBuilder::R600CodeGenPassBuilder(
+    R600TargetMachine &TM, const CGPassBuilderOption &Opts,
+    PassInstrumentationCallbacks *PIC)
+    : CodeGenPassBuilder(TM, Opts, PIC) {
+  Opt.RequiresCodeGenSCCOrder = true;
+}
+
+void R600CodeGenPassBuilder::addPreISel(AddIRPass &addPass) const {
+  // TODO: Add passes pre instruction selection.
+}
+
+void R600CodeGenPassBuilder::addAsmPrinter(AddMachinePass &addPass,
+                                           CreateMCStreamer) const {
+  // TODO: Add AsmPrinter.
+}
+
+Error R600CodeGenPassBuilder::addInstSelector(AddMachinePass &) const {
+  // TODO: Add instruction selector.
+  return Error::success();
+}
diff --git a/llvm/lib/Target/AMDGPU/R600TargetMachine.h b/llvm/lib/Target/AMDGPU/R600TargetMachine.h
index 29e370edef2c67b..b7f123a07a9c1de 100644
--- a/llvm/lib/Target/AMDGPU/R600TargetMachine.h
+++ b/llvm/lib/Target/AMDGPU/R600TargetMachine.h
@@ -55,6 +55,21 @@ class R600TargetMachine final : public AMDGPUTargetMachine {
                             const TargetSubtargetInfo *STI) const override;
 };
 
+//===----------------------------------------------------------------------===//
+// R600 CodeGen Pass Builder interface.
+//===----------------------------------------------------------------------===//
+
+class R600CodeGenPassBuilder
+    : public CodeGenPassBuilder<R600CodeGenPassBuilder, R600TargetMachine> {
+public:
+  R600CodeGenPassBuilder(R600TargetMachine &TM, const CGPassBuilderOption &Opts,
+                         PassInstrumentationCallbacks *PIC);
+
+  void addPreISel(AddIRPass &addPass) const;
+  void addAsmPrinter(AddMachinePass &, CreateMCStreamer) const;
+  Error addInstSelector(AddMachinePass &) const;
+};
+
 } // end namespace llvm
 
 #endif // LLVM_LIB_TARGET_AMDGPU_R600TARGETMACHINE_H

From b64e7e07e544d62ab132fc9f5c4153d4807a5fa3 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Mon, 19 Aug 2024 17:12:36 +0200
Subject: [PATCH 368/441] [SimplifyCFG] Add tests for hoisting of commutative
 instructions (NFC)

---
 .../SimplifyCFG/hoist-common-code.ll          | 347 ++++++++++++++++++
 1 file changed, 347 insertions(+)

diff --git a/llvm/test/Transforms/SimplifyCFG/hoist-common-code.ll b/llvm/test/Transforms/SimplifyCFG/hoist-common-code.ll
index 991b89a486e0f5a..e2705dc95ab9792 100644
--- a/llvm/test/Transforms/SimplifyCFG/hoist-common-code.ll
+++ b/llvm/test/Transforms/SimplifyCFG/hoist-common-code.ll
@@ -154,3 +154,350 @@ exit:                                             ; preds = %bb2, %bb1, %bb0
   %result = phi i1 [ %0, %bb0 ], [ %1, %bb1 ], [ %2, %bb2 ]
   ret i1 %result
 }
+
+declare void @foo()
+
+define i1 @test_icmp_simple(i1 %c, i32 %a, i32 %b) {
+; CHECK-LABEL: @test_icmp_simple(
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF:%.*]], label [[ELSE:%.*]]
+; CHECK:       common.ret:
+; CHECK-NEXT:    [[COMMON_RET_OP:%.*]] = phi i1 [ [[CMP1:%.*]], [[IF]] ], [ [[CMP2:%.*]], [[ELSE]] ]
+; CHECK-NEXT:    ret i1 [[COMMON_RET_OP]]
+; CHECK:       if:
+; CHECK-NEXT:    [[CMP1]] = icmp ult i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[COMMON_RET:%.*]]
+; CHECK:       else:
+; CHECK-NEXT:    [[CMP2]] = icmp ugt i32 [[B]], [[A]]
+; CHECK-NEXT:    call void @bar()
+; CHECK-NEXT:    br label [[COMMON_RET]]
+;
+  br i1 %c, label %if, label %else
+
+if:
+  %cmp1 = icmp ult i32 %a, %b
+  call void @foo()
+  ret i1 %cmp1
+
+else:
+  %cmp2 = icmp ugt i32 %b, %a
+  call void @bar()
+  ret i1 %cmp2
+}
+
+define void @test_icmp_complex(i1 %c, i32 %a, i32 %b) {
+; CHECK-LABEL: @test_icmp_complex(
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF:%.*]], label [[ELSE:%.*]]
+; CHECK:       if:
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    br i1 [[CMP1]], label [[IF2:%.*]], label [[ELSE2:%.*]]
+; CHECK:       else:
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ugt i32 [[B]], [[A]]
+; CHECK-NEXT:    br i1 [[CMP2]], label [[IF2]], label [[ELSE2]]
+; CHECK:       common.ret:
+; CHECK-NEXT:    ret void
+; CHECK:       if2:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[COMMON_RET:%.*]]
+; CHECK:       else2:
+; CHECK-NEXT:    call void @bar()
+; CHECK-NEXT:    br label [[COMMON_RET]]
+;
+  br i1 %c, label %if, label %else
+
+if:
+  %cmp1 = icmp ult i32 %a, %b
+  br i1 %cmp1, label %if2, label %else2
+
+else:
+  %cmp2 = icmp ugt i32 %b, %a
+  br i1 %cmp2, label %if2, label %else2
+
+if2:
+  call void @foo()
+  ret void
+
+else2:
+  call void @bar()
+  ret void
+}
+
+define i1 @test_icmp_wrong_operands(i1 %c, i32 %a, i32 %b) {
+; CHECK-LABEL: @test_icmp_wrong_operands(
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF:%.*]], label [[ELSE:%.*]]
+; CHECK:       common.ret:
+; CHECK-NEXT:    [[COMMON_RET_OP:%.*]] = phi i1 [ [[CMP1:%.*]], [[IF]] ], [ [[CMP2:%.*]], [[ELSE]] ]
+; CHECK-NEXT:    ret i1 [[COMMON_RET_OP]]
+; CHECK:       if:
+; CHECK-NEXT:    [[CMP1]] = icmp ult i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[COMMON_RET:%.*]]
+; CHECK:       else:
+; CHECK-NEXT:    [[CMP2]] = icmp ugt i32 [[A]], [[B]]
+; CHECK-NEXT:    call void @bar()
+; CHECK-NEXT:    br label [[COMMON_RET]]
+;
+  br i1 %c, label %if, label %else
+
+if:
+  %cmp1 = icmp ult i32 %a, %b
+  call void @foo()
+  ret i1 %cmp1
+
+else:
+  %cmp2 = icmp ugt i32 %a, %b
+  call void @bar()
+  ret i1 %cmp2
+}
+
+define i1 @test_icmp_wrong_pred(i1 %c, i32 %a, i32 %b) {
+; CHECK-LABEL: @test_icmp_wrong_pred(
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF:%.*]], label [[ELSE:%.*]]
+; CHECK:       common.ret:
+; CHECK-NEXT:    [[COMMON_RET_OP:%.*]] = phi i1 [ [[CMP1:%.*]], [[IF]] ], [ [[CMP2:%.*]], [[ELSE]] ]
+; CHECK-NEXT:    ret i1 [[COMMON_RET_OP]]
+; CHECK:       if:
+; CHECK-NEXT:    [[CMP1]] = icmp ult i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[COMMON_RET:%.*]]
+; CHECK:       else:
+; CHECK-NEXT:    [[CMP2]] = icmp uge i32 [[B]], [[A]]
+; CHECK-NEXT:    call void @bar()
+; CHECK-NEXT:    br label [[COMMON_RET]]
+;
+  br i1 %c, label %if, label %else
+
+if:
+  %cmp1 = icmp ult i32 %a, %b
+  call void @foo()
+  ret i1 %cmp1
+
+else:
+  %cmp2 = icmp uge i32 %b, %a
+  call void @bar()
+  ret i1 %cmp2
+}
+
+define i32 @test_binop(i1 %c, i32 %a, i32 %b) {
+; CHECK-LABEL: @test_binop(
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF:%.*]], label [[ELSE:%.*]]
+; CHECK:       common.ret:
+; CHECK-NEXT:    [[COMMON_RET_OP:%.*]] = phi i32 [ [[OP1:%.*]], [[IF]] ], [ [[OP2:%.*]], [[ELSE]] ]
+; CHECK-NEXT:    ret i32 [[COMMON_RET_OP]]
+; CHECK:       if:
+; CHECK-NEXT:    [[OP1]] = add i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[COMMON_RET:%.*]]
+; CHECK:       else:
+; CHECK-NEXT:    [[OP2]] = add i32 [[B]], [[A]]
+; CHECK-NEXT:    call void @bar()
+; CHECK-NEXT:    br label [[COMMON_RET]]
+;
+  br i1 %c, label %if, label %else
+
+if:
+  %op1 = add i32 %a, %b
+  call void @foo()
+  ret i32 %op1
+
+else:
+  %op2 = add i32 %b, %a
+  call void @bar()
+  ret i32 %op2
+}
+
+define i32 @test_binop_flags(i1 %c, i32 %a, i32 %b) {
+; CHECK-LABEL: @test_binop_flags(
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF:%.*]], label [[ELSE:%.*]]
+; CHECK:       common.ret:
+; CHECK-NEXT:    [[COMMON_RET_OP:%.*]] = phi i32 [ [[OP1:%.*]], [[IF]] ], [ [[OP2:%.*]], [[ELSE]] ]
+; CHECK-NEXT:    ret i32 [[COMMON_RET_OP]]
+; CHECK:       if:
+; CHECK-NEXT:    [[OP1]] = add nuw nsw i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[COMMON_RET:%.*]]
+; CHECK:       else:
+; CHECK-NEXT:    [[OP2]] = add nsw i32 [[B]], [[A]]
+; CHECK-NEXT:    call void @bar()
+; CHECK-NEXT:    br label [[COMMON_RET]]
+;
+  br i1 %c, label %if, label %else
+
+if:
+  %op1 = add nuw nsw i32 %a, %b
+  call void @foo()
+  ret i32 %op1
+
+else:
+  %op2 = add nsw i32 %b, %a
+  call void @bar()
+  ret i32 %op2
+}
+
+define i32 @test_binop_not_commutative(i1 %c, i32 %a, i32 %b) {
+; CHECK-LABEL: @test_binop_not_commutative(
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF:%.*]], label [[ELSE:%.*]]
+; CHECK:       common.ret:
+; CHECK-NEXT:    [[COMMON_RET_OP:%.*]] = phi i32 [ [[OP1:%.*]], [[IF]] ], [ [[OP2:%.*]], [[ELSE]] ]
+; CHECK-NEXT:    ret i32 [[COMMON_RET_OP]]
+; CHECK:       if:
+; CHECK-NEXT:    [[OP1]] = sub i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[COMMON_RET:%.*]]
+; CHECK:       else:
+; CHECK-NEXT:    [[OP2]] = sub i32 [[B]], [[A]]
+; CHECK-NEXT:    call void @bar()
+; CHECK-NEXT:    br label [[COMMON_RET]]
+;
+  br i1 %c, label %if, label %else
+
+if:
+  %op1 = sub i32 %a, %b
+  call void @foo()
+  ret i32 %op1
+
+else:
+  %op2 = sub i32 %b, %a
+  call void @bar()
+  ret i32 %op2
+}
+
+define i32 @test_binop_wrong_ops(i1 %c, i32 %a, i32 %b, i32 %d) {
+; CHECK-LABEL: @test_binop_wrong_ops(
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF:%.*]], label [[ELSE:%.*]]
+; CHECK:       common.ret:
+; CHECK-NEXT:    [[COMMON_RET_OP:%.*]] = phi i32 [ [[OP1:%.*]], [[IF]] ], [ [[OP2:%.*]], [[ELSE]] ]
+; CHECK-NEXT:    ret i32 [[COMMON_RET_OP]]
+; CHECK:       if:
+; CHECK-NEXT:    [[OP1]] = add i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[COMMON_RET:%.*]]
+; CHECK:       else:
+; CHECK-NEXT:    [[OP2]] = add i32 [[B]], [[D:%.*]]
+; CHECK-NEXT:    call void @bar()
+; CHECK-NEXT:    br label [[COMMON_RET]]
+;
+  br i1 %c, label %if, label %else
+
+if:
+  %op1 = add i32 %a, %b
+  call void @foo()
+  ret i32 %op1
+
+else:
+  %op2 = add i32 %b, %d
+  call void @bar()
+  ret i32 %op2
+}
+
+define i32 @test_intrin(i1 %c, i32 %a, i32 %b) {
+; CHECK-LABEL: @test_intrin(
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF:%.*]], label [[ELSE:%.*]]
+; CHECK:       common.ret:
+; CHECK-NEXT:    [[COMMON_RET_OP:%.*]] = phi i32 [ [[OP1:%.*]], [[IF]] ], [ [[OP2:%.*]], [[ELSE]] ]
+; CHECK-NEXT:    ret i32 [[COMMON_RET_OP]]
+; CHECK:       if:
+; CHECK-NEXT:    [[OP1]] = call i32 @llvm.umin.i32(i32 [[A:%.*]], i32 [[B:%.*]])
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[COMMON_RET:%.*]]
+; CHECK:       else:
+; CHECK-NEXT:    [[OP2]] = call i32 @llvm.umin.i32(i32 [[B]], i32 [[A]])
+; CHECK-NEXT:    call void @bar()
+; CHECK-NEXT:    br label [[COMMON_RET]]
+;
+  br i1 %c, label %if, label %else
+
+if:
+  %op1 = call i32 @llvm.umin(i32 %a, i32 %b)
+  call void @foo()
+  ret i32 %op1
+
+else:
+  %op2 = call i32 @llvm.umin(i32 %b, i32 %a)
+  call void @bar()
+  ret i32 %op2
+}
+
+define i32 @test_intrin_not_same(i1 %c, i32 %a, i32 %b) {
+; CHECK-LABEL: @test_intrin_not_same(
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF:%.*]], label [[ELSE:%.*]]
+; CHECK:       common.ret:
+; CHECK-NEXT:    [[COMMON_RET_OP:%.*]] = phi i32 [ [[OP1:%.*]], [[IF]] ], [ [[OP2:%.*]], [[ELSE]] ]
+; CHECK-NEXT:    ret i32 [[COMMON_RET_OP]]
+; CHECK:       if:
+; CHECK-NEXT:    [[OP1]] = call i32 @llvm.umin.i32(i32 [[A:%.*]], i32 [[B:%.*]])
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[COMMON_RET:%.*]]
+; CHECK:       else:
+; CHECK-NEXT:    [[OP2]] = call i32 @llvm.umax.i32(i32 [[B]], i32 [[A]])
+; CHECK-NEXT:    call void @bar()
+; CHECK-NEXT:    br label [[COMMON_RET]]
+;
+  br i1 %c, label %if, label %else
+
+if:
+  %op1 = call i32 @llvm.umin(i32 %a, i32 %b)
+  call void @foo()
+  ret i32 %op1
+
+else:
+  %op2 = call i32 @llvm.umax(i32 %b, i32 %a)
+  call void @bar()
+  ret i32 %op2
+}
+
+define float @test_intrin_3arg(i1 %c, float %a, float %b, float %d) {
+; CHECK-LABEL: @test_intrin_3arg(
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF:%.*]], label [[ELSE:%.*]]
+; CHECK:       common.ret:
+; CHECK-NEXT:    [[COMMON_RET_OP:%.*]] = phi float [ [[OP1:%.*]], [[IF]] ], [ [[OP2:%.*]], [[ELSE]] ]
+; CHECK-NEXT:    ret float [[COMMON_RET_OP]]
+; CHECK:       if:
+; CHECK-NEXT:    [[OP1]] = call float @llvm.fma.f32(float [[A:%.*]], float [[B:%.*]], float [[D:%.*]])
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[COMMON_RET:%.*]]
+; CHECK:       else:
+; CHECK-NEXT:    [[OP2]] = call float @llvm.fma.f32(float [[B]], float [[A]], float [[D]])
+; CHECK-NEXT:    call void @bar()
+; CHECK-NEXT:    br label [[COMMON_RET]]
+;
+  br i1 %c, label %if, label %else
+
+if:
+  %op1 = call float @llvm.fma(float %a, float %b, float %d)
+  call void @foo()
+  ret float %op1
+
+else:
+  %op2 = call float @llvm.fma(float %b, float %a, float %d)
+  call void @bar()
+  ret float %op2
+}
+
+define float @test_intrin_3arg_wrong_args_commuted(i1 %c, float %a, float %b, float %d) {
+; CHECK-LABEL: @test_intrin_3arg_wrong_args_commuted(
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF:%.*]], label [[ELSE:%.*]]
+; CHECK:       common.ret:
+; CHECK-NEXT:    [[COMMON_RET_OP:%.*]] = phi float [ [[OP1:%.*]], [[IF]] ], [ [[OP2:%.*]], [[ELSE]] ]
+; CHECK-NEXT:    ret float [[COMMON_RET_OP]]
+; CHECK:       if:
+; CHECK-NEXT:    [[OP1]] = call float @llvm.fma.f32(float [[A:%.*]], float [[B:%.*]], float [[D:%.*]])
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[COMMON_RET:%.*]]
+; CHECK:       else:
+; CHECK-NEXT:    [[OP2]] = call float @llvm.fma.f32(float [[A]], float [[D]], float [[B]])
+; CHECK-NEXT:    call void @bar()
+; CHECK-NEXT:    br label [[COMMON_RET]]
+;
+  br i1 %c, label %if, label %else
+
+if:
+  %op1 = call float @llvm.fma(float %a, float %b, float %d)
+  call void @foo()
+  ret float %op1
+
+else:
+  %op2 = call float @llvm.fma(float %a, float %d, float %b)
+  call void @bar()
+  ret float %op2
+}

From b1d75fe48c940ba677614db3d891fbebcb8a41a2 Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett@linaro.org>
Date: Mon, 19 Aug 2024 15:16:04 +0000
Subject: [PATCH 369/441] [lldb][test] Fix cast dropping const warnin in
 TestBreakpointSetCallback.cpp
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

When building with gcc I get this warning:
```
<...>TestBreakpointSetCallback.cpp:58:25: warning: cast from type ‘const char*’ to type ‘void*’ casts away qualifiers [-Wcast-qual]
   58 |   void *baton = (void *)"hello";
      |                         ^~~~~~~
```

Use the address of a mutable global variable instead. All we care about
is that the address passed as the baton is the same one we get later.
---
 lldb/unittests/Callback/TestBreakpointSetCallback.cpp | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/lldb/unittests/Callback/TestBreakpointSetCallback.cpp b/lldb/unittests/Callback/TestBreakpointSetCallback.cpp
index 2a7070f9349c029..3dba4a9eb719e3e 100644
--- a/lldb/unittests/Callback/TestBreakpointSetCallback.cpp
+++ b/lldb/unittests/Callback/TestBreakpointSetCallback.cpp
@@ -30,6 +30,8 @@ using namespace lldb;
 static constexpr lldb::user_id_t expected_breakpoint_id = 1;
 static constexpr lldb::user_id_t expected_breakpoint_location_id = 0;
 
+int baton_value;
+
 class BreakpointSetCallbackTest : public ::testing::Test {
 public:
   static void CheckCallbackArgs(void *baton, StoppointCallbackContext *context,
@@ -37,7 +39,7 @@ class BreakpointSetCallbackTest : public ::testing::Test {
                                 lldb::user_id_t break_loc_id,
                                 TargetSP expected_target_sp) {
     EXPECT_EQ(context->exe_ctx_ref.GetTargetSP(), expected_target_sp);
-    EXPECT_EQ(baton, "hello");
+    EXPECT_EQ(baton, &baton_value);
     EXPECT_EQ(break_id, expected_breakpoint_id);
     EXPECT_EQ(break_loc_id, expected_breakpoint_location_id);
   }
@@ -55,7 +57,6 @@ class BreakpointSetCallbackTest : public ::testing::Test {
 };
 
 TEST_F(BreakpointSetCallbackTest, TestBreakpointSetCallback) {
-  void *baton = (void *)"hello";
   // Set up the debugger, make sure that was done properly.
   TargetSP target_sp;
   ArchSpec arch("x86_64-apple-macosx-");
@@ -78,7 +79,7 @@ TEST_F(BreakpointSetCallbackTest, TestBreakpointSetCallback) {
         CheckCallbackArgs(baton, context, break_id, break_loc_id, target_sp);
         return true;
       },
-      baton, true);
+      (void *)&baton_value, true);
   ExecutionContext exe_ctx(target_sp, false);
   StoppointCallbackContext context(nullptr, exe_ctx, true);
   breakpoint_sp->InvokeCallback(&context, 0);

From c7a54bfd1d25330199c96dd0a46cef1644b1b1ce Mon Sep 17 00:00:00 2001
From: Michael Buch <michaelbuch12@gmail.com>
Date: Mon, 19 Aug 2024 16:17:27 +0100
Subject: [PATCH 370/441] [lldb][ASTUtils] Remove unused
 SemaSourceWithPriorities::addSource API

As far as I can tell, this has always been unused. My hunch is that
this was supposed to mimick the `MultiplexExternalSemaSource::AddSource`
API which `SemaSourceWithPriorities` is based on.
---
 lldb/source/Plugins/ExpressionParser/Clang/ASTUtils.h | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ASTUtils.h b/lldb/source/Plugins/ExpressionParser/Clang/ASTUtils.h
index da2b1a15f74614a..79ed74b79f04fd6 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ASTUtils.h
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ASTUtils.h
@@ -264,10 +264,6 @@ class SemaSourceWithPriorities : public clang::ExternalSemaSource {
 
   ~SemaSourceWithPriorities() override;
 
-  void addSource(clang::ExternalSemaSource &source) {
-    Sources.push_back(&source);
-  }
-
   //===--------------------------------------------------------------------===//
   // ExternalASTSource.
   //===--------------------------------------------------------------------===//

From 0abb7791614947bc24931dd851ade31d02496977 Mon Sep 17 00:00:00 2001
From: Wesley Wiser <wwiser@gmail.com>
Date: Mon, 19 Aug 2024 10:31:20 -0500
Subject: [PATCH 371/441] [LLVM] [X86] Fix integer overflows in frame layout
 for huge frames (#101840)

Fix 32-bit integer overflows in the X86 target frame layout when dealing
with frames larger than 4gb. When this occurs, we'll scavenge a scratch
register to be able to hold the correct stack offset for frame locals.

This completes reapplying #84114.

Fixes #48911
Fixes #75944
Fixes #87154
---
 llvm/lib/CodeGen/PrologEpilogInserter.cpp     |  2 +-
 llvm/lib/Target/X86/X86FrameLowering.cpp      | 11 ++-
 llvm/lib/Target/X86/X86RegisterInfo.cpp       | 36 ++++++++--
 llvm/lib/Target/X86/X86RegisterInfo.h         |  9 +++
 llvm/test/CodeGen/X86/avx512f-large-stack.ll  | 23 ++++++
 llvm/test/CodeGen/X86/huge-stack.ll           | 72 ++++++++++++++++---
 .../CodeGen/X86/win64-stackprobe-overflow.ll  |  2 +-
 7 files changed, 136 insertions(+), 19 deletions(-)
 create mode 100644 llvm/test/CodeGen/X86/avx512f-large-stack.ll

diff --git a/llvm/lib/CodeGen/PrologEpilogInserter.cpp b/llvm/lib/CodeGen/PrologEpilogInserter.cpp
index ee03eaa8ae527c6..c03ea5878052130 100644
--- a/llvm/lib/CodeGen/PrologEpilogInserter.cpp
+++ b/llvm/lib/CodeGen/PrologEpilogInserter.cpp
@@ -1553,7 +1553,7 @@ void PEI::replaceFrameIndices(MachineBasicBlock *BB, MachineFunction &MF,
       // If this instruction has a FrameIndex operand, we need to
       // use that target machine register info object to eliminate
       // it.
-      TRI.eliminateFrameIndex(MI, SPAdj, i);
+      TRI.eliminateFrameIndex(MI, SPAdj, i, RS);
 
       // Reset the iterator if we were at the beginning of the BB.
       if (AtBeginning) {
diff --git a/llvm/lib/Target/X86/X86FrameLowering.cpp b/llvm/lib/Target/X86/X86FrameLowering.cpp
index 8404f2231680d61..d850f4fd7683118 100644
--- a/llvm/lib/Target/X86/X86FrameLowering.cpp
+++ b/llvm/lib/Target/X86/X86FrameLowering.cpp
@@ -24,6 +24,7 @@
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/RegisterScavenging.h"
 #include "llvm/CodeGen/WinEHFuncInfo.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/EHPersonalities.h"
@@ -2616,7 +2617,7 @@ StackOffset X86FrameLowering::getFrameIndexReference(const MachineFunction &MF,
   // object.
   // We need to factor in additional offsets applied during the prologue to the
   // frame, base, and stack pointer depending on which is used.
-  int Offset = MFI.getObjectOffset(FI) - getOffsetOfLocalArea();
+  int64_t Offset = MFI.getObjectOffset(FI) - getOffsetOfLocalArea();
   const X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
   unsigned CSSize = X86FI->getCalleeSavedFrameSize();
   uint64_t StackSize = MFI.getStackSize();
@@ -4140,6 +4141,14 @@ void X86FrameLowering::processFunctionBeforeFrameFinalized(
   // emitPrologue if it gets called and emits CFI.
   MF.setHasWinCFI(false);
 
+  MachineFrameInfo &MFI = MF.getFrameInfo();
+  // If the frame is big enough that we might need to scavenge a register to
+  // handle huge offsets, reserve a stack slot for that now.
+  if (!isInt<32>(MFI.estimateStackSize(MF))) {
+    int FI = MFI.CreateStackObject(SlotSize, Align(SlotSize), false);
+    RS->addScavengingFrameIndex(FI);
+  }
+
   // If we are using Windows x64 CFI, ensure that the stack is always 8 byte
   // aligned. The format doesn't support misaligned stack adjustments.
   if (MF.getTarget().getMCAsmInfo()->usesWindowsCFI())
diff --git a/llvm/lib/Target/X86/X86RegisterInfo.cpp b/llvm/lib/Target/X86/X86RegisterInfo.cpp
index 638eb1c4f11e417..79ee9ecfdf3ce7e 100644
--- a/llvm/lib/Target/X86/X86RegisterInfo.cpp
+++ b/llvm/lib/Target/X86/X86RegisterInfo.cpp
@@ -13,6 +13,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "X86RegisterInfo.h"
+#include "MCTargetDesc/X86BaseInfo.h"
 #include "X86FrameLowering.h"
 #include "X86MachineFunctionInfo.h"
 #include "X86Subtarget.h"
@@ -24,6 +25,7 @@
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/RegisterScavenging.h"
 #include "llvm/CodeGen/TargetFrameLowering.h"
 #include "llvm/CodeGen/TargetInstrInfo.h"
 #include "llvm/CodeGen/TileShapeInfo.h"
@@ -905,7 +907,7 @@ X86RegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
   int FrameIndex = MI.getOperand(FIOperandNum).getIndex();
 
   // Determine base register and offset.
-  int FIOffset;
+  int64_t FIOffset;
   Register BasePtr;
   if (MI.isReturn()) {
     assert((!hasStackRealignment(MF) ||
@@ -956,10 +958,34 @@ X86RegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
   }
 
   if (MI.getOperand(FIOperandNum+3).isImm()) {
-    // Offset is a 32-bit integer.
-    int Imm = (int)(MI.getOperand(FIOperandNum + 3).getImm());
-    int Offset = FIOffset + Imm;
-    assert((!Is64Bit || isInt<32>((long long)FIOffset + Imm)) &&
+    int64_t Imm = MI.getOperand(FIOperandNum + 3).getImm();
+    int64_t Offset = FIOffset + Imm;
+    bool FitsIn32Bits = isInt<32>(Offset);
+    // If the offset will not fit in a 32-bit displacement,
+    // then for 64-bit targets, scavenge a register to hold it.
+    // Otherwise, for 32-bit targets, this is a bug!
+    if (Is64Bit && !FitsIn32Bits) {
+      assert(RS && "RegisterScavenger was NULL");
+      const X86InstrInfo *TII = MF.getSubtarget<X86Subtarget>().getInstrInfo();
+      const DebugLoc &DL = MI.getDebugLoc();
+
+      RS->enterBasicBlockEnd(MBB);
+      RS->backward(std::next(II));
+
+      Register ScratchReg = RS->scavengeRegisterBackwards(
+          X86::GR64RegClass, II, /*RestoreAfter=*/false, /*SPAdj=*/0,
+          /*AllowSpill=*/true);
+      assert(ScratchReg != 0 && "scratch reg was 0");
+      RS->setRegUsed(ScratchReg);
+
+      BuildMI(MBB, II, DL, TII->get(X86::MOV64ri), ScratchReg).addImm(Offset);
+
+      MI.getOperand(FIOperandNum + 3).setImm(0);
+      MI.getOperand(FIOperandNum + 2).setReg(ScratchReg);
+
+      return false;
+    }
+    assert((Is64Bit || FitsIn32Bits) &&
            "Requesting 64-bit offset in 32-bit immediate!");
     if (Offset != 0 || !tryOptimizeLEAtoMOV(II))
       MI.getOperand(FIOperandNum + 3).ChangeToImmediate(Offset);
diff --git a/llvm/lib/Target/X86/X86RegisterInfo.h b/llvm/lib/Target/X86/X86RegisterInfo.h
index 7296a5f021e4ad4..dd03a108fb8e693 100644
--- a/llvm/lib/Target/X86/X86RegisterInfo.h
+++ b/llvm/lib/Target/X86/X86RegisterInfo.h
@@ -13,6 +13,8 @@
 #ifndef LLVM_LIB_TARGET_X86_X86REGISTERINFO_H
 #define LLVM_LIB_TARGET_X86_X86REGISTERINFO_H
 
+#include "llvm/CodeGen/MachineFrameInfo.h"
+#include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/TargetRegisterInfo.h"
 
 #define GET_REGINFO_HEADER
@@ -176,6 +178,13 @@ class X86RegisterInfo final : public X86GenRegisterInfo {
                              SmallVectorImpl<MCPhysReg> &Hints,
                              const MachineFunction &MF, const VirtRegMap *VRM,
                              const LiveRegMatrix *Matrix) const override;
+
+  bool requiresRegisterScavenging(const MachineFunction &MF) const override {
+    const MachineFrameInfo &MFI = MF.getFrameInfo();
+
+    // We need to register scavenge if the frame is very large.
+    return !isInt<32>(MFI.estimateStackSize(MF));
+  }
 };
 
 } // End llvm namespace
diff --git a/llvm/test/CodeGen/X86/avx512f-large-stack.ll b/llvm/test/CodeGen/X86/avx512f-large-stack.ll
new file mode 100644
index 000000000000000..3cb5391c56abf5c
--- /dev/null
+++ b/llvm/test/CodeGen/X86/avx512f-large-stack.ll
@@ -0,0 +1,23 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --no_x86_scrub_sp --version 4
+; RUN: llc -O0 -mtriple=x86_64 -mattr=+avx512f -verify-machineinstrs < %s | FileCheck %s --check-prefix=CHECK
+define void @f(i16 %LGV2, i1 %LGV3) {
+; CHECK-LABEL: f:
+; CHECK:       # %bb.0: # %BB
+; CHECK-NEXT:    subq $2147483528, %rsp # imm = 0x7FFFFF88
+; CHECK-NEXT:    .cfi_def_cfa_offset 2147483536
+; CHECK-NEXT:    movb %sil, %cl
+; CHECK-NEXT:    movw %di, %ax
+; CHECK-NEXT:    movswq %ax, %rax
+; CHECK-NEXT:    andb $1, %cl
+; CHECK-NEXT:    movabsq $-2147483768, %rdx # imm = 0xFFFFFFFF7FFFFF88
+; CHECK-NEXT:    movb %cl, (%rsp,%rdx)
+; CHECK-NEXT:    addq $2147483528, %rsp # imm = 0x7FFFFF88
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
+BB:
+  %A = alloca i1, i33 2147483648, align 1
+  %G = getelementptr i1, ptr %A, i16 %LGV2
+  %G4 = getelementptr i1, ptr %G, i32 -2147483648
+  store i1 %LGV3, ptr %G4, align 1
+  ret void
+}
diff --git a/llvm/test/CodeGen/X86/huge-stack.ll b/llvm/test/CodeGen/X86/huge-stack.ll
index 920033ba1182c3f..41b8a0141b63d8b 100644
--- a/llvm/test/CodeGen/X86/huge-stack.ll
+++ b/llvm/test/CodeGen/X86/huge-stack.ll
@@ -5,20 +5,70 @@
 define void @foo() unnamed_addr #0 {
 ; CHECK-LABEL: foo:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    movabsq $8589934462, %rax # imm = 0x1FFFFFF7E
+; CHECK-NEXT:    movabsq $8589934472, %rax # imm = 0x1FFFFFF88
 ; CHECK-NEXT:    subq %rax, %rsp
-; CHECK-NEXT:    .cfi_def_cfa_offset 8589934470
-; CHECK-NEXT:    movb $42, -129(%rsp)
-; CHECK-NEXT:    movb $43, -128(%rsp)
-; CHECK-NEXT:    movabsq $8589934462, %rax # imm = 0x1FFFFFF7E
+; CHECK-NEXT:    .cfi_def_cfa_offset 8589934480
+; CHECK-NEXT:    movabsq $4294967177, %rax # imm = 0xFFFFFF89
+; CHECK-NEXT:    movb $42, (%rsp,%rax)
+; CHECK-NEXT:    movb $43, -118(%rsp)
+; CHECK-NEXT:    movabsq $8589934472, %rax # imm = 0x1FFFFFF88
 ; CHECK-NEXT:    addq %rax, %rsp
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-NEXT:    retq
-  %1 = alloca %large, align 1
-  %2 = alloca %large, align 1
-  %3 = getelementptr inbounds %large, ptr %1, i64 0, i64 0
-  store i8 42, ptr %3, align 1
-  %4 = getelementptr inbounds %large, ptr %2, i64 0, i64 0
-  store i8 43, ptr %4, align 1
+  %large1 = alloca %large, align 1
+  %large2 = alloca %large, align 1
+  %ptrLarge1 = getelementptr inbounds %large, ptr %large1, i64 0, i64 0
+  store i8 42, ptr %ptrLarge1, align 1
+  %ptrLarge2 = getelementptr inbounds %large, ptr %large2, i64 0, i64 0
+  store i8 43, ptr %ptrLarge2, align 1
   ret void
 }
+
+declare ptr @baz(ptr, ptr, ptr, ptr)
+
+define ptr @scavenge_spill() unnamed_addr #0 {
+; CHECK-LABEL: scavenge_spill:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movabsq $25769803816, %rax # imm = 0x600000028
+; CHECK-NEXT:    subq %rax, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 25769803824
+; CHECK-NEXT:    movabsq $21474836521, %rax # imm = 0x500000029
+; CHECK-NEXT:    leaq (%rsp,%rax), %rdi
+; CHECK-NEXT:    movabsq $17179869226, %rax # imm = 0x40000002A
+; CHECK-NEXT:    leaq (%rsp,%rax), %rsi
+; CHECK-NEXT:    movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    movabsq $12884901931, %rax # imm = 0x30000002B
+; CHECK-NEXT:    leaq (%rsp,%rax), %rdx
+; CHECK-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    movabsq $8589934636, %rax # imm = 0x20000002C
+; CHECK-NEXT:    leaq (%rsp,%rax), %rcx
+; CHECK-NEXT:    movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    callq baz@PLT
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; CHECK-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    leaq 46(%rsp), %rdi
+; CHECK-NEXT:    callq baz@PLT
+; CHECK-NEXT:    # kill: def $rcx killed $rax
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; CHECK-NEXT:    movabsq $25769803816, %rcx # imm = 0x600000028
+; CHECK-NEXT:    addq %rcx, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
+  %large1 = alloca %large, align 1
+  %ptrLarge1 = getelementptr inbounds %large, ptr %large1, i64 0, i64 0
+  %large2 = alloca %large, align 1
+  %ptrLarge2 = getelementptr inbounds %large, ptr %large2, i64 0, i64 0
+  %large3 = alloca %large, align 1
+  %ptrLarge3 = getelementptr inbounds %large, ptr %large3, i64 0, i64 0
+  %large4 = alloca %large, align 1
+  %ptrLarge4 = getelementptr inbounds %large, ptr %large4, i64 0, i64 0
+  %large5 = alloca %large, align 1
+  %ptrLarge5 = getelementptr inbounds %large, ptr %large5, i64 0, i64 0
+  %ret1 = call ptr @baz(ptr %ptrLarge1, ptr %ptrLarge2, ptr %ptrLarge3, ptr %ptrLarge4)
+  %large6 = alloca %large, align 1
+  %ptrLarge6 = getelementptr inbounds %large, ptr %large6, i64 0, i64 0
+  %ret2 = call ptr @baz(ptr %ptrLarge6, ptr %ptrLarge2, ptr %ptrLarge3, ptr %ptrLarge4)
+  ret ptr %ret1
+}
diff --git a/llvm/test/CodeGen/X86/win64-stackprobe-overflow.ll b/llvm/test/CodeGen/X86/win64-stackprobe-overflow.ll
index 9555ce032db90cb..732fc6543e31410 100644
--- a/llvm/test/CodeGen/X86/win64-stackprobe-overflow.ll
+++ b/llvm/test/CodeGen/X86/win64-stackprobe-overflow.ll
@@ -10,5 +10,5 @@ start:
 attributes #0 = { nonlazybind uwtable "probe-stack"="probe_stack" "target-cpu"="x86-64" }
 
 ; CHECK-LABEL: foo:
-; CHECK: movabsq $4294967304, %rax
+; CHECK: movabsq $4294967312, %rax
 ; CHECK-NEXT: callq probe_stack

From 576aa3a50960f02e4dfe2d6ed2c55aa66006adcf Mon Sep 17 00:00:00 2001
From: Akshat Oke <76596238+Akshat-Oke@users.noreply.github.com>
Date: Mon, 19 Aug 2024 21:05:39 +0530
Subject: [PATCH 372/441] [TableGen] Resolve References at top level (#104578)

Add a dummy resolver to resolve references outside records. This invokes
Fold() with isFinal to force resolution.

Fixes #102447

Co-authored-by: Akshat Oke <Akshat.Oke@amd.com>
---
 llvm/lib/TableGen/TGParser.cpp | 9 +++++++--
 llvm/test/TableGen/dump.td     | 3 +++
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/TableGen/TGParser.cpp b/llvm/lib/TableGen/TGParser.cpp
index ce9da960545c231..d1d46105c814e96 100644
--- a/llvm/lib/TableGen/TGParser.cpp
+++ b/llvm/lib/TableGen/TGParser.cpp
@@ -4460,8 +4460,13 @@ bool TGParser::ParseDump(MultiClass *CurMultiClass, Record *CurRec) {
 
   if (CurRec)
     CurRec->addDump(Loc, Message);
-  else
-    addEntry(std::make_unique<Record::DumpInfo>(Loc, Message));
+  else {
+    HasReferenceResolver resolver{nullptr};
+    resolver.setFinal(true);
+    // force a resolution with a dummy resolver
+    Init *ResolvedMessage = Message->resolveReferences(resolver);
+    addEntry(std::make_unique<Record::DumpInfo>(Loc, ResolvedMessage));
+  }
 
   return false;
 }
diff --git a/llvm/test/TableGen/dump.td b/llvm/test/TableGen/dump.td
index 633e73802ffa3ea..7f021755e2e821a 100644
--- a/llvm/test/TableGen/dump.td
+++ b/llvm/test/TableGen/dump.td
@@ -101,3 +101,6 @@ def Three : BaseClassForSet;
 }
 // CHECK: [[FILE]]:[[@LINE+1]]:1: note: TheSet = [Subset_One, Subset_Two, Three]
 dump "TheSet = " # !repr(TheSet);
+
+// CHECK: [[FILE]]:[[@LINE+1]]:1: note: 0
+dump !repr(!exists<BaseClassForSet>("non-existent-record"));
\ No newline at end of file

From 08201cb4245b0a03e1af664e00a22ea4db1fc2fb Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett@linaro.org>
Date: Mon, 19 Aug 2024 16:33:46 +0100
Subject: [PATCH 373/441] [lldb][test] Fix GCC warnings in
 TestGetControlFlowKindX86.cpp
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

```
<...>/TestGetControlFlowKindx86.cpp:148:8: warning: suggest explicit braces to avoid ambiguous ‘else’ [-Wdangling-else]
  148 |     if (kind == eInstructionControlFlowKindReturn)
      |        ^
```

Usually llvm is a "no braces for single line body" project but
for whatever reason gcc objects to it here. Perhaps because it's
within a for loop.

Added the newlines just for readability.
---
 .../Disassembler/x86/TestGetControlFlowKindx86.cpp    | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/lldb/unittests/Disassembler/x86/TestGetControlFlowKindx86.cpp b/lldb/unittests/Disassembler/x86/TestGetControlFlowKindx86.cpp
index fecd93361e56338..e9d5ae3f7b3cdba 100644
--- a/lldb/unittests/Disassembler/x86/TestGetControlFlowKindx86.cpp
+++ b/lldb/unittests/Disassembler/x86/TestGetControlFlowKindx86.cpp
@@ -145,14 +145,19 @@ TEST_F(TestGetControlFlowKindx86, TestX86_64Instruction) {
     EXPECT_EQ(kind, result[i]);
 
     // Also, test the DisassemblerLLVMC::MCDisasmInstance methods.
-    if (kind == eInstructionControlFlowKindReturn)
+    if (kind == eInstructionControlFlowKindReturn) {
       EXPECT_FALSE(inst_sp->IsCall());
-    if (kind == eInstructionControlFlowKindCall)
+    }
+
+    if (kind == eInstructionControlFlowKindCall) {
       EXPECT_TRUE(inst_sp->IsCall());
+    }
+
     if (kind == eInstructionControlFlowKindCall ||
         kind == eInstructionControlFlowKindJump ||
         kind == eInstructionControlFlowKindCondJump ||
-        kind == eInstructionControlFlowKindReturn)
+        kind == eInstructionControlFlowKindReturn) {
       EXPECT_TRUE(inst_sp->DoesBranch());
+    }
   }
 }

From 8d165136ac36a53e51b50c620a12039d6c1e628e Mon Sep 17 00:00:00 2001
From: meehatpa <gune30@gmail.com>
Date: Mon, 19 Aug 2024 21:09:52 +0530
Subject: [PATCH 374/441] [mlir][[spirv] Add support for math.log2 and
 math.log10 to GLSL/OpenCL SPIRV Backends (#104608)

As log2 and log10 are not available in spirv, realize them as a
decomposition using spirv.CL.log/spirv.GL.Log.
---
 .../Conversion/MathToSPIRV/MathToSPIRV.cpp    | 63 +++++++++++++++++++
 .../MathToSPIRV/math-to-gl-spirv.mlir         | 42 +++++++++----
 .../MathToSPIRV/math-to-opencl-spirv.mlir     | 46 +++++++++-----
 3 files changed, 123 insertions(+), 28 deletions(-)

diff --git a/mlir/lib/Conversion/MathToSPIRV/MathToSPIRV.cpp b/mlir/lib/Conversion/MathToSPIRV/MathToSPIRV.cpp
index 5b3c2fb15e70263..52ff138bedf65bd 100644
--- a/mlir/lib/Conversion/MathToSPIRV/MathToSPIRV.cpp
+++ b/mlir/lib/Conversion/MathToSPIRV/MathToSPIRV.cpp
@@ -291,6 +291,65 @@ struct Log1pOpPattern final : public OpConversionPattern<math::Log1pOp> {
   }
 };
 
+/// Converts math.log2 and math.log10 to SPIR-V ops.
+///
+/// SPIR-V does not have direct operations for log2 and log10. Explicitly
+/// lower to these operations using:
+///   log2(x) = log(x) * 1/log(2)
+///   log10(x) = log(x) * 1/log(10)
+
+template <typename MathLogOp, typename SpirvLogOp>
+struct Log2Log10OpPattern final : public OpConversionPattern<MathLogOp> {
+  using OpConversionPattern<MathLogOp>::OpConversionPattern;
+  using typename OpConversionPattern<MathLogOp>::OpAdaptor;
+
+  static constexpr double log2Reciprocal =
+      1.442695040888963407359924681001892137426645954152985934135449407;
+  static constexpr double log10Reciprocal =
+      0.4342944819032518276511289189166050822943970058036665661144537832;
+
+  LogicalResult
+  matchAndRewrite(MathLogOp operation, OpAdaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override {
+    assert(adaptor.getOperands().size() == 1);
+    if (LogicalResult res = checkSourceOpTypes(rewriter, operation);
+        failed(res))
+      return res;
+
+    Location loc = operation.getLoc();
+    Type type = this->getTypeConverter()->convertType(operation.getType());
+    if (!type)
+      return rewriter.notifyMatchFailure(operation, "type conversion failed");
+
+    auto getConstantValue = [&](double value) {
+      if (auto floatType = dyn_cast<FloatType>(type)) {
+        return rewriter.create<spirv::ConstantOp>(
+            loc, type, rewriter.getFloatAttr(floatType, value));
+      }
+      if (auto vectorType = dyn_cast<VectorType>(type)) {
+        Type elemType = vectorType.getElementType();
+
+        if (isa<FloatType>(elemType)) {
+          return rewriter.create<spirv::ConstantOp>(
+              loc, type,
+              DenseFPElementsAttr::get(
+                  vectorType, FloatAttr::get(elemType, value).getValue()));
+        }
+      }
+
+      llvm_unreachable("unimplemented types for log2/log10");
+    };
+
+    Value constantValue = getConstantValue(
+        std::is_same<MathLogOp, math::Log2Op>() ? log2Reciprocal
+                                                : log10Reciprocal);
+    Value log = rewriter.create<SpirvLogOp>(loc, adaptor.getOperand());
+    rewriter.replaceOpWithNewOp<spirv::FMulOp>(operation, type, log,
+                                               constantValue);
+    return success();
+  }
+};
+
 /// Converts math.powf to SPIRV-Ops.
 struct PowFOpPattern final : public OpConversionPattern<math::PowFOp> {
   using OpConversionPattern::OpConversionPattern;
@@ -411,6 +470,8 @@ void populateMathToSPIRVPatterns(SPIRVTypeConverter &typeConverter,
   // GLSL patterns
   patterns
       .add<CountLeadingZerosPattern, Log1pOpPattern<spirv::GLLogOp>,
+           Log2Log10OpPattern<math::Log2Op, spirv::GLLogOp>,
+           Log2Log10OpPattern<math::Log10Op, spirv::GLLogOp>,
            ExpM1OpPattern<spirv::GLExpOp>, PowFOpPattern, RoundOpPattern,
            CheckedElementwiseOpPattern<math::AbsFOp, spirv::GLFAbsOp>,
            CheckedElementwiseOpPattern<math::AbsIOp, spirv::GLSAbsOp>,
@@ -430,6 +491,8 @@ void populateMathToSPIRVPatterns(SPIRVTypeConverter &typeConverter,
 
   // OpenCL patterns
   patterns.add<Log1pOpPattern<spirv::CLLogOp>, ExpM1OpPattern<spirv::CLExpOp>,
+               Log2Log10OpPattern<math::Log2Op, spirv::CLLogOp>,
+               Log2Log10OpPattern<math::Log10Op, spirv::CLLogOp>,
                CheckedElementwiseOpPattern<math::AbsFOp, spirv::CLFAbsOp>,
                CheckedElementwiseOpPattern<math::AbsIOp, spirv::CLSAbsOp>,
                CheckedElementwiseOpPattern<math::AtanOp, spirv::CLAtanOp>,
diff --git a/mlir/test/Conversion/MathToSPIRV/math-to-gl-spirv.mlir b/mlir/test/Conversion/MathToSPIRV/math-to-gl-spirv.mlir
index a93976673934295..ecbd59e54971ef4 100644
--- a/mlir/test/Conversion/MathToSPIRV/math-to-gl-spirv.mlir
+++ b/mlir/test/Conversion/MathToSPIRV/math-to-gl-spirv.mlir
@@ -22,22 +22,30 @@ func.func @float32_unary_scalar(%arg0: f32) {
   // CHECK: %[[ADDONE:.+]] = spirv.FAdd %[[ONE]], %{{.+}}
   // CHECK: spirv.GL.Log %[[ADDONE]]
   %5 = math.log1p %arg0 : f32
+  // CHECK: %[[LOG2_RECIPROCAL:.+]] = spirv.Constant 1.44269502 : f32
+  // CHECK: %[[LOG0:.+]] = spirv.GL.Log {{.+}}
+  // CHECK: spirv.FMul %[[LOG0]], %[[LOG2_RECIPROCAL]]
+  %6 = math.log2 %arg0 : f32
+  // CHECK: %[[LOG10_RECIPROCAL:.+]] = spirv.Constant 0.434294492 : f32
+  // CHECK: %[[LOG1:.+]] = spirv.GL.Log {{.+}}
+  // CHECK: spirv.FMul %[[LOG1]], %[[LOG10_RECIPROCAL]]
+  %7 = math.log10 %arg0 : f32
   // CHECK: spirv.GL.RoundEven %{{.*}}: f32
-  %6 = math.roundeven %arg0 : f32
+  %8 = math.roundeven %arg0 : f32
   // CHECK: spirv.GL.InverseSqrt %{{.*}}: f32
-  %7 = math.rsqrt %arg0 : f32
+  %9 = math.rsqrt %arg0 : f32
   // CHECK: spirv.GL.Sqrt %{{.*}}: f32
-  %8 = math.sqrt %arg0 : f32
+  %10 = math.sqrt %arg0 : f32
   // CHECK: spirv.GL.Tanh %{{.*}}: f32
-  %9 = math.tanh %arg0 : f32
+  %11 = math.tanh %arg0 : f32
   // CHECK: spirv.GL.Sin %{{.*}}: f32
-  %10 = math.sin %arg0 : f32
+  %12 = math.sin %arg0 : f32
   // CHECK: spirv.GL.FAbs %{{.*}}: f32
-  %11 = math.absf %arg0 : f32
+  %13 = math.absf %arg0 : f32
   // CHECK: spirv.GL.Ceil %{{.*}}: f32
-  %12 = math.ceil %arg0 : f32
+  %14 = math.ceil %arg0 : f32
   // CHECK: spirv.GL.Floor %{{.*}}: f32
-  %13 = math.floor %arg0 : f32
+  %15 = math.floor %arg0 : f32
   return
 }
 
@@ -59,16 +67,24 @@ func.func @float32_unary_vector(%arg0: vector<3xf32>) {
   // CHECK: %[[ADDONE:.+]] = spirv.FAdd %[[ONE]], %{{.+}}
   // CHECK: spirv.GL.Log %[[ADDONE]]
   %5 = math.log1p %arg0 : vector<3xf32>
+  // CHECK: %[[LOG2_RECIPROCAL:.+]] = spirv.Constant dense<1.44269502> : vector<3xf32>
+  // CHECK: %[[LOG0:.+]] = spirv.GL.Log {{.+}}
+  // CHECK: spirv.FMul %[[LOG0]], %[[LOG2_RECIPROCAL]]
+  %6 = math.log2 %arg0 : vector<3xf32>
+  // CHECK: %[[LOG10_RECIPROCAL:.+]] = spirv.Constant dense<0.434294492> : vector<3xf32>
+  // CHECK: %[[LOG1:.+]] = spirv.GL.Log {{.+}}
+  // CHECK: spirv.FMul %[[LOG1]], %[[LOG10_RECIPROCAL]]
+  %7 = math.log10 %arg0 : vector<3xf32>
   // CHECK: spirv.GL.RoundEven %{{.*}}: vector<3xf32>
-  %6 = math.roundeven %arg0 : vector<3xf32>
+  %8 = math.roundeven %arg0 : vector<3xf32>
   // CHECK: spirv.GL.InverseSqrt %{{.*}}: vector<3xf32>
-  %7 = math.rsqrt %arg0 : vector<3xf32>
+  %9 = math.rsqrt %arg0 : vector<3xf32>
   // CHECK: spirv.GL.Sqrt %{{.*}}: vector<3xf32>
-  %8 = math.sqrt %arg0 : vector<3xf32>
+  %10 = math.sqrt %arg0 : vector<3xf32>
   // CHECK: spirv.GL.Tanh %{{.*}}: vector<3xf32>
-  %9 = math.tanh %arg0 : vector<3xf32>
+  %11 = math.tanh %arg0 : vector<3xf32>
   // CHECK: spirv.GL.Sin %{{.*}}: vector<3xf32>
-  %10 = math.sin %arg0 : vector<3xf32>
+  %12 = math.sin %arg0 : vector<3xf32>
   return
 }
 
diff --git a/mlir/test/Conversion/MathToSPIRV/math-to-opencl-spirv.mlir b/mlir/test/Conversion/MathToSPIRV/math-to-opencl-spirv.mlir
index e9ca838354c0de5..393a910c1fb1d74 100644
--- a/mlir/test/Conversion/MathToSPIRV/math-to-opencl-spirv.mlir
+++ b/mlir/test/Conversion/MathToSPIRV/math-to-opencl-spirv.mlir
@@ -20,26 +20,34 @@ func.func @float32_unary_scalar(%arg0: f32) {
   // CHECK: %[[ADDONE:.+]] = spirv.FAdd %[[ONE]], %{{.+}}
   // CHECK: spirv.CL.log %[[ADDONE]]
   %5 = math.log1p %arg0 : f32
+  // CHECK: %[[LOG2_RECIPROCAL:.+]] = spirv.Constant 1.44269502 : f32
+  // CHECK: %[[LOG0:.+]] = spirv.CL.log {{.+}}
+  // CHECK: spirv.FMul %[[LOG0]], %[[LOG2_RECIPROCAL]]
+  %6 = math.log2 %arg0 : f32
+  // CHECK: %[[LOG10_RECIPROCAL:.+]] = spirv.Constant 0.434294492 : f32
+  // CHECK: %[[LOG1:.+]] = spirv.CL.log {{.+}}
+  // CHECK: spirv.FMul %[[LOG1]], %[[LOG10_RECIPROCAL]]
+  %7 = math.log10 %arg0 : f32
   // CHECK: spirv.CL.rint %{{.*}}: f32
-  %6 = math.roundeven %arg0 : f32
+  %8 = math.roundeven %arg0 : f32
   // CHECK: spirv.CL.rsqrt %{{.*}}: f32
-  %7 = math.rsqrt %arg0 : f32
+  %9 = math.rsqrt %arg0 : f32
   // CHECK: spirv.CL.sqrt %{{.*}}: f32
-  %8 = math.sqrt %arg0 : f32
+  %10 = math.sqrt %arg0 : f32
   // CHECK: spirv.CL.tanh %{{.*}}: f32
-  %9 = math.tanh %arg0 : f32
+  %11 = math.tanh %arg0 : f32
   // CHECK: spirv.CL.sin %{{.*}}: f32
-  %10 = math.sin %arg0 : f32
+  %12 = math.sin %arg0 : f32
   // CHECK: spirv.CL.fabs %{{.*}}: f32
-  %11 = math.absf %arg0 : f32
+  %13 = math.absf %arg0 : f32
   // CHECK: spirv.CL.ceil %{{.*}}: f32
-  %12 = math.ceil %arg0 : f32
+  %14 = math.ceil %arg0 : f32
   // CHECK: spirv.CL.floor %{{.*}}: f32
-  %13 = math.floor %arg0 : f32
+  %15 = math.floor %arg0 : f32
   // CHECK: spirv.CL.erf %{{.*}}: f32
-  %14 = math.erf %arg0 : f32
+  %16 = math.erf %arg0 : f32
   // CHECK: spirv.CL.round %{{.*}}: f32
-  %15 = math.round %arg0 : f32
+  %17 = math.round %arg0 : f32
   return
 }
 
@@ -61,16 +69,24 @@ func.func @float32_unary_vector(%arg0: vector<3xf32>) {
   // CHECK: %[[ADDONE:.+]] = spirv.FAdd %[[ONE]], %{{.+}}
   // CHECK: spirv.CL.log %[[ADDONE]]
   %5 = math.log1p %arg0 : vector<3xf32>
+  // CHECK: %[[LOG2_RECIPROCAL:.+]] = spirv.Constant dense<1.44269502> : vector<3xf32>
+  // CHECK: %[[LOG0:.+]] = spirv.CL.log {{.+}}
+  // CHECK: spirv.FMul %[[LOG0]], %[[LOG2_RECIPROCAL]]
+  %6 = math.log2 %arg0 : vector<3xf32>
+  // CHECK: %[[LOG10_RECIPROCAL:.+]] = spirv.Constant dense<0.434294492> : vector<3xf32>
+  // CHECK: %[[LOG1:.+]] = spirv.CL.log {{.+}}
+  // CHECK: spirv.FMul %[[LOG1]], %[[LOG10_RECIPROCAL]]
+  %7 = math.log10 %arg0 : vector<3xf32>
   // CHECK: spirv.CL.rint %{{.*}}: vector<3xf32>
-  %6 = math.roundeven %arg0 : vector<3xf32>
+  %8 = math.roundeven %arg0 : vector<3xf32>
   // CHECK: spirv.CL.rsqrt %{{.*}}: vector<3xf32>
-  %7 = math.rsqrt %arg0 : vector<3xf32>
+  %9 = math.rsqrt %arg0 : vector<3xf32>
   // CHECK: spirv.CL.sqrt %{{.*}}: vector<3xf32>
-  %8 = math.sqrt %arg0 : vector<3xf32>
+  %10 = math.sqrt %arg0 : vector<3xf32>
   // CHECK: spirv.CL.tanh %{{.*}}: vector<3xf32>
-  %9 = math.tanh %arg0 : vector<3xf32>
+  %11 = math.tanh %arg0 : vector<3xf32>
   // CHECK: spirv.CL.sin %{{.*}}: vector<3xf32>
-  %10 = math.sin %arg0 : vector<3xf32>
+  %12 = math.sin %arg0 : vector<3xf32>
   return
 }
 

From 7a06ebdeb6440d80fbcaeccd33314c6e039c6795 Mon Sep 17 00:00:00 2001
From: Fred Grim <fgrim@apple.com>
Date: Mon, 19 Aug 2024 08:41:11 -0700
Subject: [PATCH 375/441] [lldb] PopulatePrpsInfoTest can fail due to hardcoded
 priority value (#104617)

In implementing this test one of the assertions assumes that the
priority is the default in linux (0) but, evidently, some of the build
runners prioritize the test to, at least, 5. This ensures that
regardless of the priority the test passes by validating that its the
process's priority
```
fgrim@host001 :~/llvm-project/debug_build> nice -n 15 tools/lldb/unittests/Process/elf-core/ProcessElfCoreTests
[==========] Running 2 tests from 1 test suite.
[----------] Global test environment set-up.
[----------] 2 tests from ElfCoreTest
[ RUN      ] ElfCoreTest.PopulatePrpsInfoTest
[       OK ] ElfCoreTest.PopulatePrpsInfoTest (4 ms)
[ RUN      ] ElfCoreTest.PopulatePrStatusTest
[       OK ] ElfCoreTest.PopulatePrStatusTest (3 ms)
[----------] 2 tests from ElfCoreTest (7 ms total)

[----------] Global test environment tear-down
[==========] 2 tests from 1 test suite ran. (8 ms total)
[  PASSED  ] 2 tests.
===(10:03)===
fgrim@host001 :~/llvm-project/debug_build>
```
---
 lldb/unittests/Process/elf-core/ThreadElfCoreTest.cpp | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/lldb/unittests/Process/elf-core/ThreadElfCoreTest.cpp b/lldb/unittests/Process/elf-core/ThreadElfCoreTest.cpp
index ce146f62b0d8264..3bc8b9053d20091 100644
--- a/lldb/unittests/Process/elf-core/ThreadElfCoreTest.cpp
+++ b/lldb/unittests/Process/elf-core/ThreadElfCoreTest.cpp
@@ -21,6 +21,7 @@
 
 #include <memory>
 #include <mutex>
+#include <sys/resource.h>
 #include <unistd.h>
 
 using namespace lldb_private;
@@ -120,7 +121,10 @@ TEST_F(ElfCoreTest, PopulatePrpsInfoTest) {
   ASSERT_EQ(prpsinfo_opt->pr_state, 0);
   ASSERT_EQ(prpsinfo_opt->pr_sname, 'R');
   ASSERT_EQ(prpsinfo_opt->pr_zomb, 0);
-  ASSERT_EQ(prpsinfo_opt->pr_nice, 0);
+  int priority = getpriority(PRIO_PROCESS, getpid());
+  if (priority == -1)
+    ASSERT_EQ(errno, 0);
+  ASSERT_EQ(prpsinfo_opt->pr_nice, priority);
   ASSERT_EQ(prpsinfo_opt->pr_flag, 0UL);
   ASSERT_EQ(prpsinfo_opt->pr_uid, getuid());
   ASSERT_EQ(prpsinfo_opt->pr_gid, getgid());

From d27278a835ba9110b5bf775852b1a407d1c2bc62 Mon Sep 17 00:00:00 2001
From: Timm Baeder <tbaeder@redhat.com>
Date: Mon, 19 Aug 2024 17:51:27 +0200
Subject: [PATCH 376/441] [clang][bytecode] Discard NullToPointer cast SubExpr
 (#104782)

So we still properly abort if they fail.
---
 clang/lib/AST/ByteCode/Compiler.cpp | 2 ++
 clang/test/AST/ByteCode/codegen.cpp | 9 +++++++++
 2 files changed, 11 insertions(+)

diff --git a/clang/lib/AST/ByteCode/Compiler.cpp b/clang/lib/AST/ByteCode/Compiler.cpp
index 53144a50ccf4a92..c839364d4c8d25c 100644
--- a/clang/lib/AST/ByteCode/Compiler.cpp
+++ b/clang/lib/AST/ByteCode/Compiler.cpp
@@ -327,6 +327,8 @@ bool Compiler<Emitter>::VisitCastExpr(const CastExpr *CE) {
 
   case CK_NullToPointer:
   case CK_NullToMemberPointer: {
+    if (!this->discard(SubExpr))
+      return false;
     if (DiscardResult)
       return true;
 
diff --git a/clang/test/AST/ByteCode/codegen.cpp b/clang/test/AST/ByteCode/codegen.cpp
index 920a5ad7ba3e8fc..9fac28a52d3150d 100644
--- a/clang/test/AST/ByteCode/codegen.cpp
+++ b/clang/test/AST/ByteCode/codegen.cpp
@@ -64,3 +64,12 @@ namespace Bitfield {
   // CHECK: store ptr @_ZGRN8Bitfield1rE_, ptr @_ZN8Bitfield1rE, align 8
   int &&r = S().a;
 }
+
+namespace Null {
+  decltype(nullptr) null();
+  // CHECK: call {{.*}} @_ZN4Null4nullEv(
+  int *p = null();
+  struct S {};
+  // CHECK: call {{.*}} @_ZN4Null4nullEv(
+  int S::*q = null();
+}

From fade54ac6608255e9b3c425432871f177236d6e8 Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Mon, 19 Aug 2024 15:53:47 +0000
Subject: [PATCH 377/441] [gn build] Port a449b857241d

---
 llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn | 1 -
 1 file changed, 1 deletion(-)

diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
index 9fb898445ed6fa6..a2b1e5a3f850bcd 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
@@ -209,7 +209,6 @@ static_library("LLVMAMDGPUCodeGen") {
     "GCNVOPDUtils.cpp",
     "R600AsmPrinter.cpp",
     "R600ClauseMergePass.cpp",
-    "R600CodeGenPassBuilder.cpp",
     "R600ControlFlowFinalizer.cpp",
     "R600EmitClauseMarkers.cpp",
     "R600ExpandSpecialInstrs.cpp",

From c6d6186b586f4249e393910b099815e616138cde Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Mon, 19 Aug 2024 15:53:48 +0000
Subject: [PATCH 378/441] [gn build] Port a56663591573

---
 llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn | 1 -
 1 file changed, 1 deletion(-)

diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
index a2b1e5a3f850bcd..c9b1b4d9ac31359 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
@@ -139,7 +139,6 @@ static_library("LLVMAMDGPUCodeGen") {
     "AMDGPUAtomicOptimizer.cpp",
     "AMDGPUAttributor.cpp",
     "AMDGPUCallLowering.cpp",
-    "AMDGPUCodeGenPassBuilder.cpp",
     "AMDGPUCodeGenPrepare.cpp",
     "AMDGPUCombinerHelper.cpp",
     "AMDGPUCtorDtorLowering.cpp",

From 0ee0857363aadf9ce0f403e7e0da10f0a9d94887 Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett@linaro.org>
Date: Mon, 19 Aug 2024 17:02:36 +0100
Subject: [PATCH 379/441] [lldb][Python] Silence GCC warning for modules error
 workaround
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

```
lldb-python.h:16:30: warning: ‘g_fcxx_modules_workaround’ defined but not used [-Wunused-variable]
   16 | static llvm::Expected<bool> *g_fcxx_modules_workaround;
      |
```

Workaround originally added in 36cb29cbbe1b22dcd298ad65e1fabe899b7d7249.
---
 lldb/source/Plugins/ScriptInterpreter/Python/lldb-python.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lldb/source/Plugins/ScriptInterpreter/Python/lldb-python.h b/lldb/source/Plugins/ScriptInterpreter/Python/lldb-python.h
index c99372fa110cd22..378b9fa2a598651 100644
--- a/lldb/source/Plugins/ScriptInterpreter/Python/lldb-python.h
+++ b/lldb/source/Plugins/ScriptInterpreter/Python/lldb-python.h
@@ -13,7 +13,7 @@
 // This declaration works around a clang module build failure.
 // It should be deleted ASAP.
 #include "llvm/Support/Error.h"
-static llvm::Expected<bool> *g_fcxx_modules_workaround;
+static llvm::Expected<bool> *g_fcxx_modules_workaround [[maybe_unused]];
 // END
 
 #include "lldb/Host/Config.h"

From 42386dc46d8217b2dfed8dfc41afeec366ec2015 Mon Sep 17 00:00:00 2001
From: Phil Camp <phil.camp@sony.com>
Date: Mon, 19 Aug 2024 17:16:19 +0100
Subject: [PATCH 380/441] [llvm-mca] Add bottle-neck analysis to JSON output.
 (#90056)

This patch implements the bottle-neck analysis data in the JSON dump
mode.
---
 .../llvm-mca/JSON/X86/views-bottleneck.s      | 73 +++++++++++++++++++
 .../JSON/X86/views-custom-parameters.s        |  3 +
 .../X86/views-multiple-anonymous-regions.s    | 69 ++++++++++++++++++
 .../llvm-mca/JSON/X86/views-multiple-region.s | 26 +++++++
 llvm/test/tools/llvm-mca/JSON/X86/views.s     |  3 +
 .../llvm-mca/Views/BottleneckAnalysis.cpp     | 46 ++++++++++++
 .../tools/llvm-mca/Views/BottleneckAnalysis.h |  3 +-
 7 files changed, 222 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/tools/llvm-mca/JSON/X86/views-bottleneck.s

diff --git a/llvm/test/tools/llvm-mca/JSON/X86/views-bottleneck.s b/llvm/test/tools/llvm-mca/JSON/X86/views-bottleneck.s
new file mode 100644
index 000000000000000..c8ad60ee542093c
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/JSON/X86/views-bottleneck.s
@@ -0,0 +1,73 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=haswell --json --timeline-max-iterations=1 --bottleneck-analysis --resource-pressure=false --instruction-info=false < %s | FileCheck %s
+vaddps %xmm0, %xmm0, %xmm1
+vblendvps %xmm1, (%rdi), %xmm2, %xmm3
+
+# CHECK:      {
+# CHECK-NEXT:   "CodeRegions": [
+# CHECK-NEXT:     {
+# CHECK-NEXT:       "BottleneckAnalysis": {
+# CHECK-NEXT:         "DataDependencyCycles": 0,
+# CHECK-NEXT:         "DependencyEdge": [
+# CHECK-NEXT:           {
+# CHECK-NEXT:             "FromID": 1,
+# CHECK-NEXT:             "ResourceOrRegID": 128,
+# CHECK-NEXT:             "ToID": 3,
+# CHECK-NEXT:             "Type": 3
+# CHECK-NEXT:           },
+# CHECK-NEXT:           {
+# CHECK-NEXT:             "FromID": 3,
+# CHECK-NEXT:             "ResourceOrRegID": 128,
+# CHECK-NEXT:             "ToID": 5,
+# CHECK-NEXT:             "Type": 3
+# CHECK-NEXT:           }
+# CHECK-NEXT:         ],
+# CHECK-NEXT:         "MemoryDependencyCycles": 0,
+# CHECK-NEXT:         "PressureIncreaseCycles": 56,
+# CHECK-NEXT:         "RegisterDependencyCycles": 0,
+# CHECK-NEXT:         "ResourcePressure": [
+# CHECK-NEXT:           {
+# CHECK-NEXT:             "HWPort5": 56
+# CHECK-NEXT:           }
+# CHECK-NEXT:         ],
+# CHECK-NEXT:         "ResourcePressureCycles": 56,
+# CHECK-NEXT:         "TotalCycles": 209
+# CHECK-NEXT:       },
+# CHECK-NEXT:       "Instructions": [
+# CHECK-NEXT:         "vaddps\t%xmm0, %xmm0, %xmm1",
+# CHECK-NEXT:         "vblendvps\t%xmm1, (%rdi), %xmm2, %xmm3"
+# CHECK-NEXT:       ],
+# CHECK-NEXT:       "Name": "",
+# CHECK-NEXT:       "SummaryView": {
+# CHECK-NEXT:         "BlockRThroughput": 2,
+# CHECK-NEXT:         "DispatchWidth": 4,
+# CHECK-NEXT:         "IPC": 0.9569377990430622,
+# CHECK-NEXT:         "Instructions": 200,
+# CHECK-NEXT:         "Iterations": 100,
+# CHECK-NEXT:         "TotalCycles": 209,
+# CHECK-NEXT:         "TotaluOps": 400,
+# CHECK-NEXT:         "uOpsPerCycle": 1.9138755980861244
+# CHECK-NEXT:       }
+# CHECK-NEXT:     }
+# CHECK-NEXT:   ],
+# CHECK-NEXT:   "SimulationParameters": {
+# CHECK-NEXT:     "-march": "x86_64",
+# CHECK-NEXT:     "-mcpu": "haswell",
+# CHECK-NEXT:     "-mtriple": "x86_64-unknown-unknown"
+# CHECK-NEXT:   },
+# CHECK-NEXT:   "TargetInfo": {
+# CHECK-NEXT:     "CPUName": "haswell",
+# CHECK-NEXT:     "Resources": [
+# CHECK-NEXT:       "HWDivider",
+# CHECK-NEXT:       "HWFPDivider",
+# CHECK-NEXT:       "HWPort0",
+# CHECK-NEXT:       "HWPort1",
+# CHECK-NEXT:       "HWPort2",
+# CHECK-NEXT:       "HWPort3",
+# CHECK-NEXT:       "HWPort4",
+# CHECK-NEXT:       "HWPort5",
+# CHECK-NEXT:       "HWPort6",
+# CHECK-NEXT:       "HWPort7"
+# CHECK-NEXT:     ]
+# CHECK-NEXT:   }
+# CHECK-NEXT: }
diff --git a/llvm/test/tools/llvm-mca/JSON/X86/views-custom-parameters.s b/llvm/test/tools/llvm-mca/JSON/X86/views-custom-parameters.s
index 1301c4acda8321f..538175f6ac6155e 100644
--- a/llvm/test/tools/llvm-mca/JSON/X86/views-custom-parameters.s
+++ b/llvm/test/tools/llvm-mca/JSON/X86/views-custom-parameters.s
@@ -16,6 +16,9 @@ add %edx, %edx
 # CHECK:      {
 # CHECK-NEXT:   "CodeRegions": [
 # CHECK-NEXT:     {
+# CHECK-NEXT:       "BottleneckAnalysis": {
+# CHECK-NEXT:         "PressureIncreaseCycles": 0
+# CHECK-NEXT:       },
 # CHECK-NEXT:       "DispatchStatistics": {
 # CHECK-NEXT:         "GROUP": 0,
 # CHECK-NEXT:         "LQ": 0,
diff --git a/llvm/test/tools/llvm-mca/JSON/X86/views-multiple-anonymous-regions.s b/llvm/test/tools/llvm-mca/JSON/X86/views-multiple-anonymous-regions.s
index 8cd8d41a6ff65f3..ea9dbb632d7c1ca 100644
--- a/llvm/test/tools/llvm-mca/JSON/X86/views-multiple-anonymous-regions.s
+++ b/llvm/test/tools/llvm-mca/JSON/X86/views-multiple-anonymous-regions.s
@@ -19,6 +19,29 @@ add %edx, %edx
 # CHECK:      {
 # CHECK-NEXT:   "CodeRegions": [
 # CHECK-NEXT:     {
+# CHECK-NEXT:       "BottleneckAnalysis": {
+# CHECK-NEXT:         "DataDependencyCycles": 39,
+# CHECK-NEXT:         "DependencyEdge": [
+# CHECK-NEXT:           {
+# CHECK-NEXT:             "FromID": 0,
+# CHECK-NEXT:             "ResourceOrRegID": 22,
+# CHECK-NEXT:             "ToID": 1,
+# CHECK-NEXT:             "Type": 1
+# CHECK-NEXT:           },
+# CHECK-NEXT:           {
+# CHECK-NEXT:             "FromID": 1,
+# CHECK-NEXT:             "ResourceOrRegID": 22,
+# CHECK-NEXT:             "ToID": 2,
+# CHECK-NEXT:             "Type": 1
+# CHECK-NEXT:           }
+# CHECK-NEXT:         ],
+# CHECK-NEXT:         "MemoryDependencyCycles": 0,
+# CHECK-NEXT:         "PressureIncreaseCycles": 39,
+# CHECK-NEXT:         "RegisterDependencyCycles": 39,
+# CHECK-NEXT:         "ResourcePressure": [],
+# CHECK-NEXT:         "ResourcePressureCycles": 0,
+# CHECK-NEXT:         "TotalCycles": 103
+# CHECK-NEXT:       },
 # CHECK-NEXT:       "DispatchStatistics": {
 # CHECK-NEXT:         "GROUP": 0,
 # CHECK-NEXT:         "LQ": 0,
@@ -175,6 +198,29 @@ add %edx, %edx
 # CHECK-NEXT:       }
 # CHECK-NEXT:     },
 # CHECK-NEXT:     {
+# CHECK-NEXT:       "BottleneckAnalysis": {
+# CHECK-NEXT:         "DataDependencyCycles": 69,
+# CHECK-NEXT:         "DependencyEdge": [
+# CHECK-NEXT:           {
+# CHECK-NEXT:             "FromID": 0,
+# CHECK-NEXT:             "ResourceOrRegID": 24,
+# CHECK-NEXT:             "ToID": 2,
+# CHECK-NEXT:             "Type": 1
+# CHECK-NEXT:           },
+# CHECK-NEXT:           {
+# CHECK-NEXT:             "FromID": 2,
+# CHECK-NEXT:             "ResourceOrRegID": 24,
+# CHECK-NEXT:             "ToID": 4,
+# CHECK-NEXT:             "Type": 1
+# CHECK-NEXT:           }
+# CHECK-NEXT:         ],
+# CHECK-NEXT:         "MemoryDependencyCycles": 0,
+# CHECK-NEXT:         "PressureIncreaseCycles": 69,
+# CHECK-NEXT:         "RegisterDependencyCycles": 69,
+# CHECK-NEXT:         "ResourcePressure": [],
+# CHECK-NEXT:         "ResourcePressureCycles": 0,
+# CHECK-NEXT:         "TotalCycles": 103
+# CHECK-NEXT:       },
 # CHECK-NEXT:       "DispatchStatistics": {
 # CHECK-NEXT:         "GROUP": 0,
 # CHECK-NEXT:         "LQ": 0,
@@ -411,6 +457,29 @@ add %edx, %edx
 # CHECK-NEXT:       }
 # CHECK-NEXT:     },
 # CHECK-NEXT:     {
+# CHECK-NEXT:       "BottleneckAnalysis": {
+# CHECK-NEXT:         "DataDependencyCycles": 39,
+# CHECK-NEXT:         "DependencyEdge": [
+# CHECK-NEXT:           {
+# CHECK-NEXT:             "FromID": 0,
+# CHECK-NEXT:             "ResourceOrRegID": 27,
+# CHECK-NEXT:             "ToID": 1,
+# CHECK-NEXT:             "Type": 1
+# CHECK-NEXT:           },
+# CHECK-NEXT:           {
+# CHECK-NEXT:             "FromID": 1,
+# CHECK-NEXT:             "ResourceOrRegID": 27,
+# CHECK-NEXT:             "ToID": 2,
+# CHECK-NEXT:             "Type": 1
+# CHECK-NEXT:           }
+# CHECK-NEXT:         ],
+# CHECK-NEXT:         "MemoryDependencyCycles": 0,
+# CHECK-NEXT:         "PressureIncreaseCycles": 39,
+# CHECK-NEXT:         "RegisterDependencyCycles": 39,
+# CHECK-NEXT:         "ResourcePressure": [],
+# CHECK-NEXT:         "ResourcePressureCycles": 0,
+# CHECK-NEXT:         "TotalCycles": 103
+# CHECK-NEXT:       },
 # CHECK-NEXT:       "DispatchStatistics": {
 # CHECK-NEXT:         "GROUP": 0,
 # CHECK-NEXT:         "LQ": 0,
diff --git a/llvm/test/tools/llvm-mca/JSON/X86/views-multiple-region.s b/llvm/test/tools/llvm-mca/JSON/X86/views-multiple-region.s
index 0b7bb16beeb9169..a000e9b7d21757b 100644
--- a/llvm/test/tools/llvm-mca/JSON/X86/views-multiple-region.s
+++ b/llvm/test/tools/llvm-mca/JSON/X86/views-multiple-region.s
@@ -20,6 +20,9 @@ add %edx, %edx
 # CHECK:      {
 # CHECK-NEXT:   "CodeRegions": [
 # CHECK-NEXT:     {
+# CHECK-NEXT:       "BottleneckAnalysis": {
+# CHECK-NEXT:         "PressureIncreaseCycles": 0
+# CHECK-NEXT:       },
 # CHECK-NEXT:       "DispatchStatistics": {
 # CHECK-NEXT:         "GROUP": 0,
 # CHECK-NEXT:         "LQ": 0,
@@ -164,6 +167,29 @@ add %edx, %edx
 # CHECK-NEXT:       }
 # CHECK-NEXT:     },
 # CHECK-NEXT:     {
+# CHECK-NEXT:       "BottleneckAnalysis": {
+# CHECK-NEXT:         "DataDependencyCycles": 69,
+# CHECK-NEXT:         "DependencyEdge": [
+# CHECK-NEXT:           {
+# CHECK-NEXT:             "FromID": 0,
+# CHECK-NEXT:             "ResourceOrRegID": 24,
+# CHECK-NEXT:             "ToID": 2,
+# CHECK-NEXT:             "Type": 1
+# CHECK-NEXT:           },
+# CHECK-NEXT:           {
+# CHECK-NEXT:             "FromID": 2,
+# CHECK-NEXT:             "ResourceOrRegID": 24,
+# CHECK-NEXT:             "ToID": 4,
+# CHECK-NEXT:             "Type": 1
+# CHECK-NEXT:           }
+# CHECK-NEXT:         ],
+# CHECK-NEXT:         "MemoryDependencyCycles": 0,
+# CHECK-NEXT:         "PressureIncreaseCycles": 69,
+# CHECK-NEXT:         "RegisterDependencyCycles": 69,
+# CHECK-NEXT:         "ResourcePressure": [],
+# CHECK-NEXT:         "ResourcePressureCycles": 0,
+# CHECK-NEXT:         "TotalCycles": 103
+# CHECK-NEXT:       },
 # CHECK-NEXT:       "DispatchStatistics": {
 # CHECK-NEXT:         "GROUP": 0,
 # CHECK-NEXT:         "LQ": 0,
diff --git a/llvm/test/tools/llvm-mca/JSON/X86/views.s b/llvm/test/tools/llvm-mca/JSON/X86/views.s
index df85f2203f314bc..324d7b8efa07a2b 100644
--- a/llvm/test/tools/llvm-mca/JSON/X86/views.s
+++ b/llvm/test/tools/llvm-mca/JSON/X86/views.s
@@ -16,6 +16,9 @@ add %edx, %edx
 # CHECK:      {
 # CHECK-NEXT:   "CodeRegions": [
 # CHECK-NEXT:     {
+# CHECK-NEXT:       "BottleneckAnalysis": {
+# CHECK-NEXT:         "PressureIncreaseCycles": 0
+# CHECK-NEXT:       },
 # CHECK-NEXT:       "DispatchStatistics": {
 # CHECK-NEXT:         "GROUP": 0,
 # CHECK-NEXT:         "LQ": 0,
diff --git a/llvm/tools/llvm-mca/Views/BottleneckAnalysis.cpp b/llvm/tools/llvm-mca/Views/BottleneckAnalysis.cpp
index 409a7010b80c8d6..ece5b6709a2e4c7 100644
--- a/llvm/tools/llvm-mca/Views/BottleneckAnalysis.cpp
+++ b/llvm/tools/llvm-mca/Views/BottleneckAnalysis.cpp
@@ -641,5 +641,51 @@ void BottleneckAnalysis::printView(raw_ostream &OS) const {
   printCriticalSequence(OS);
 }
 
+json::Value BottleneckAnalysis::toJSON() const {
+  if (!SeenStallCycles || !BPI.PressureIncreaseCycles) {
+    json::Object JO({{"PressureIncreaseCycles", 0}});
+    return JO;
+  }
+
+  json::Array CriticalSequence;
+  // get critical sequence
+  SmallVector<const DependencyEdge *, 16> Seq;
+  DG.getCriticalSequence(Seq);
+  if (!Seq.empty()) {
+    for (const DependencyEdge *&DE : Seq) {
+      json::Object DEJO({{"FromID", DE->FromIID},
+                         {"ToID", DE->ToIID},
+                         {"Type", static_cast<unsigned>(DE->Dep.Type)},
+                         {"ResourceOrRegID", DE->Dep.ResourceOrRegID}});
+      CriticalSequence.push_back(std::move(DEJO));
+    }
+  }
+
+  json::Array ResourcePressure;
+  if (BPI.PressureIncreaseCycles) {
+    ArrayRef<unsigned> Distribution = Tracker.getResourcePressureDistribution();
+    const MCSchedModel &SM = getSubTargetInfo().getSchedModel();
+    for (unsigned I = 0, E = Distribution.size(); I < E; ++I) {
+      unsigned ReleaseAtCycles = Distribution[I];
+      if (ReleaseAtCycles) {
+        const MCProcResourceDesc &PRDesc = *SM.getProcResource(I);
+        json::Object RPJO({{PRDesc.Name, ReleaseAtCycles}});
+        ResourcePressure.push_back(std::move(RPJO));
+      }
+    }
+  }
+
+  json::Object JO({{"PressureIncreaseCycles", BPI.PressureIncreaseCycles},
+                   {"ResourcePressureCycles", BPI.ResourcePressureCycles},
+                   {"DataDependencyCycles", BPI.DataDependencyCycles},
+                   {"RegisterDependencyCycles", BPI.RegisterDependencyCycles},
+                   {"MemoryDependencyCycles", BPI.MemoryDependencyCycles},
+                   {"TotalCycles", TotalCycles},
+                   {"DependencyEdge", std::move(CriticalSequence)},
+                   {"ResourcePressure", std::move(ResourcePressure)}});
+
+  return JO;
+}
+
 } // namespace mca.
 } // namespace llvm
diff --git a/llvm/tools/llvm-mca/Views/BottleneckAnalysis.h b/llvm/tools/llvm-mca/Views/BottleneckAnalysis.h
index e709b25c3f76803..529090cf543fc41 100644
--- a/llvm/tools/llvm-mca/Views/BottleneckAnalysis.h
+++ b/llvm/tools/llvm-mca/Views/BottleneckAnalysis.h
@@ -335,7 +335,8 @@ class BottleneckAnalysis : public InstructionView {
 
   void printView(raw_ostream &OS) const override;
   StringRef getNameAsString() const override { return "BottleneckAnalysis"; }
-  bool isSerializable() const override { return false; }
+  bool isSerializable() const override { return true; }
+  json::Value toJSON() const override;
 
 #ifndef NDEBUG
   void dump(raw_ostream &OS, MCInstPrinter &MCIP) const { DG.dump(OS, MCIP); }

From f95026dbf66e353128a3a3d7b55f3e52d5985535 Mon Sep 17 00:00:00 2001
From: Lei Huang <lei@ca.ibm.com>
Date: Mon, 19 Aug 2024 12:17:44 -0400
Subject: [PATCH 381/441] [PowerPC] Fix codegen for transparent_union function
 params (#101738)

Update codegen for func param with transparent_union attr to be that of
the first union member.

PPC fix for: https://github.com/llvm/llvm-project/issues/76773
---
 clang/lib/CodeGen/ABIInfoImpl.cpp             |  5 +-
 clang/lib/CodeGen/Targets/PPC.cpp             | 10 +--
 .../test/CodeGen/PowerPC/transparent_union.c  | 67 +++++++++++++++++++
 3 files changed, 76 insertions(+), 6 deletions(-)
 create mode 100644 clang/test/CodeGen/PowerPC/transparent_union.c

diff --git a/clang/lib/CodeGen/ABIInfoImpl.cpp b/clang/lib/CodeGen/ABIInfoImpl.cpp
index 35e8f79ba1bac7b..be91b85e3a816f8 100644
--- a/clang/lib/CodeGen/ABIInfoImpl.cpp
+++ b/clang/lib/CodeGen/ABIInfoImpl.cpp
@@ -38,8 +38,9 @@ ABIArgInfo DefaultABIInfo::classifyArgumentType(QualType Ty) const {
                                 : Context.LongLongTy))
       return getNaturalAlignIndirect(Ty);
 
-  return (isPromotableIntegerTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty)
-                                            : ABIArgInfo::getDirect());
+  return (isPromotableIntegerTypeForABI(Ty)
+              ? ABIArgInfo::getExtend(Ty, CGT.ConvertType(Ty))
+              : ABIArgInfo::getDirect());
 }
 
 ABIArgInfo DefaultABIInfo::classifyReturnType(QualType RetTy) const {
diff --git a/clang/lib/CodeGen/Targets/PPC.cpp b/clang/lib/CodeGen/Targets/PPC.cpp
index e4155810963eb81..989e46f4b66a7d6 100644
--- a/clang/lib/CodeGen/Targets/PPC.cpp
+++ b/clang/lib/CodeGen/Targets/PPC.cpp
@@ -217,8 +217,9 @@ ABIArgInfo AIXABIInfo::classifyArgumentType(QualType Ty) const {
                                    /*Realign*/ TyAlign > CCAlign);
   }
 
-  return (isPromotableTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty)
-                                     : ABIArgInfo::getDirect());
+  return (isPromotableTypeForABI(Ty)
+              ? ABIArgInfo::getExtend(Ty, CGT.ConvertType(Ty))
+              : ABIArgInfo::getDirect());
 }
 
 CharUnits AIXABIInfo::getParamTypeAlignment(QualType Ty) const {
@@ -891,8 +892,9 @@ PPC64_SVR4_ABIInfo::classifyArgumentType(QualType Ty) const {
                                    /*Realign=*/TyAlign > ABIAlign);
   }
 
-  return (isPromotableTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty)
-                                     : ABIArgInfo::getDirect());
+  return (isPromotableTypeForABI(Ty)
+              ? ABIArgInfo::getExtend(Ty, CGT.ConvertType(Ty))
+              : ABIArgInfo::getDirect());
 }
 
 ABIArgInfo
diff --git a/clang/test/CodeGen/PowerPC/transparent_union.c b/clang/test/CodeGen/PowerPC/transparent_union.c
new file mode 100644
index 000000000000000..968a385c0ee45fe
--- /dev/null
+++ b/clang/test/CodeGen/PowerPC/transparent_union.c
@@ -0,0 +1,67 @@
+// RUN: %clang_cc1 -triple powerpc64le-unknown-linux -O2 -target-cpu pwr7 \
+// RUN:   -emit-llvm -fshort-enums %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-64
+// RUN: %clang_cc1 -triple powerpc64-unknown-linux -O2 -target-cpu pwr7 \
+// RUN:   -emit-llvm -fshort-enums %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-64
+// RUN: %clang_cc1 -triple powerpc-unknown-linux -O2 -target-cpu pwr7 \
+// RUN:   -emit-llvm -fshort-enums %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-32
+// RUN: %clang_cc1 -triple powerpc64-unknown-aix -O2 -target-cpu pwr7 \
+// RUN:   -emit-llvm -fshort-enums %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-64
+// RUN: %clang_cc1 -triple powerpc-unknown-aix -O2 -target-cpu pwr7 \
+// RUN:   -emit-llvm -fshort-enums %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-32
+
+typedef union tu_c {
+  signed char a;
+  signed char b;
+} tu_c_t __attribute__((transparent_union));
+
+typedef union tu_s {
+  short a;
+} tu_s_t __attribute__((transparent_union));
+
+typedef union tu_us {
+  unsigned short a;
+} tu_us_t __attribute__((transparent_union));
+
+typedef union tu_l {
+  long a;
+} tu_l_t __attribute__((transparent_union));
+
+// CHECK-LABEL: define{{.*}} void @ftest0(
+// CHECK-SAME: i8 noundef signext [[UC_COERCE:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret void
+void ftest0(tu_c_t uc) { }
+
+// CHECK-LABEL: define{{.*}} void @ftest1(
+// CHECK-SAME: i16 noundef signext [[UC_COERCE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret void
+void ftest1(tu_s_t uc) { }
+
+// CHECK-LABEL: define{{.*}} void @ftest2(
+// CHECK-SAME: i16 noundef zeroext [[UC_COERCE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret void
+void ftest2(tu_us_t uc) { }
+
+// CHECK-64-LABEL: define{{.*}} void @ftest3(
+// CHECK-64-SAME: i64 [[UC_COERCE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-64-NEXT:  [[ENTRY:.*:]]
+// CHECK-64-NEXT:    ret void
+//
+// CHECK-32-LABEL: define{{.*}} void @ftest3(
+// CHECK-32-SAME: i32 [[UC_COERCE:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-32-NEXT:  [[ENTRY:.*:]]
+// CHECK-32-NEXT:    ret void
+void ftest3(tu_l_t uc) { }
+
+typedef union etest {
+  enum flag {red, yellow, blue} fl;
+  enum weekend {sun, sat} b;
+} etest_t __attribute__((transparent_union));
+
+// CHECK-LABEL: define{{.*}} void @ftest4(
+// CHECK-SAME: i8 noundef zeroext [[A_COERCE:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    ret void
+void ftest4(etest_t a) {}

From a3fea0643da544b2e5d5c48f2ea942dad22d01f2 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Mon, 19 Aug 2024 09:42:14 -0700
Subject: [PATCH 382/441] [SmallPtrSet] Optimize find/erase

Port #100517 for DenseMap.

Pull Request: https://github.com/llvm/llvm-project/pull/104740
---
 llvm/include/llvm/ADT/SmallPtrSet.h |  8 ++++----
 llvm/lib/Support/SmallPtrSet.cpp    | 20 +++++++++++++++++++-
 2 files changed, 23 insertions(+), 5 deletions(-)

diff --git a/llvm/include/llvm/ADT/SmallPtrSet.h b/llvm/include/llvm/ADT/SmallPtrSet.h
index e668df08c3e6363..1fc2318342ae784 100644
--- a/llvm/include/llvm/ADT/SmallPtrSet.h
+++ b/llvm/include/llvm/ADT/SmallPtrSet.h
@@ -185,8 +185,8 @@ class SmallPtrSetImplBase : public DebugEpochBase {
       return false;
     }
 
-    auto *Bucket = FindBucketFor(Ptr);
-    if (*Bucket != Ptr)
+    auto *Bucket = doFind(Ptr);
+    if (!Bucket)
       return false;
 
     *const_cast<const void **>(Bucket) = getTombstoneMarker();
@@ -211,8 +211,7 @@ class SmallPtrSetImplBase : public DebugEpochBase {
     }
 
     // Big set case.
-    auto *Bucket = FindBucketFor(Ptr);
-    if (*Bucket == Ptr)
+    if (auto *Bucket = doFind(Ptr))
       return Bucket;
     return EndPointer();
   }
@@ -222,6 +221,7 @@ class SmallPtrSetImplBase : public DebugEpochBase {
 private:
   std::pair<const void *const *, bool> insert_imp_big(const void *Ptr);
 
+  const void *const *doFind(const void *Ptr) const;
   const void * const *FindBucketFor(const void *Ptr) const;
   void shrink_and_clear();
 
diff --git a/llvm/lib/Support/SmallPtrSet.cpp b/llvm/lib/Support/SmallPtrSet.cpp
index cbb87ea8717cfc2..1bec911f39b13e8 100644
--- a/llvm/lib/Support/SmallPtrSet.cpp
+++ b/llvm/lib/Support/SmallPtrSet.cpp
@@ -62,7 +62,25 @@ SmallPtrSetImplBase::insert_imp_big(const void *Ptr) {
   return std::make_pair(Bucket, true);
 }
 
-const void * const *SmallPtrSetImplBase::FindBucketFor(const void *Ptr) const {
+const void *const *SmallPtrSetImplBase::doFind(const void *Ptr) const {
+  unsigned BucketNo =
+      DenseMapInfo<void *>::getHashValue(Ptr) & (CurArraySize - 1);
+  unsigned ProbeAmt = 1;
+  while (true) {
+    const void *const *Bucket = CurArray + BucketNo;
+    if (LLVM_LIKELY(*Bucket == Ptr))
+      return Bucket;
+    if (LLVM_LIKELY(*Bucket == getEmptyMarker()))
+      return nullptr;
+
+    // Otherwise, it's a hash collision or a tombstone, continue quadratic
+    // probing.
+    BucketNo += ProbeAmt++;
+    BucketNo &= CurArraySize - 1;
+  }
+}
+
+const void *const *SmallPtrSetImplBase::FindBucketFor(const void *Ptr) const {
   unsigned Bucket = DenseMapInfo<void *>::getHashValue(Ptr) & (CurArraySize-1);
   unsigned ArraySize = CurArraySize;
   unsigned ProbeAmt = 1;

From e9b2e16dc98345bb1b91b1a6dacb3cec85f49e31 Mon Sep 17 00:00:00 2001
From: eddyz87 <eddyz87@gmail.com>
Date: Mon, 19 Aug 2024 19:49:11 +0300
Subject: [PATCH 383/441] [BPF] introduce `__attribute__((bpf_fastcall))`
 (#101228)

This commit introduces attribute bpf_fastcall to declare BPF functions
that do not clobber some of the caller saved registers (R0-R5).

The idea is to generate the code complying with generic BPF ABI, but
allow compatible Linux Kernel to remove unnecessary spills and fills of
non-scratched registers (given some compiler assistance).

For such functions do register allocation as-if caller saved registers
are not clobbered, but later wrap the calls with spill and fill patterns
that are simple to recognize in kernel.

For example for the following C code:

     #define __bpf_fastcall __attribute__((bpf_fastcall))

     void bar(void) __bpf_fastcall;
     void buz(long i, long j, long k);

     void foo(long i, long j, long k) {
       bar();
       buz(i, j, k);
     }

First allocate registers as if:

    foo:
      call bar    # note: no spills for i,j,k (r1,r2,r3)
      call buz
      exit

And later insert spills fills on the peephole phase:

    foo:
      *(u64 *)(r10 - 8) = r1;  # Such call pattern is
      *(u64 *)(r10 - 16) = r2; # correct when used with
      *(u64 *)(r10 - 24) = r3; # old kernels.
      call bar
      r3 = *(u64 *)(r10 - 24); # But also allows new
      r2 = *(u64 *)(r10 - 16); # kernels to recognize the
      r1 = *(u64 *)(r10 - 8);  # pattern and remove spills/fills.
      call buz
      exit

The offsets for generated spills/fills are picked as minimal stack
offsets for the function. Allocated stack slots are not used for any
other purposes, in order to simplify in-kernel analysis.

Corresponding functionality had been merged in Linux Kernel as
[this](https://lore.kernel.org/bpf/172179364482.1919.9590705031832457529.git-patchwork-notify@kernel.org/)
patch set (the patch assumed that `no_caller_saved_regsiters` attribute
would be used by LLVM, naming does not matter for the Kernel).
---
 clang/include/clang/Basic/Attr.td             |   9 ++
 clang/include/clang/Basic/AttrDocs.td         |  19 +++
 clang/lib/CodeGen/CGCall.cpp                  |   2 +
 clang/test/CodeGen/bpf-attr-bpf-fastcall-1.c  |  24 ++++
 ...a-attribute-supported-attributes-list.test |   1 +
 clang/test/Sema/bpf-attr-bpf-fastcall.c       |  14 +++
 llvm/lib/Target/BPF/BPFCallingConv.td         |   1 +
 llvm/lib/Target/BPF/BPFISelLowering.cpp       |  31 +++++
 llvm/lib/Target/BPF/BPFInstrInfo.td           |   4 +-
 llvm/lib/Target/BPF/BPFMIPeephole.cpp         |  86 ++++++++++++++
 llvm/lib/Target/BPF/BPFRegisterInfo.cpp       |  11 ++
 llvm/lib/Target/BPF/BPFRegisterInfo.h         |   3 +
 llvm/test/CodeGen/BPF/bpf-fastcall-1.ll       |  46 ++++++++
 llvm/test/CodeGen/BPF/bpf-fastcall-2.ll       |  68 +++++++++++
 llvm/test/CodeGen/BPF/bpf-fastcall-3.ll       |  62 ++++++++++
 .../CodeGen/BPF/bpf-fastcall-regmask-1.ll     | 110 ++++++++++++++++++
 16 files changed, 488 insertions(+), 3 deletions(-)
 create mode 100644 clang/test/CodeGen/bpf-attr-bpf-fastcall-1.c
 create mode 100644 clang/test/Sema/bpf-attr-bpf-fastcall.c
 create mode 100644 llvm/test/CodeGen/BPF/bpf-fastcall-1.ll
 create mode 100644 llvm/test/CodeGen/BPF/bpf-fastcall-2.ll
 create mode 100644 llvm/test/CodeGen/BPF/bpf-fastcall-3.ll
 create mode 100644 llvm/test/CodeGen/BPF/bpf-fastcall-regmask-1.ll

diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td
index 10a9d9e899e007f..98bedfe20f5d980 100644
--- a/clang/include/clang/Basic/Attr.td
+++ b/clang/include/clang/Basic/Attr.td
@@ -2200,6 +2200,15 @@ def BTFTypeTag : TypeAttr {
   let LangOpts = [COnly];
 }
 
+def BPFFastCall : InheritableAttr,
+                  TargetSpecificAttr<TargetBPF> {
+  let Spellings = [Clang<"bpf_fastcall">];
+  let Subjects = SubjectList<[FunctionLike]>;
+  let Documentation = [BPFFastCallDocs];
+  let LangOpts = [COnly];
+  let SimpleHandler = 1;
+}
+
 def WebAssemblyExportName : InheritableAttr,
                             TargetSpecificAttr<TargetWebAssembly> {
   let Spellings = [Clang<"export_name">];
diff --git a/clang/include/clang/Basic/AttrDocs.td b/clang/include/clang/Basic/AttrDocs.td
index 19cbb9a0111a28a..df36a2163b9f0b3 100644
--- a/clang/include/clang/Basic/AttrDocs.td
+++ b/clang/include/clang/Basic/AttrDocs.td
@@ -2345,6 +2345,25 @@ section.
   }];
 }
 
+def BPFFastCallDocs : Documentation {
+  let Category = DocCatType;
+  let Content = [{
+Functions annotated with this attribute are likely to be inlined by BPF JIT.
+It is assumed that inlined implementation uses less caller saved registers,
+than a regular function.
+Specifically, the following registers are likely to be preserved:
+- ``R0`` if function return value is ``void``;
+- ``R2-R5` if function takes 1 argument;
+- ``R3-R5` if function takes 2 arguments;
+- ``R4-R5` if function takes 3 arguments;
+- ``R5`` if function takes 4 arguments;
+
+For such functions Clang generates code pattern that allows BPF JIT
+to recognize and remove unnecessary spills and fills of the preserved
+registers.
+  }];
+}
+
 def MipsInterruptDocs : Documentation {
   let Category = DocCatFunction;
   let Heading = "interrupt (MIPS)";
diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp
index e4f221ae55eefa1..09803712569755d 100644
--- a/clang/lib/CodeGen/CGCall.cpp
+++ b/clang/lib/CodeGen/CGCall.cpp
@@ -2422,6 +2422,8 @@ void CodeGenModule::ConstructAttributeList(StringRef Name,
       FuncAttrs.addAttribute(llvm::Attribute::NoCfCheck);
     if (TargetDecl->hasAttr<LeafAttr>())
       FuncAttrs.addAttribute(llvm::Attribute::NoCallback);
+    if (TargetDecl->hasAttr<BPFFastCallAttr>())
+      FuncAttrs.addAttribute("bpf_fastcall");
 
     HasOptnone = TargetDecl->hasAttr<OptimizeNoneAttr>();
     if (auto *AllocSize = TargetDecl->getAttr<AllocSizeAttr>()) {
diff --git a/clang/test/CodeGen/bpf-attr-bpf-fastcall-1.c b/clang/test/CodeGen/bpf-attr-bpf-fastcall-1.c
new file mode 100644
index 000000000000000..fa740d8e44ff519
--- /dev/null
+++ b/clang/test/CodeGen/bpf-attr-bpf-fastcall-1.c
@@ -0,0 +1,24 @@
+// REQUIRES: bpf-registered-target
+// RUN: %clang_cc1 -triple bpf -emit-llvm -disable-llvm-passes %s -o - | FileCheck %s
+
+#define __bpf_fastcall __attribute__((bpf_fastcall))
+
+void test(void) __bpf_fastcall;
+void (*ptr)(void) __bpf_fastcall;
+
+void foo(void) {
+  test();
+  (*ptr)();
+}
+
+// CHECK: @ptr = global ptr null
+// CHECK: define {{.*}} void @foo()
+// CHECK: entry:
+// CHECK:   call void @test() #[[call_attr:[0-9]+]]
+// CHECK:   %[[ptr:.*]] = load ptr, ptr @ptr, align 8
+// CHECK:   call void %[[ptr]]() #[[call_attr]]
+// CHECK:   ret void
+
+// CHECK: declare void @test() #[[func_attr:[0-9]+]]
+// CHECK: attributes #[[func_attr]] = { {{.*}}"bpf_fastcall"{{.*}} }
+// CHECK: attributes #[[call_attr]] = { "bpf_fastcall" }
diff --git a/clang/test/Misc/pragma-attribute-supported-attributes-list.test b/clang/test/Misc/pragma-attribute-supported-attributes-list.test
index 1a71556213bb16d..a7e425e3d5f4313 100644
--- a/clang/test/Misc/pragma-attribute-supported-attributes-list.test
+++ b/clang/test/Misc/pragma-attribute-supported-attributes-list.test
@@ -22,6 +22,7 @@
 // CHECK-NEXT: AssumeAligned (SubjectMatchRule_objc_method, SubjectMatchRule_function)
 // CHECK-NEXT: Availability ((SubjectMatchRule_record, SubjectMatchRule_enum, SubjectMatchRule_enum_constant, SubjectMatchRule_field, SubjectMatchRule_function, SubjectMatchRule_namespace, SubjectMatchRule_objc_category, SubjectMatchRule_objc_implementation, SubjectMatchRule_objc_interface, SubjectMatchRule_objc_method, SubjectMatchRule_objc_property, SubjectMatchRule_objc_protocol, SubjectMatchRule_record, SubjectMatchRule_type_alias, SubjectMatchRule_variable))
 // CHECK-NEXT: AvailableOnlyInDefaultEvalMethod (SubjectMatchRule_type_alias)
+// CHECK-NEXT: BPFFastCall (SubjectMatchRule_hasType_functionType)
 // CHECK-NEXT: BPFPreserveAccessIndex (SubjectMatchRule_record)
 // CHECK-NEXT: BPFPreserveStaticOffset (SubjectMatchRule_record)
 // CHECK-NEXT: BTFDeclTag (SubjectMatchRule_variable, SubjectMatchRule_function, SubjectMatchRule_record, SubjectMatchRule_field, SubjectMatchRule_type_alias)
diff --git a/clang/test/Sema/bpf-attr-bpf-fastcall.c b/clang/test/Sema/bpf-attr-bpf-fastcall.c
new file mode 100644
index 000000000000000..178b1f50741e877
--- /dev/null
+++ b/clang/test/Sema/bpf-attr-bpf-fastcall.c
@@ -0,0 +1,14 @@
+// REQUIRES: bpf-registered-target
+// RUN: %clang_cc1 %s -triple bpf -verify
+
+__attribute__((bpf_fastcall)) int var; // expected-warning {{'bpf_fastcall' attribute only applies to functions and function pointers}}
+
+__attribute__((bpf_fastcall)) void func();
+__attribute__((bpf_fastcall(1))) void func_invalid(); // expected-error {{'bpf_fastcall' attribute takes no arguments}}
+
+void (*ptr1)(void) __attribute__((bpf_fastcall));
+void (*ptr2)(void);
+void foo(void) {
+  ptr2 = ptr1; // not an error
+  ptr1 = ptr2; // not an error
+}
diff --git a/llvm/lib/Target/BPF/BPFCallingConv.td b/llvm/lib/Target/BPF/BPFCallingConv.td
index ef4ef1930aa8fb4..a557211437e95f9 100644
--- a/llvm/lib/Target/BPF/BPFCallingConv.td
+++ b/llvm/lib/Target/BPF/BPFCallingConv.td
@@ -46,3 +46,4 @@ def CC_BPF32 : CallingConv<[
 ]>;
 
 def CSR : CalleeSavedRegs<(add R6, R7, R8, R9, R10)>;
+def CSR_PreserveAll : CalleeSavedRegs<(add R0, R1, R2, R3, R4, R5, R6, R7, R8, R9, R10)>;
diff --git a/llvm/lib/Target/BPF/BPFISelLowering.cpp b/llvm/lib/Target/BPF/BPFISelLowering.cpp
index 071fe004806e3e5..ff23d3b055d0d5c 100644
--- a/llvm/lib/Target/BPF/BPFISelLowering.cpp
+++ b/llvm/lib/Target/BPF/BPFISelLowering.cpp
@@ -402,6 +402,21 @@ SDValue BPFTargetLowering::LowerFormalArguments(
 
 const size_t BPFTargetLowering::MaxArgs = 5;
 
+static void resetRegMaskBit(const TargetRegisterInfo *TRI, uint32_t *RegMask,
+                            MCRegister Reg) {
+  for (MCPhysReg SubReg : TRI->subregs_inclusive(Reg))
+    RegMask[SubReg / 32] &= ~(1u << (SubReg % 32));
+}
+
+static uint32_t *regMaskFromTemplate(const TargetRegisterInfo *TRI,
+                                     MachineFunction &MF,
+                                     const uint32_t *BaseRegMask) {
+  uint32_t *RegMask = MF.allocateRegMask();
+  unsigned RegMaskSize = MachineOperand::getRegMaskSize(TRI->getNumRegs());
+  memcpy(RegMask, BaseRegMask, sizeof(RegMask[0]) * RegMaskSize);
+  return RegMask;
+}
+
 SDValue BPFTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
                                      SmallVectorImpl<SDValue> &InVals) const {
   SelectionDAG &DAG = CLI.DAG;
@@ -513,6 +528,22 @@ SDValue BPFTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
   for (auto &Reg : RegsToPass)
     Ops.push_back(DAG.getRegister(Reg.first, Reg.second.getValueType()));
 
+  bool HasFastCall =
+      (CLI.CB && isa<CallInst>(CLI.CB) && CLI.CB->hasFnAttr("bpf_fastcall"));
+  const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
+  if (HasFastCall) {
+    uint32_t *RegMask = regMaskFromTemplate(
+        TRI, MF, TRI->getCallPreservedMask(MF, CallingConv::PreserveAll));
+    for (auto const &RegPair : RegsToPass)
+      resetRegMaskBit(TRI, RegMask, RegPair.first);
+    if (!CLI.CB->getType()->isVoidTy())
+      resetRegMaskBit(TRI, RegMask, BPF::R0);
+    Ops.push_back(DAG.getRegisterMask(RegMask));
+  } else {
+    Ops.push_back(
+        DAG.getRegisterMask(TRI->getCallPreservedMask(MF, CLI.CallConv)));
+  }
+
   if (InGlue.getNode())
     Ops.push_back(InGlue);
 
diff --git a/llvm/lib/Target/BPF/BPFInstrInfo.td b/llvm/lib/Target/BPF/BPFInstrInfo.td
index 2ee630e29790f34..4baeeb017699d69 100644
--- a/llvm/lib/Target/BPF/BPFInstrInfo.td
+++ b/llvm/lib/Target/BPF/BPFInstrInfo.td
@@ -677,9 +677,7 @@ let isBranch = 1, isTerminator = 1, hasDelaySlot=0, isBarrier = 1 in {
 }
 
 // Jump and link
-let isCall=1, hasDelaySlot=0, Uses = [R11],
-    // Potentially clobbered registers
-    Defs = [R0, R1, R2, R3, R4, R5] in {
+let isCall=1, hasDelaySlot=0, Uses = [R11] in {
   def JAL  : CALL<"call">;
   def JALX  : CALLX<"callx">;
 }
diff --git a/llvm/lib/Target/BPF/BPFMIPeephole.cpp b/llvm/lib/Target/BPF/BPFMIPeephole.cpp
index f0edf706bd8fd7a..58dad71b9205526 100644
--- a/llvm/lib/Target/BPF/BPFMIPeephole.cpp
+++ b/llvm/lib/Target/BPF/BPFMIPeephole.cpp
@@ -24,6 +24,8 @@
 #include "BPFInstrInfo.h"
 #include "BPFTargetMachine.h"
 #include "llvm/ADT/Statistic.h"
+#include "llvm/CodeGen/LivePhysRegs.h"
+#include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
@@ -319,6 +321,7 @@ struct BPFMIPreEmitPeephole : public MachineFunctionPass {
   bool in16BitRange(int Num);
   bool eliminateRedundantMov();
   bool adjustBranch();
+  bool insertMissingCallerSavedSpills();
 
 public:
 
@@ -333,6 +336,7 @@ struct BPFMIPreEmitPeephole : public MachineFunctionPass {
     Changed = eliminateRedundantMov();
     if (SupportGotol)
       Changed = adjustBranch() || Changed;
+    Changed |= insertMissingCallerSavedSpills();
     return Changed;
   }
 };
@@ -596,6 +600,88 @@ bool BPFMIPreEmitPeephole::adjustBranch() {
   return Changed;
 }
 
+static const unsigned CallerSavedRegs[] = {BPF::R0, BPF::R1, BPF::R2,
+                                           BPF::R3, BPF::R4, BPF::R5};
+
+struct BPFFastCall {
+  MachineInstr *MI;
+  unsigned LiveCallerSavedRegs;
+};
+
+static void collectBPFFastCalls(const TargetRegisterInfo *TRI,
+                                LivePhysRegs &LiveRegs, MachineBasicBlock &BB,
+                                SmallVectorImpl<BPFFastCall> &Calls) {
+  LiveRegs.init(*TRI);
+  LiveRegs.addLiveOuts(BB);
+  Calls.clear();
+  for (MachineInstr &MI : llvm::reverse(BB)) {
+    if (MI.isCall()) {
+      unsigned LiveCallerSavedRegs = 0;
+      for (MCRegister R : CallerSavedRegs) {
+        bool DoSpillFill = !MI.definesRegister(R, TRI) && LiveRegs.contains(R);
+        if (!DoSpillFill)
+          continue;
+        LiveCallerSavedRegs |= 1 << R;
+      }
+      if (LiveCallerSavedRegs)
+        Calls.push_back({&MI, LiveCallerSavedRegs});
+    }
+    LiveRegs.stepBackward(MI);
+  }
+}
+
+static int64_t computeMinFixedObjOffset(MachineFrameInfo &MFI,
+                                        unsigned SlotSize) {
+  int64_t MinFixedObjOffset = 0;
+  // Same logic as in X86FrameLowering::adjustFrameForMsvcCxxEh()
+  for (int I = MFI.getObjectIndexBegin(); I < MFI.getObjectIndexEnd(); ++I) {
+    if (MFI.isDeadObjectIndex(I))
+      continue;
+    MinFixedObjOffset = std::min(MinFixedObjOffset, MFI.getObjectOffset(I));
+  }
+  MinFixedObjOffset -=
+      (SlotSize + MinFixedObjOffset % SlotSize) & (SlotSize - 1);
+  return MinFixedObjOffset;
+}
+
+bool BPFMIPreEmitPeephole::insertMissingCallerSavedSpills() {
+  MachineFrameInfo &MFI = MF->getFrameInfo();
+  SmallVector<BPFFastCall, 8> Calls;
+  LivePhysRegs LiveRegs;
+  const unsigned SlotSize = 8;
+  int64_t MinFixedObjOffset = computeMinFixedObjOffset(MFI, SlotSize);
+  bool Changed = false;
+  for (MachineBasicBlock &BB : *MF) {
+    collectBPFFastCalls(TRI, LiveRegs, BB, Calls);
+    Changed |= !Calls.empty();
+    for (BPFFastCall &Call : Calls) {
+      int64_t CurOffset = MinFixedObjOffset;
+      for (MCRegister Reg : CallerSavedRegs) {
+        if (((1 << Reg) & Call.LiveCallerSavedRegs) == 0)
+          continue;
+        // Allocate stack object
+        CurOffset -= SlotSize;
+        MFI.CreateFixedSpillStackObject(SlotSize, CurOffset);
+        // Generate spill
+        BuildMI(BB, Call.MI->getIterator(), Call.MI->getDebugLoc(),
+                TII->get(BPF::STD))
+            .addReg(Reg)
+            .addReg(BPF::R10)
+            .addImm(CurOffset)
+            .addImm(0);
+        // Generate fill
+        BuildMI(BB, ++Call.MI->getIterator(), Call.MI->getDebugLoc(),
+                TII->get(BPF::LDD))
+            .addReg(Reg)
+            .addReg(BPF::R10)
+            .addImm(CurOffset)
+            .addImm(0);
+      }
+    }
+  }
+  return Changed;
+}
+
 } // end default namespace
 
 INITIALIZE_PASS(BPFMIPreEmitPeephole, "bpf-mi-pemit-peephole",
diff --git a/llvm/lib/Target/BPF/BPFRegisterInfo.cpp b/llvm/lib/Target/BPF/BPFRegisterInfo.cpp
index 84af6806abb36c0..69e1318954a973a 100644
--- a/llvm/lib/Target/BPF/BPFRegisterInfo.cpp
+++ b/llvm/lib/Target/BPF/BPFRegisterInfo.cpp
@@ -40,6 +40,17 @@ BPFRegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const {
   return CSR_SaveList;
 }
 
+const uint32_t *
+BPFRegisterInfo::getCallPreservedMask(const MachineFunction &MF,
+                                      CallingConv::ID CC) const {
+  switch (CC) {
+  default:
+    return CSR_RegMask;
+  case CallingConv::PreserveAll:
+    return CSR_PreserveAll_RegMask;
+  }
+}
+
 BitVector BPFRegisterInfo::getReservedRegs(const MachineFunction &MF) const {
   BitVector Reserved(getNumRegs());
   markSuperRegs(Reserved, BPF::W10); // [W|R]10 is read only frame pointer
diff --git a/llvm/lib/Target/BPF/BPFRegisterInfo.h b/llvm/lib/Target/BPF/BPFRegisterInfo.h
index f7dea75ebea6f9e..db868769a1579a5 100644
--- a/llvm/lib/Target/BPF/BPFRegisterInfo.h
+++ b/llvm/lib/Target/BPF/BPFRegisterInfo.h
@@ -26,6 +26,9 @@ struct BPFRegisterInfo : public BPFGenRegisterInfo {
 
   const MCPhysReg *getCalleeSavedRegs(const MachineFunction *MF) const override;
 
+  const uint32_t *getCallPreservedMask(const MachineFunction &MF,
+                                       CallingConv::ID) const override;
+
   BitVector getReservedRegs(const MachineFunction &MF) const override;
 
   bool eliminateFrameIndex(MachineBasicBlock::iterator MI, int SPAdj,
diff --git a/llvm/test/CodeGen/BPF/bpf-fastcall-1.ll b/llvm/test/CodeGen/BPF/bpf-fastcall-1.ll
new file mode 100644
index 000000000000000..fd81314a495ef87
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/bpf-fastcall-1.ll
@@ -0,0 +1,46 @@
+; RUN: llc -O2 --march=bpfel %s -o - | FileCheck %s
+
+; Generated from the following C code:
+;
+;   #define __bpf_fastcall __attribute__((bpf_fastcall))
+;
+;   void bar(void) __bpf_fastcall;
+;   void buz(long i, long j, long k);
+;
+;   void foo(long i, long j, long k) {
+;     bar();
+;     buz(i, j, k);
+;   }
+;
+; Using the following command:
+;
+;   clang --target=bpf -emit-llvm -O2 -S -o - t.c
+;
+; (unnecessary attrs removed maually)
+
+; Check that function marked with bpf_fastcall does not clobber R1-R5.
+
+define dso_local void @foo(i64 noundef %i, i64 noundef %j, i64 noundef %k) {
+entry:
+  tail call void @bar() #1
+  tail call void @buz(i64 noundef %i, i64 noundef %j, i64 noundef %k)
+  ret void
+}
+
+; CHECK:      foo:
+; CHECK:      # %bb.0:
+; CHECK-NEXT:   *(u64 *)(r10 - 8) = r1
+; CHECK-NEXT:   *(u64 *)(r10 - 16) = r2
+; CHECK-NEXT:   *(u64 *)(r10 - 24) = r3
+; CHECK-NEXT:   call bar
+; CHECK-NEXT:   r3 = *(u64 *)(r10 - 24)
+; CHECK-NEXT:   r2 = *(u64 *)(r10 - 16)
+; CHECK-NEXT:   r1 = *(u64 *)(r10 - 8)
+; CHECK-NEXT:   call buz
+; CHECK-NEXT:   exit
+
+declare dso_local void @bar() #0
+declare dso_local void @buz(i64 noundef, i64 noundef, i64 noundef)
+
+attributes #0 = { "bpf_fastcall" }
+attributes #1 = { nounwind "bpf_fastcall" }
diff --git a/llvm/test/CodeGen/BPF/bpf-fastcall-2.ll b/llvm/test/CodeGen/BPF/bpf-fastcall-2.ll
new file mode 100644
index 000000000000000..e3e29cdddca8ea6
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/bpf-fastcall-2.ll
@@ -0,0 +1,68 @@
+; RUN: llc -O2 --march=bpfel %s -o - | FileCheck %s
+
+; Generated from the following C code:
+;
+;   #define __bpf_fastcall __attribute__((bpf_fastcall))
+;
+;   void bar(void) __bpf_fastcall;
+;   void buz(long i, long j);
+;
+;   void foo(long i, long j, long k, long l) {
+;     bar();
+;     if (k > 42l)
+;       buz(i, 1);
+;     else
+;       buz(1, j);
+;   }
+;
+; Using the following command:
+;
+;   clang --target=bpf -emit-llvm -O2 -S -o - t.c
+;
+; (unnecessary attrs removed maually)
+
+; Check that function marked with bpf_fastcall does not clobber R1-R5.
+; Use R1 in one branch following call and R2 in another branch following call.
+
+define dso_local void @foo(i64 noundef %i, i64 noundef %j, i64 noundef %k, i64 noundef %l) {
+entry:
+  tail call void @bar() #0
+  %cmp = icmp sgt i64 %k, 42
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:
+  tail call void @buz(i64 noundef %i, i64 noundef 1)
+  br label %if.end
+
+if.else:
+  tail call void @buz(i64 noundef 1, i64 noundef %j)
+  br label %if.end
+
+if.end:
+  ret void
+}
+
+; CHECK:      foo:                                    # @foo
+; CHECK:      # %bb.0:                                # %entry
+; CHECK-NEXT:   *(u64 *)(r10 - 8) = r1
+; CHECK-NEXT:   *(u64 *)(r10 - 16) = r2
+; CHECK-NEXT:   *(u64 *)(r10 - 24) = r3
+; CHECK-NEXT:   call bar
+; CHECK-NEXT:   r3 = *(u64 *)(r10 - 24)
+; CHECK-NEXT:   r2 = *(u64 *)(r10 - 16)
+; CHECK-NEXT:   r1 = *(u64 *)(r10 - 8)
+; CHECK-NEXT:   r4 = 43
+; CHECK-NEXT:   if r4 s> r3 goto [[ELSE:.*]]
+; CHECK-NEXT: # %bb.1:                                # %if.then
+; CHECK-NEXT:   r2 = 1
+; CHECK-NEXT:   goto [[END:.*]]
+; CHECK-NEXT: [[ELSE]]:                               # %if.else
+; CHECK-NEXT:   r1 = 1
+; CHECK-NEXT: [[END]]:                                # %if.end
+; CHECK-NEXT:   call buz
+; CHECK-NEXT:   exit
+
+declare dso_local void @bar() #0
+declare dso_local void @buz(i64 noundef, i64 noundef)
+
+attributes #0 = { "bpf_fastcall" }
diff --git a/llvm/test/CodeGen/BPF/bpf-fastcall-3.ll b/llvm/test/CodeGen/BPF/bpf-fastcall-3.ll
new file mode 100644
index 000000000000000..81ca4e1ac57bc7c
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/bpf-fastcall-3.ll
@@ -0,0 +1,62 @@
+; RUN: llc -O2 --march=bpfel %s -o - | FileCheck %s
+
+; Generated from the following C code:
+;
+; #define __bpf_fastcall __attribute__((bpf_fastcall))
+;
+; void quux(void *);
+; void bar(long) __bpf_fastcall;
+; void buz(long i, long j);
+;
+; void foo(long i, long j) {
+;   long k;
+;   bar(i);
+;   bar(i);
+;   buz(i, j);
+;   quux(&k);
+; }
+;
+; Using the following command:
+;
+;   clang --target=bpf -emit-llvm -O2 -S -o - t.c
+;
+; (unnecessary attrs removed maually)
+
+; Check that function marked with bpf_fastcall does not clobber R1-R5.
+; Check that spills/fills wrapping the call use and reuse lowest stack offsets.
+
+define dso_local void @foo(i64 noundef %i, i64 noundef %j) {
+entry:
+  %k = alloca i64, align 8
+  tail call void @bar(i64 noundef %i) #0
+  tail call void @bar(i64 noundef %i) #0
+  tail call void @buz(i64 noundef %i, i64 noundef %j)
+  call void @quux(ptr noundef nonnull %k)
+  ret void
+}
+
+; CHECK:      # %bb.0:
+; CHECK-NEXT:   r3 = r1
+; CHECK-NEXT:   *(u64 *)(r10 - 16) = r2
+; CHECK-NEXT:   *(u64 *)(r10 - 24) = r3
+; CHECK-NEXT:   call bar
+; CHECK-NEXT:   r3 = *(u64 *)(r10 - 24)
+; CHECK-NEXT:   r2 = *(u64 *)(r10 - 16)
+; CHECK-NEXT:   r1 = r3
+; CHECK-NEXT:   *(u64 *)(r10 - 16) = r2
+; CHECK-NEXT:   *(u64 *)(r10 - 24) = r3
+; CHECK-NEXT:   call bar
+; CHECK-NEXT:   r3 = *(u64 *)(r10 - 24)
+; CHECK-NEXT:   r2 = *(u64 *)(r10 - 16)
+; CHECK-NEXT:   r1 = r3
+; CHECK-NEXT:   call buz
+; CHECK-NEXT:   r1 = r10
+; CHECK-NEXT:   r1 += -8
+; CHECK-NEXT:   call quux
+; CHECK-NEXT:   exit
+
+declare dso_local void @bar(i64 noundef) #0
+declare dso_local void @buz(i64 noundef, i64 noundef)
+declare dso_local void @quux(ptr noundef)
+
+attributes #0 = { "bpf_fastcall" }
diff --git a/llvm/test/CodeGen/BPF/bpf-fastcall-regmask-1.ll b/llvm/test/CodeGen/BPF/bpf-fastcall-regmask-1.ll
new file mode 100644
index 000000000000000..857d2f000d1d5a3
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/bpf-fastcall-regmask-1.ll
@@ -0,0 +1,110 @@
+; RUN: llc -O2 --march=bpfel \
+; RUN:   -print-after=stack-slot-coloring %s \
+; RUN:   -o /dev/null 2>&1 | FileCheck %s
+
+; Generated from the following C code:
+;
+;   #define __bpf_fastcall __attribute__((bpf_fastcall))
+;
+;   void bar1(void) __bpf_fastcall;
+;   void buz1(long i, long j, long k);
+;   void foo1(long i, long j, long k) {
+;     bar1();
+;     buz1(i, j, k);
+;   }
+;
+;   long bar2(void) __bpf_fastcall;
+;   void buz2(long i, long j, long k);
+;   void foo2(long i, long j, long k) {
+;     bar2();
+;     buz2(i, j, k);
+;   }
+;
+;   void bar3(long) __bpf_fastcall;
+;   void buz3(long i, long j, long k);
+;   void foo3(long i, long j, long k) {
+;     bar3(i);
+;     buz3(i, j, k);
+;   }
+;
+;   long bar4(long, long) __bpf_fastcall;
+;   void buz4(long i, long j, long k);
+;   void foo4(long i, long j, long k) {
+;     bar4(i, j);
+;     buz4(i, j, k);
+;   }
+;
+; Using the following command:
+;
+;   clang --target=bpf -emit-llvm -O2 -S -o - t.c
+;
+; (unnecessary attrs removed maually)
+
+; Check regmask for calls to functions marked with bpf_fastcall:
+; - void function w/o parameters
+; - non-void function w/o parameters
+; - void function with parameters
+; - non-void function with parameters
+
+declare dso_local void @bar1() #0
+declare dso_local void @buz1(i64 noundef, i64 noundef, i64 noundef)
+define dso_local void @foo1(i64 noundef %i, i64 noundef %j, i64 noundef %k) {
+entry:
+  tail call void @bar1() #1
+  tail call void @buz1(i64 noundef %i, i64 noundef %j, i64 noundef %k)
+  ret void
+}
+
+; CHECK:      JAL @bar1, <regmask $r0 $r1 $r2 $r3 $r4 $r5 $r6 $r7 $r8 $r9 $r10
+; CHECK-SAME:                     $w0 $w1 $w2 $w3 $w4 $w5 $w6 $w7 $w8 $w9 $w10>
+; CHECK-SAME:          , implicit $r11, implicit-def $r11
+; CHECK:      JAL @buz1, <regmask $r6 $r7 $r8 $r9 $r10 $w6 $w7 $w8 $w9 $w10>
+; CHECK-SAME:          , implicit $r11, implicit $r1, implicit $r2, implicit $r3, implicit-def $r11
+
+declare dso_local i64 @bar2() #0
+declare dso_local void @buz2(i64 noundef, i64 noundef, i64 noundef)
+define dso_local void @foo2(i64 noundef %i, i64 noundef %j, i64 noundef %k) {
+entry:
+  tail call i64 @bar2() #1
+  tail call void @buz2(i64 noundef %i, i64 noundef %j, i64 noundef %k)
+  ret void
+}
+
+; CHECK:      JAL @bar2, <regmask $r1 $r2 $r3 $r4 $r5 $r6 $r7 $r8 $r9 $r10
+; CHECK-SAME:                     $w1 $w2 $w3 $w4 $w5 $w6 $w7 $w8 $w9 $w10>
+; CHECK-SAME:          , implicit $r11, implicit-def $r11, implicit-def dead $r0
+; CHECK:      JAL @buz2, <regmask $r6 $r7 $r8 $r9 $r10 $w6 $w7 $w8 $w9 $w10>
+; CHECK-SAME:          , implicit $r11, implicit $r1, implicit $r2, implicit $r3, implicit-def $r11
+
+declare dso_local void @bar3(i64) #0
+declare dso_local void @buz3(i64 noundef, i64 noundef, i64 noundef)
+define dso_local void @foo3(i64 noundef %i, i64 noundef %j, i64 noundef %k) {
+entry:
+  tail call void @bar3(i64 noundef %i) #1
+  tail call void @buz3(i64 noundef %i, i64 noundef %j, i64 noundef %k)
+  ret void
+}
+
+; CHECK:      JAL @bar3, <regmask $r0 $r2 $r3 $r4 $r5 $r6 $r7 $r8 $r9 $r10
+; CHECK-SAME:                     $w0 $w2 $w3 $w4 $w5 $w6 $w7 $w8 $w9 $w10>
+; CHECK-SAME:          , implicit $r11, implicit $r1, implicit-def $r11
+; CHECK:      JAL @buz3, <regmask $r6 $r7 $r8 $r9 $r10 $w6 $w7 $w8 $w9 $w10>
+; CHECK-SAME:          , implicit $r11, implicit $r1, implicit $r2, implicit $r3, implicit-def $r11
+
+declare dso_local i64 @bar4(i64 noundef, i64 noundef) #0
+declare dso_local void @buz4(i64 noundef, i64 noundef, i64 noundef)
+define dso_local void @foo4(i64 noundef %i, i64 noundef %j, i64 noundef %k) {
+entry:
+  tail call i64 @bar4(i64 noundef %i, i64 noundef %j) #1
+  tail call void @buz4(i64 noundef %i, i64 noundef %j, i64 noundef %k)
+  ret void
+}
+
+; CHECK:      JAL @bar4, <regmask $r3 $r4 $r5 $r6 $r7 $r8 $r9 $r10
+; CHECK-SAME:                     $w3 $w4 $w5 $w6 $w7 $w8 $w9 $w10>
+; CHECK-SAME:          , implicit $r11, implicit $r1, implicit $r2, implicit-def $r11, implicit-def dead $r0
+; CHECK:      JAL @buz4, <regmask $r6 $r7 $r8 $r9 $r10 $w6 $w7 $w8 $w9 $w10>
+; CHECK-SAME:          , implicit $r11, implicit $r1, implicit $r2, implicit $r3, implicit-def $r11
+
+attributes #0 = { "bpf_fastcall" }
+attributes #1 = { nounwind "bpf_fastcall" }

From 9b65558d2f20463b1a32374165d6fa06696623e9 Mon Sep 17 00:00:00 2001
From: Sam Elliott <quic_aelliott@quicinc.com>
Date: Mon, 19 Aug 2024 17:51:46 +0100
Subject: [PATCH 384/441] [lld][ELF] Combine uniqued small data sections
 (#104485)

RISC-V GCC with `-fdata-sections` will emit `.sbss.<name>`,
`.srodata.<name>`, and `.sdata.<name>` sections for small data items of
different kinds. Clang/LLVM already emits `.srodata.*` sections, and we
intend to emit the other two section name patterns in #87040.

This change ensures that any input sections starting `.sbss` are
combined into one output section called `.sbss`, and the same
respectively for `.srodata` and `.sdata`. This also allows the existing
RISC-V specific code for determining an output order for `.sbss` and
`.sdata` sections to apply to placing the sections.
---
 lld/ELF/LinkerScript.cpp            | 11 ++++++----
 lld/test/ELF/riscv-section-layout.s | 34 ++++++++++++++++++-----------
 lld/test/ELF/section-name.s         | 19 ++++++++++++++--
 3 files changed, 45 insertions(+), 19 deletions(-)

diff --git a/lld/ELF/LinkerScript.cpp b/lld/ELF/LinkerScript.cpp
index 0c4ba1abb4778c2..01c15d79e748aed 100644
--- a/lld/ELF/LinkerScript.cpp
+++ b/lld/ELF/LinkerScript.cpp
@@ -106,10 +106,13 @@ static StringRef getOutputSectionName(const InputSectionBase *s) {
     return ".text";
   }
 
-  for (StringRef v :
-       {".data.rel.ro", ".data", ".rodata", ".bss.rel.ro", ".bss", ".ldata",
-        ".lrodata", ".lbss", ".gcc_except_table", ".init_array", ".fini_array",
-        ".tbss", ".tdata", ".ARM.exidx", ".ARM.extab", ".ctors", ".dtors"})
+  for (StringRef v : {".data.rel.ro", ".data",       ".rodata",
+                      ".bss.rel.ro",  ".bss",        ".ldata",
+                      ".lrodata",     ".lbss",       ".gcc_except_table",
+                      ".init_array",  ".fini_array", ".tbss",
+                      ".tdata",       ".ARM.exidx",  ".ARM.extab",
+                      ".ctors",       ".dtors",      ".sbss",
+                      ".sdata",       ".srodata"})
     if (isSectionPrefix(v, s->name))
       return v;
 
diff --git a/lld/test/ELF/riscv-section-layout.s b/lld/test/ELF/riscv-section-layout.s
index 10e0febf7c4260a..123b5f056ad66ba 100644
--- a/lld/test/ELF/riscv-section-layout.s
+++ b/lld/test/ELF/riscv-section-layout.s
@@ -15,12 +15,13 @@
 # RUN: ld.lld -pie %t.64s.o -o %t.64s
 # RUN: llvm-readelf -S -sX %t.64s | FileCheck %s
 
-# NOSDATA:      .text
+# NOSDATA:      .rodata  PROGBITS
+# NOSDATA-NEXT: .text    PROGBITS
 # NOSDATA-NEXT: .tdata   PROGBITS [[#%x,TDATA:]]
-# NOSDATA-NEXT: .tbss
-# NOSDATA-NEXT: .dynamic
-# NOSDATA-NEXT: .got
-# NOSDATA-NEXT: .relro_padding
+# NOSDATA-NEXT: .tbss    NOBITS
+# NOSDATA-NEXT: .dynamic DYNAMIC
+# NOSDATA-NEXT: .got     PROGBITS
+# NOSDATA-NEXT: .relro_padding NOBITS
 # NOSDATA-NEXT: .data    PROGBITS [[#%x,DATA:]]
 # NOSDATA-NEXT: .bss     NOBITS   [[#%x,BSS:]]
 
@@ -33,16 +34,18 @@
 # NOSDATA-DAG:  [[#]]: {{0*}}[[#BSS]]         0 NOTYPE  GLOBAL DEFAULT [[#]] (.bss) __bss_start
 # NOSDATA-DAG:  [[#]]: {{0*}}800              0 NOTYPE  GLOBAL DEFAULT  1 (.dynsym) __global_pointer$
 
-# CHECK:      .text
-# CHECK-NEXT: .tdata
-# CHECK-NEXT: .tbss
-# CHECK-NEXT: .dynamic
-# CHECK-NEXT: .got
-# CHECK-NEXT: .relro_padding
-# CHECK-NEXT: .data
+# CHECK:      .rodata    PROGBITS
+# CHECK-NEXT: .srodata   PROGBITS
+# CHECK-NEXT: .text      PROGBITS
+# CHECK-NEXT: .tdata     PROGBITS
+# CHECK-NEXT: .tbss      NOBITS
+# CHECK-NEXT: .dynamic   DYNAMIC
+# CHECK-NEXT: .got       PROGBITS
+# CHECK-NEXT: .relro_padding NOBITS
+# CHECK-NEXT: .data      PROGBITS
 # CHECK-NEXT: .sdata     PROGBITS [[#%x,SDATA:]]
 # CHECK-NEXT: .sbss      NOBITS   [[#%x,SBSS:]]
-# CHECK-NEXT: .bss
+# CHECK-NEXT: .bss       NOBITS
 
 # CHECK-DAG:  [[#]]: {{0*}}[[#SBSS]]        0 NOTYPE  GLOBAL DEFAULT [[#]] (.sdata) _edata
 # CHECK-DAG:  [[#]]: {{0*}}[[#SBSS]]        0 NOTYPE  GLOBAL DEFAULT [[#]] (.sbss) __bss_start
@@ -51,11 +54,16 @@
 .globl _etext, _edata, __bss_start
   lla gp, __global_pointer$
 
+.section .rodata,"a",@progbits; .space 1
 .section .data,"aw",@progbits; .long _GLOBAL_OFFSET_TABLE_ - .
 .section .bss,"aw",@nobits; .space 1
 .section .tdata,"awT",@progbits; .space 1
 .section .tbss,"awT",@nobits; .space 1
 .ifdef SDATA
 .section .sdata,"aw",@progbits; .space 1
+.section .sdata.suffix,"aw",@progbits; .space 1
 .section .sbss,"aw",@nobits; .space 1
+.section .sbss.suffix,"aw",@nobits; .space 1
+.section .srodata,"a",@progbits; .space 1
+.section .srodata.suffix,"a",@progbits; .space 1
 .endif
diff --git a/lld/test/ELF/section-name.s b/lld/test/ELF/section-name.s
index 819cd9c14d95001..7793cc547295996 100644
--- a/lld/test/ELF/section-name.s
+++ b/lld/test/ELF/section-name.s
@@ -41,19 +41,34 @@ _start:
 .byte 0
 .section .tdata.foo,"aGwT",@progbits,foo,comdat
 .byte 0
+.section .sdata,"aw"
+.byte 0
+.section .sdata.foo,"aw"
+.byte 0
+.section .sbss,"aw",@nobits
+.byte 0
+.section .sbss.foo,"aw",@nobits
+.byte 0
+.section .srodata,"a"
+.byte 0
+.section .srodata.foo,"a"
+.byte 0
 
 // CHECK:      .rodata           00000002
 // CHECK-NEXT: .gcc_except_table 00000001
+// CHECK-NEXT: .srodata          00000002
 // CHECK-NEXT: .text             00000002
 // CHECK-NEXT: .tdata            00000001
 // CHECK-NEXT: .tbss             00000001
 // CHECK-NEXT: .data.rel.ro      00000004
-// CHECK-NEXT: .relro_padding    00000df5
+// CHECK-NEXT: .relro_padding    00000df1
 // CHECK-NEXT: .data             00000002
 // CHECK-NEXT: .foo.a            00000001
 // CHECK-NEXT: .foo              00000001
+// CHECK-NEXT: .sdata            00000002
 // CHECK-NEXT: .bss              00000002
+// CHECK-NEXT: .sbss             00000002
 // CHECK-NEXT: .comment          00000008
 // CHECK-NEXT: .symtab           00000030
-// CHECK-NEXT: .shstrtab         00000084
+// CHECK-NEXT: .shstrtab         0000009a
 // CHECK-NEXT: .strtab           00000008

From 93d0f82272dc230ff4d53d48c43b2f1518754a7b Mon Sep 17 00:00:00 2001
From: Vitaly Buka <vitalybuka@google.com>
Date: Mon, 19 Aug 2024 09:53:03 -0700
Subject: [PATCH 385/441] [docs] Add note about "Re-request review" (#104735)

---
 llvm/docs/CodeReview.rst | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/llvm/docs/CodeReview.rst b/llvm/docs/CodeReview.rst
index ca57cffc519de8a..0128e37465de321 100644
--- a/llvm/docs/CodeReview.rst
+++ b/llvm/docs/CodeReview.rst
@@ -125,6 +125,11 @@ commit message).
 If you suggest changes in a code review, but don't wish the suggestion to be
 interpreted this strongly, please state so explicitly.
 
+.. note::
+   After responding to reviewer comments,
+   press `Re-request review <https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/requesting-a-pull-request-review#:~:text=After%20your%20pull%20request%20is%20reviewed>`_
+   to bring the Pull Request to the reviewers' attention.
+
 Aim to Make Efficient Use of Everyone's Time
 --------------------------------------------
 

From 7d60f4648b9f794b8965b84a7a5641dfc5e07fae Mon Sep 17 00:00:00 2001
From: Justin Bogner <mail@justinbogner.com>
Date: Mon, 19 Aug 2024 09:55:15 -0700
Subject: [PATCH 386/441] [DirectX] Differentiate between 0/1 overloads in the
 OpBuilder. NFC

DXIL operations that only have one signature behave one of two ways - either
they are always suffixed with a type like `dx.op.ThreadId.i32` and hence have
exactly one overload, or they're never suffixed like `dx.op.CreateHandle` and
hence have zero overloads.

Update DXIL.td for operations that have one overload and remove the hack in the
builder that was adjusting names for unoverloaded ops.

Pull Request: https://github.com/llvm/llvm-project/pull/104246
---
 llvm/lib/Target/DirectX/DXIL.td           | 18 +++++++++++-------
 llvm/lib/Target/DirectX/DXILOpBuilder.cpp |  9 ++++-----
 2 files changed, 15 insertions(+), 12 deletions(-)

diff --git a/llvm/lib/Target/DirectX/DXIL.td b/llvm/lib/Target/DirectX/DXIL.td
index 67015cff78a79a0..60185c20606b22b 100644
--- a/llvm/lib/Target/DirectX/DXIL.td
+++ b/llvm/lib/Target/DirectX/DXIL.td
@@ -671,8 +671,9 @@ def Dot4 :  DXILOp<56, dot4> {
 def ThreadId :  DXILOp<93, threadId> {
   let Doc = "Reads the thread ID";
   let LLVMIntrinsic = int_dx_thread_id;
-  let arguments = [i32Ty];
-  let result = i32Ty;
+  let arguments = [overloadTy];
+  let result = overloadTy;
+  let overloads = [Overloads<DXIL1_0, [i32Ty]>];
   let stages = [Stages<DXIL1_0, [compute, mesh, amplification, node]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -680,8 +681,9 @@ def ThreadId :  DXILOp<93, threadId> {
 def GroupId :  DXILOp<94, groupId> {
   let Doc = "Reads the group ID (SV_GroupID)";
   let LLVMIntrinsic = int_dx_group_id;
-  let arguments = [i32Ty];
-  let result = i32Ty;
+  let arguments = [overloadTy];
+  let result = overloadTy;
+  let overloads = [Overloads<DXIL1_0, [i32Ty]>];
   let stages = [Stages<DXIL1_0, [compute, mesh, amplification, node]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -689,8 +691,9 @@ def GroupId :  DXILOp<94, groupId> {
 def ThreadIdInGroup :  DXILOp<95, threadIdInGroup> {
   let Doc = "Reads the thread ID within the group  (SV_GroupThreadID)";
   let LLVMIntrinsic = int_dx_thread_id_in_group;
-  let arguments = [i32Ty];
-  let result = i32Ty;
+  let arguments = [overloadTy];
+  let result = overloadTy;
+  let overloads = [Overloads<DXIL1_0, [i32Ty]>];
   let stages = [Stages<DXIL1_0, [compute, mesh, amplification, node]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -699,7 +702,8 @@ def FlattenedThreadIdInGroup :  DXILOp<96, flattenedThreadIdInGroup> {
   let Doc = "Provides a flattened index for a given thread within a given "
             "group (SV_GroupIndex)";
   let LLVMIntrinsic = int_dx_flattened_thread_id_in_group;
-  let result = i32Ty;
+  let result = overloadTy;
+  let overloads = [Overloads<DXIL1_0, [i32Ty]>];
   let stages = [Stages<DXIL1_0, [compute, mesh, amplification, node]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
diff --git a/llvm/lib/Target/DirectX/DXILOpBuilder.cpp b/llvm/lib/Target/DirectX/DXILOpBuilder.cpp
index 91e6931b3f788eb..0e2b4601112b5fa 100644
--- a/llvm/lib/Target/DirectX/DXILOpBuilder.cpp
+++ b/llvm/lib/Target/DirectX/DXILOpBuilder.cpp
@@ -87,6 +87,9 @@ static const char *getOverloadTypeName(OverloadKind Kind) {
 }
 
 static OverloadKind getOverloadKind(Type *Ty) {
+  if (!Ty)
+    return OverloadKind::VOID;
+
   Type::TypeID T = Ty->getTypeID();
   switch (T) {
   case Type::VoidTyID:
@@ -379,11 +382,7 @@ Expected<CallInst *> DXILOpBuilder::tryCreateOp(dxil::OpCode OpCode,
 
   uint16_t ValidTyMask = Prop->Overloads[*OlIndexOrErr].ValidTys;
 
-  // If we don't have an overload type, use the function's return type. This is
-  // a bit of a hack, but it's necessary to get the type suffix on unoverloaded
-  // DXIL ops correct, like `dx.op.threadId.i32`.
-  OverloadKind Kind =
-      getOverloadKind(OverloadTy ? OverloadTy : DXILOpFT->getReturnType());
+  OverloadKind Kind = getOverloadKind(OverloadTy);
 
   // Check if the operation supports overload types and OverloadTy is valid
   // per the specified types for the operation

From 6bb5a0b171e22ef0a17ae8bf9bf4a3f7ad682901 Mon Sep 17 00:00:00 2001
From: cor3ntin <corentinjabot@gmail.com>
Date: Mon, 19 Aug 2024 19:03:21 +0200
Subject: [PATCH 387/441] [Clang] Check constraints for an explicit
 instantiation of a member function (#104438)

Because there may be multiple constrained function of the same type, we
need to perform overload resolution to find the best viable function to
specialize.


Fixes #46029
---
 clang/docs/ReleaseNotes.rst                   |  3 +-
 .../clang/Basic/DiagnosticSemaKinds.td        |  2 +
 clang/lib/Sema/SemaTemplate.cpp               | 64 +++++++++++++++----
 .../explicit-instantiation-cxx20.cpp          | 40 ++++++++++++
 4 files changed, 95 insertions(+), 14 deletions(-)
 create mode 100644 clang/test/SemaTemplate/explicit-instantiation-cxx20.cpp

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index b78c0229de4898b..017df3cae468a1e 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -73,7 +73,7 @@ C++ Specific Potentially Breaking Changes
     template <> // error: extraneous template head
     template <typename T>
     void f();
-    
+
 ABI Changes in This Version
 ---------------------------
 
@@ -273,6 +273,7 @@ Bug Fixes to C++ Support
 - Properly reject defaulted copy/move assignment operators that have a non-reference explicit object parameter.
 - Clang now properly handles the order of attributes in `extern` blocks. (#GH101990).
 - Fixed an assertion failure by preventing null explicit object arguments from being deduced. (#GH102025).
+- Correctly check constraints of explicit instantiations of member functions. (#GH46029)
 
 Bug Fixes to AST Handling
 ^^^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 0814e15c0f9dd55..4b6aadd635786a9 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -5667,6 +5667,8 @@ def err_explicit_instantiation_internal_linkage : Error<
 def err_explicit_instantiation_not_known : Error<
   "explicit instantiation of %0 does not refer to a function template, "
   "variable template, member function, member class, or static data member">;
+def err_explicit_instantiation_no_candidate : Error<
+    "no viable candidate for explicit instantiation of %0">;
 def note_explicit_instantiation_here : Note<
   "explicit instantiation refers here">;
 def err_explicit_instantiation_data_member_not_instantiated : Error<
diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp
index 25585f683752ac4..0afe6064bab185e 100644
--- a/clang/lib/Sema/SemaTemplate.cpp
+++ b/clang/lib/Sema/SemaTemplate.cpp
@@ -10124,8 +10124,9 @@ DeclResult Sema::ActOnExplicitInstantiation(Scope *S,
   //  instantiated from the member definition associated with its class
   //  template.
   UnresolvedSet<8> TemplateMatches;
-  FunctionDecl *NonTemplateMatch = nullptr;
-  TemplateSpecCandidateSet FailedCandidates(D.getIdentifierLoc());
+  OverloadCandidateSet NonTemplateMatches(D.getBeginLoc(),
+                                          OverloadCandidateSet::CSK_Normal);
+  TemplateSpecCandidateSet FailedTemplateCandidates(D.getIdentifierLoc());
   for (LookupResult::iterator P = Previous.begin(), PEnd = Previous.end();
        P != PEnd; ++P) {
     NamedDecl *Prev = *P;
@@ -10137,9 +10138,18 @@ DeclResult Sema::ActOnExplicitInstantiation(Scope *S,
           if (Method->getPrimaryTemplate()) {
             TemplateMatches.addDecl(Method, P.getAccess());
           } else {
-            // FIXME: Can this assert ever happen?  Needs a test.
-            assert(!NonTemplateMatch && "Multiple NonTemplateMatches");
-            NonTemplateMatch = Method;
+            OverloadCandidate &C = NonTemplateMatches.addCandidate();
+            C.FoundDecl = P.getPair();
+            C.Function = Method;
+            C.Viable = true;
+            ConstraintSatisfaction S;
+            if (Method->getTrailingRequiresClause() &&
+                (CheckFunctionConstraints(Method, S, D.getIdentifierLoc(),
+                                          /*ForOverloadResolution=*/true) ||
+                 !S.IsSatisfied)) {
+              C.Viable = false;
+              C.FailureKind = ovl_fail_constraints_not_satisfied;
+            }
           }
         }
       }
@@ -10149,16 +10159,16 @@ DeclResult Sema::ActOnExplicitInstantiation(Scope *S,
     if (!FunTmpl)
       continue;
 
-    TemplateDeductionInfo Info(FailedCandidates.getLocation());
+    TemplateDeductionInfo Info(FailedTemplateCandidates.getLocation());
     FunctionDecl *Specialization = nullptr;
     if (TemplateDeductionResult TDK = DeduceTemplateArguments(
             FunTmpl, (HasExplicitTemplateArgs ? &TemplateArgs : nullptr), R,
             Specialization, Info);
         TDK != TemplateDeductionResult::Success) {
       // Keep track of almost-matches.
-      FailedCandidates.addCandidate()
-          .set(P.getPair(), FunTmpl->getTemplatedDecl(),
-               MakeDeductionFailureInfo(Context, TDK, Info));
+      FailedTemplateCandidates.addCandidate().set(
+          P.getPair(), FunTmpl->getTemplatedDecl(),
+          MakeDeductionFailureInfo(Context, TDK, Info));
       (void)TDK;
       continue;
     }
@@ -10172,7 +10182,7 @@ DeclResult Sema::ActOnExplicitInstantiation(Scope *S,
         CUDA().IdentifyTarget(Specialization,
                               /* IgnoreImplicitHDAttr = */ true) !=
             CUDA().IdentifyTarget(D.getDeclSpec().getAttributes())) {
-      FailedCandidates.addCandidate().set(
+      FailedTemplateCandidates.addCandidate().set(
           P.getPair(), FunTmpl->getTemplatedDecl(),
           MakeDeductionFailureInfo(
               Context, TemplateDeductionResult::CUDATargetMismatch, Info));
@@ -10182,12 +10192,40 @@ DeclResult Sema::ActOnExplicitInstantiation(Scope *S,
     TemplateMatches.addDecl(Specialization, P.getAccess());
   }
 
-  FunctionDecl *Specialization = NonTemplateMatch;
+  FunctionDecl *Specialization = nullptr;
+  if (!NonTemplateMatches.empty()) {
+    unsigned Msg = 0;
+    OverloadCandidateDisplayKind DisplayKind;
+    OverloadCandidateSet::iterator Best;
+    switch (NonTemplateMatches.BestViableFunction(*this, D.getIdentifierLoc(),
+                                                  Best)) {
+    case OR_Success:
+    case OR_Deleted:
+      Specialization = cast<FunctionDecl>(Best->Function);
+      break;
+    case OR_Ambiguous:
+      Msg = diag::err_explicit_instantiation_ambiguous;
+      DisplayKind = OCD_AmbiguousCandidates;
+      break;
+    case OR_No_Viable_Function:
+      Msg = diag::err_explicit_instantiation_no_candidate;
+      DisplayKind = OCD_AllCandidates;
+      break;
+    }
+    if (Msg) {
+      PartialDiagnostic Diag = PDiag(Msg) << Name;
+      NonTemplateMatches.NoteCandidates(
+          PartialDiagnosticAt(D.getIdentifierLoc(), Diag), *this, DisplayKind,
+          {});
+      return true;
+    }
+  }
+
   if (!Specialization) {
     // Find the most specialized function template specialization.
     UnresolvedSetIterator Result = getMostSpecialized(
-        TemplateMatches.begin(), TemplateMatches.end(), FailedCandidates,
-        D.getIdentifierLoc(),
+        TemplateMatches.begin(), TemplateMatches.end(),
+        FailedTemplateCandidates, D.getIdentifierLoc(),
         PDiag(diag::err_explicit_instantiation_not_known) << Name,
         PDiag(diag::err_explicit_instantiation_ambiguous) << Name,
         PDiag(diag::note_explicit_instantiation_candidate));
diff --git a/clang/test/SemaTemplate/explicit-instantiation-cxx20.cpp b/clang/test/SemaTemplate/explicit-instantiation-cxx20.cpp
new file mode 100644
index 000000000000000..33500baf1e37b10
--- /dev/null
+++ b/clang/test/SemaTemplate/explicit-instantiation-cxx20.cpp
@@ -0,0 +1,40 @@
+// RUN: %clang_cc1 -fsyntax-only -verify -std=c++20  %s
+
+template<class T>
+concept C = true;
+
+template<class T>
+concept D = C<T> && true;
+
+template<typename T>
+struct a {
+    void no_candidate() requires(false) {}
+    // expected-note@-1 {{candidate function not viable: constraints not satisfied}} \
+    // expected-note@-1 {{because 'false' evaluated to false}}
+    void no_candidate() requires(false && false) {}
+    // expected-note@-1 {{candidate function not viable: constraints not satisfied}} \
+    // expected-note@-1 {{because 'false' evaluated to false}}
+
+    void subsumes();
+    void subsumes() requires C<T>;
+    void subsumes() requires D<T> {};
+
+    void ok() requires false;
+    void ok() requires true {};
+
+    void ok2() requires false;
+    void ok2(){};
+
+    void ambiguous() requires true;
+    // expected-note@-1 {{candidate function}}
+    void ambiguous() requires C<T>;
+    // expected-note@-1 {{candidate function}}
+};
+template void a<int>::no_candidate();
+// expected-error@-1 {{no viable candidate for explicit instantiation of 'no_candidate'}}
+
+template void a<int>::ambiguous();
+// expected-error@-1 {{partial ordering for explicit instantiation of 'ambiguous' is ambiguous}}
+
+template void a<int>::ok();
+template void a<int>::ok2();

From f357fe371d3f752878abf1fb1d5fb550e2650c8e Mon Sep 17 00:00:00 2001
From: Justin Bogner <mail@justinbogner.com>
Date: Mon, 19 Aug 2024 10:09:46 -0700
Subject: [PATCH 388/441] [DirectX] Disentangle DXIL.td's op types from
 LLVMType. NFC

LLVMType is both too broad and too narrow for defining DXIL operations, in
different ways. It's too broad in the sense that we don't need the full set of
MVTs - the set of types DXIL operations work on is much smaller. It's too
narrow in the sense that it's difficult to use it for the various fixed
structure types in DXIL, like `%dx.types.Handle` or `%dx.Types.ResRet.f32`.

Replace the usage of LLVMType in DXIL.td with DXILOpParamType, a simple class
that we can define an enum of types from. Further, use this to replace the
"ParameterKind" enum in DXILABI.h that has nothing to do with DXIL's ABI.

Pull Request: https://github.com/llvm/llvm-project/pull/104247
---
 llvm/docs/DirectX/DXILOpTableGenDesign.rst |  28 +--
 llvm/include/llvm/Support/DXILABI.h        |  17 --
 llvm/lib/Target/DirectX/DXIL.td            | 259 +++++++++++----------
 llvm/lib/Target/DirectX/DXILConstants.h    |   5 +
 llvm/lib/Target/DirectX/DXILOpBuilder.cpp  |  74 +++---
 llvm/utils/TableGen/DXILEmitter.cpp        | 176 ++++----------
 6 files changed, 234 insertions(+), 325 deletions(-)

diff --git a/llvm/docs/DirectX/DXILOpTableGenDesign.rst b/llvm/docs/DirectX/DXILOpTableGenDesign.rst
index 50d801bd05efdbc..46106ee2a50f572 100644
--- a/llvm/docs/DirectX/DXILOpTableGenDesign.rst
+++ b/llvm/docs/DirectX/DXILOpTableGenDesign.rst
@@ -93,18 +93,14 @@ properties are specified as fields of the ``DXILOp`` class as described below.
         class DXILOpClass;
 
    Concrete operation records, such as ``unary`` are defined by inheriting from ``DXILOpClass``.
-6. Return type of the operation is represented as ``LLVMType``.
-7. Operation arguments are represented as a list of ``LLVMType`` with each type
-   corresponding to the argument position. An overload type, if supported by the operation, is
-   denoted as the positional type ``overloadTy`` in the argument or in the result, where
-   ``overloadTy`` is defined to be synonymous to ``llvm_any_ty``.
-
-   .. code-block::
-
-      defvar overloadTy = llvm_any_ty
-
-   Empty list, ``[]`` represents an operation with no arguments.
-
+6. A set of type names are defined that represent return and argument types,
+   which all inherit from ``DXILOpParamType``. These represent simple types
+   like ``int32Ty``, DXIL types like ``dx.types.Handle``, and a special
+   ``overloadTy`` which can be any type allowed by ``Overloads``, described
+   below.
+7. Operation return type is represented as a ``DXILOpParamType``, and arguments
+   are represented as a list of the same. An operation with no return value
+   shall specify ``VoidTy`` as its return.
 8. Valid operation overload types predicated on DXIL version are specified as
    a list of ``Overloads`` records. Representation of ``Overloads``
    class is described in a later section.
@@ -145,10 +141,10 @@ TableGen representations of its properties described above.
      Intrinsic LLVMIntrinsic = ?;
 
      // Result type of the op.
-     LLVMType result;
+     DXILOpParamType result;
 
      // List of argument types of the op. Default to 0 arguments.
-     list<LLVMType> arguments = [];
+     list<DXILOpParamType> arguments = [];
 
      // List of valid overload types predicated by DXIL version
      list<Overloads> overloads;
@@ -233,9 +229,9 @@ overloads predicated on DXIL version as list of records of the following class
 
 .. code-block::
 
-   class Overloads<Version minver, list<LLVMType> ols> {
+   class Overloads<Version minver, list<DXILOpParamType> ols> {
      Version dxil_version = minver;
-     list<LLVMType> overload_types = ols;
+     list<DXILOpParamType> overload_types = ols;
    }
 
 Following is an example specification of valid overload types for ``DXIL1_0`` and
diff --git a/llvm/include/llvm/Support/DXILABI.h b/llvm/include/llvm/Support/DXILABI.h
index a2222eec09ba85a..cf2c42c689889d8 100644
--- a/llvm/include/llvm/Support/DXILABI.h
+++ b/llvm/include/llvm/Support/DXILABI.h
@@ -22,23 +22,6 @@
 namespace llvm {
 namespace dxil {
 
-enum class ParameterKind : uint8_t {
-  Invalid = 0,
-  Void,
-  Half,
-  Float,
-  Double,
-  I1,
-  I8,
-  I16,
-  I32,
-  I64,
-  Overload,
-  CBufferRet,
-  ResourceRet,
-  DXILHandle,
-};
-
 enum class ResourceClass : uint8_t {
   SRV = 0,
   UAV,
diff --git a/llvm/lib/Target/DirectX/DXIL.td b/llvm/lib/Target/DirectX/DXIL.td
index 60185c20606b22b..34c7f84b1ca5b2e 100644
--- a/llvm/lib/Target/DirectX/DXIL.td
+++ b/llvm/lib/Target/DirectX/DXIL.td
@@ -24,19 +24,24 @@ foreach i = 0...8 in {
   def DXIL1_ #i : Version<1, i>;
 }
 
-// Overload type alias of llvm_any_ty
-defvar overloadTy = llvm_any_ty;
-
-// Type aliases for DXIL Op types to LLVM Types.
-// TODO: Define DXIL Types independent of LLVM types
-defvar i1Ty = llvm_i1_ty;
-defvar i8Ty = llvm_i8_ty;
-defvar i16Ty = llvm_i16_ty;
-defvar i32Ty = llvm_i32_ty;
-defvar i64Ty = llvm_i64_ty;
-defvar halfTy = llvm_half_ty;
-defvar floatTy = llvm_float_ty;
-defvar doubleTy = llvm_double_ty;
+class DXILOpParamType {
+  int isOverload = 0;
+}
+
+let isOverload = 1 in {
+  def OverloadTy : DXILOpParamType;
+}
+def VoidTy : DXILOpParamType;
+def Int1Ty : DXILOpParamType;
+def Int8Ty : DXILOpParamType;
+def Int16Ty : DXILOpParamType;
+def Int32Ty : DXILOpParamType;
+def Int64Ty : DXILOpParamType;
+def HalfTy : DXILOpParamType;
+def FloatTy : DXILOpParamType;
+def DoubleTy : DXILOpParamType;
+def ResRetTy : DXILOpParamType;
+def HandleTy : DXILOpParamType;
 
 class DXILOpClass;
 
@@ -268,9 +273,9 @@ def IsWave : DXILAttribute;
 def NeedsUniformInputs : DXILAttribute;
 def IsBarrier : DXILAttribute;
 
-class Overloads<Version ver, list<LLVMType> ols> {
+class Overloads<Version ver, list<DXILOpParamType> ols> {
   Version dxil_version = ver;
-  list<LLVMType> overload_types = ols;
+  list<DXILOpParamType> overload_types = ols;
 }
 
 class Stages<Version ver, list<DXILShaderStage> st> {
@@ -298,10 +303,10 @@ class DXILOp<int opcode, DXILOpClass opclass> {
   Intrinsic LLVMIntrinsic = ?;
 
   // Result type of the op
-  LLVMType result;
+  DXILOpParamType result;
 
   // List of argument types of the op. Default to 0 arguments.
-  list<LLVMType> arguments = [];
+  list<DXILOpParamType> arguments = [];
 
   // List of valid overload types predicated by DXIL version
   list<Overloads> overloads = [];
@@ -318,9 +323,9 @@ class DXILOp<int opcode, DXILOpClass opclass> {
 def Abs :  DXILOp<6, unary> {
   let Doc = "Returns the absolute value of the input.";
   let LLVMIntrinsic = int_fabs;
-  let arguments = [overloadTy];
-  let result = overloadTy;
-  let overloads = [Overloads<DXIL1_0, [halfTy, floatTy, doubleTy]>];
+  let arguments = [OverloadTy];
+  let result = OverloadTy;
+  let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy, DoubleTy]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -328,9 +333,9 @@ def Abs :  DXILOp<6, unary> {
 def IsInf :  DXILOp<9, isSpecialFloat> {
   let Doc = "Determines if the specified value is infinite.";
   let LLVMIntrinsic = int_dx_isinf;
-  let arguments = [overloadTy];
-  let result = i1Ty;
-  let overloads = [Overloads<DXIL1_0, [halfTy, floatTy]>];
+  let arguments = [OverloadTy];
+  let result = Int1Ty;
+  let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -338,9 +343,9 @@ def IsInf :  DXILOp<9, isSpecialFloat> {
 def Cos :  DXILOp<12, unary> {
   let Doc = "Returns cosine(theta) for theta in radians.";
   let LLVMIntrinsic = int_cos;
-  let arguments = [overloadTy];
-  let result = overloadTy;
-  let overloads = [Overloads<DXIL1_0, [halfTy, floatTy]>];
+  let arguments = [OverloadTy];
+  let result = OverloadTy;
+  let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -348,9 +353,9 @@ def Cos :  DXILOp<12, unary> {
 def Sin :  DXILOp<13, unary> {
   let Doc = "Returns sine(theta) for theta in radians.";
   let LLVMIntrinsic = int_sin;
-  let arguments = [overloadTy];
-  let result = overloadTy;
-  let overloads = [Overloads<DXIL1_0, [halfTy, floatTy]>];
+  let arguments = [OverloadTy];
+  let result = OverloadTy;
+  let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -358,9 +363,9 @@ def Sin :  DXILOp<13, unary> {
 def Tan :  DXILOp<14, unary> {
   let Doc = "Returns tangent(theta) for theta in radians.";
   let LLVMIntrinsic = int_tan;
-  let arguments = [overloadTy];
-  let result = overloadTy;
-  let overloads = [Overloads<DXIL1_0, [halfTy, floatTy]>];
+  let arguments = [OverloadTy];
+  let result = OverloadTy;
+  let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -368,9 +373,9 @@ def Tan :  DXILOp<14, unary> {
 def ACos :  DXILOp<15, unary> {
   let Doc = "Returns the arccosine of the specified value.";
   let LLVMIntrinsic = int_acos;
-  let arguments = [overloadTy];
-  let result = overloadTy;
-  let overloads = [Overloads<DXIL1_0, [halfTy, floatTy]>];
+  let arguments = [OverloadTy];
+  let result = OverloadTy;
+  let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -378,9 +383,9 @@ def ACos :  DXILOp<15, unary> {
 def ASin :  DXILOp<16, unary> {
   let Doc = "Returns the arcsine of the specified value.";
   let LLVMIntrinsic = int_asin;
-  let arguments = [overloadTy];
-  let result = overloadTy;
-  let overloads = [Overloads<DXIL1_0, [halfTy, floatTy]>];
+  let arguments = [OverloadTy];
+  let result = OverloadTy;
+  let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -388,9 +393,9 @@ def ASin :  DXILOp<16, unary> {
 def ATan :  DXILOp<17, unary> {
   let Doc = "Returns the arctangent of the specified value.";
   let LLVMIntrinsic = int_atan;
-  let arguments = [overloadTy];
-  let result = overloadTy;
-  let overloads = [Overloads<DXIL1_0, [halfTy, floatTy]>];
+  let arguments = [OverloadTy];
+  let result = OverloadTy;
+  let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -398,9 +403,9 @@ def ATan :  DXILOp<17, unary> {
 def HCos :  DXILOp<18, unary> {
   let Doc = "Returns the hyperbolic cosine of the specified value.";
   let LLVMIntrinsic = int_cosh;
-  let arguments = [overloadTy];
-  let result = overloadTy;
-  let overloads = [Overloads<DXIL1_0, [halfTy, floatTy]>];
+  let arguments = [OverloadTy];
+  let result = OverloadTy;
+  let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -408,9 +413,9 @@ def HCos :  DXILOp<18, unary> {
 def HSin :  DXILOp<19, unary> {
   let Doc = "Returns the hyperbolic sine of the specified value.";
   let LLVMIntrinsic = int_sinh;
-  let arguments = [overloadTy];
-  let result = overloadTy;
-  let overloads = [Overloads<DXIL1_0, [halfTy, floatTy]>];
+  let arguments = [OverloadTy];
+  let result = OverloadTy;
+  let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -418,9 +423,9 @@ def HSin :  DXILOp<19, unary> {
 def HTan :  DXILOp<20, unary> {
   let Doc = "Returns the hyperbolic tan of the specified value.";
   let LLVMIntrinsic = int_tanh;
-  let arguments = [overloadTy];
-  let result = overloadTy;
-  let overloads = [Overloads<DXIL1_0, [halfTy, floatTy]>];
+  let arguments = [OverloadTy];
+  let result = OverloadTy;
+  let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -429,9 +434,9 @@ def Exp2 :  DXILOp<21, unary> {
   let Doc = "Returns the base 2 exponential, or 2**x, of the specified value. "
             "exp2(x) = 2**x.";
   let LLVMIntrinsic = int_exp2;
-  let arguments = [overloadTy];
-  let result = overloadTy;
-  let overloads = [Overloads<DXIL1_0, [halfTy, floatTy]>];
+  let arguments = [OverloadTy];
+  let result = OverloadTy;
+  let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -440,9 +445,9 @@ def Frac :  DXILOp<22, unary> {
   let Doc = "Returns a fraction from 0 to 1 that represents the decimal part "
             "of the input.";
   let LLVMIntrinsic = int_dx_frac;
-  let arguments = [overloadTy];
-  let result = overloadTy;
-  let overloads = [Overloads<DXIL1_0, [halfTy, floatTy]>];
+  let arguments = [OverloadTy];
+  let result = OverloadTy;
+  let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -450,9 +455,9 @@ def Frac :  DXILOp<22, unary> {
 def Log2 :  DXILOp<23, unary> {
   let Doc = "Returns the base-2 logarithm of the specified value.";
   let LLVMIntrinsic = int_log2;
-  let arguments = [overloadTy];
-  let result = overloadTy;
-  let overloads = [Overloads<DXIL1_0, [halfTy, floatTy]>];
+  let arguments = [OverloadTy];
+  let result = OverloadTy;
+  let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -461,9 +466,9 @@ def Sqrt :  DXILOp<24, unary> {
   let Doc = "Returns the square root of the specified floating-point value, "
             "per component.";
   let LLVMIntrinsic = int_sqrt;
-  let arguments = [overloadTy];
-  let result = overloadTy;
-  let overloads = [Overloads<DXIL1_0, [halfTy, floatTy]>];
+  let arguments = [OverloadTy];
+  let result = OverloadTy;
+  let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -472,9 +477,9 @@ def RSqrt :  DXILOp<25, unary> {
   let Doc = "Returns the reciprocal of the square root of the specified value. "
             "rsqrt(x) = 1 / sqrt(x).";
   let LLVMIntrinsic = int_dx_rsqrt;
-  let arguments = [overloadTy];
-  let result = overloadTy;
-  let overloads = [Overloads<DXIL1_0, [halfTy, floatTy]>];
+  let arguments = [OverloadTy];
+  let result = OverloadTy;
+  let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -483,9 +488,9 @@ def Round :  DXILOp<26, unary> {
   let Doc = "Returns the input rounded to the nearest integer within a "
             "floating-point type.";
   let LLVMIntrinsic = int_roundeven;
-  let arguments = [overloadTy];
-  let result = overloadTy;
-  let overloads = [Overloads<DXIL1_0, [halfTy, floatTy]>];
+  let arguments = [OverloadTy];
+  let result = OverloadTy;
+  let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -494,9 +499,9 @@ def Floor :  DXILOp<27, unary> {
   let Doc =
       "Returns the largest integer that is less than or equal to the input.";
   let LLVMIntrinsic = int_floor;
-  let arguments = [overloadTy];
-  let result = overloadTy;
-  let overloads = [Overloads<DXIL1_0, [halfTy, floatTy]>];
+  let arguments = [OverloadTy];
+  let result = OverloadTy;
+  let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -505,9 +510,9 @@ def Ceil :  DXILOp<28, unary> {
   let Doc = "Returns the smallest integer that is greater than or equal to the "
             "input.";
   let LLVMIntrinsic = int_ceil;
-  let arguments = [overloadTy];
-  let result = overloadTy;
-  let overloads = [Overloads<DXIL1_0, [halfTy, floatTy]>];
+  let arguments = [OverloadTy];
+  let result = OverloadTy;
+  let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -515,9 +520,9 @@ def Ceil :  DXILOp<28, unary> {
 def Trunc :  DXILOp<29, unary> {
   let Doc = "Returns the specified value truncated to the integer component.";
   let LLVMIntrinsic = int_trunc;
-  let arguments = [overloadTy];
-  let result = overloadTy;
-  let overloads = [Overloads<DXIL1_0, [halfTy, floatTy]>];
+  let arguments = [OverloadTy];
+  let result = OverloadTy;
+  let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -525,10 +530,10 @@ def Trunc :  DXILOp<29, unary> {
 def Rbits :  DXILOp<30, unary> {
   let Doc = "Returns the specified value with its bits reversed.";
   let LLVMIntrinsic = int_bitreverse;
-  let arguments = [overloadTy];
-  let result = overloadTy;
+  let arguments = [OverloadTy];
+  let result = OverloadTy;
   let overloads =
-      [Overloads<DXIL1_0, [i16Ty, i32Ty, i64Ty]>];
+      [Overloads<DXIL1_0, [Int16Ty, Int32Ty, Int64Ty]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -536,10 +541,10 @@ def Rbits :  DXILOp<30, unary> {
 def FMax :  DXILOp<35, binary> {
   let Doc = "Float maximum. FMax(a,b) = a > b ? a : b";
   let LLVMIntrinsic = int_maxnum;
-  let arguments = [overloadTy, overloadTy];
-  let result = overloadTy;
+  let arguments = [OverloadTy, OverloadTy];
+  let result = OverloadTy;
   let overloads =
-      [Overloads<DXIL1_0, [halfTy, floatTy, doubleTy]>];
+      [Overloads<DXIL1_0, [HalfTy, FloatTy, DoubleTy]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -547,10 +552,10 @@ def FMax :  DXILOp<35, binary> {
 def FMin :  DXILOp<36, binary> {
   let Doc = "Float minimum. FMin(a,b) = a < b ? a : b";
   let LLVMIntrinsic = int_minnum;
-  let arguments = [overloadTy, overloadTy];
-  let result = overloadTy;
+  let arguments = [OverloadTy, OverloadTy];
+  let result = OverloadTy;
   let overloads =
-      [Overloads<DXIL1_0, [halfTy, floatTy, doubleTy]>];
+      [Overloads<DXIL1_0, [HalfTy, FloatTy, DoubleTy]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -558,10 +563,10 @@ def FMin :  DXILOp<36, binary> {
 def SMax :  DXILOp<37, binary> {
   let Doc = "Signed integer maximum. SMax(a,b) = a > b ? a : b";
   let LLVMIntrinsic = int_smax;
-  let arguments = [overloadTy, overloadTy];
-  let result = overloadTy;
+  let arguments = [OverloadTy, OverloadTy];
+  let result = OverloadTy;
   let overloads =
-      [Overloads<DXIL1_0, [i16Ty, i32Ty, i64Ty]>];
+      [Overloads<DXIL1_0, [Int16Ty, Int32Ty, Int64Ty]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -569,10 +574,10 @@ def SMax :  DXILOp<37, binary> {
 def SMin :  DXILOp<38, binary> {
   let Doc = "Signed integer minimum. SMin(a,b) = a < b ? a : b";
   let LLVMIntrinsic = int_smin;
-  let arguments = [overloadTy, overloadTy];
-  let result = overloadTy;
+  let arguments = [OverloadTy, OverloadTy];
+  let result = OverloadTy;
   let overloads =
-      [Overloads<DXIL1_0, [i16Ty, i32Ty, i64Ty]>];
+      [Overloads<DXIL1_0, [Int16Ty, Int32Ty, Int64Ty]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -580,10 +585,10 @@ def SMin :  DXILOp<38, binary> {
 def UMax :  DXILOp<39, binary> {
   let Doc = "Unsigned integer maximum. UMax(a,b) = a > b ? a : b";
   let LLVMIntrinsic = int_umax;
-  let arguments = [overloadTy, overloadTy];
-  let result = overloadTy;
+  let arguments = [OverloadTy, OverloadTy];
+  let result = OverloadTy;
   let overloads =
-      [Overloads<DXIL1_0, [i16Ty, i32Ty, i64Ty]>];
+      [Overloads<DXIL1_0, [Int16Ty, Int32Ty, Int64Ty]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -591,10 +596,10 @@ def UMax :  DXILOp<39, binary> {
 def UMin :  DXILOp<40, binary> {
   let Doc = "Unsigned integer minimum. UMin(a,b) = a < b ? a : b";
   let LLVMIntrinsic = int_umin;
-  let arguments = [overloadTy, overloadTy];
-  let result = overloadTy;
+  let arguments = [OverloadTy, OverloadTy];
+  let result = OverloadTy;
   let overloads =
-      [Overloads<DXIL1_0, [i16Ty, i32Ty, i64Ty]>];
+      [Overloads<DXIL1_0, [Int16Ty, Int32Ty, Int64Ty]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -603,10 +608,10 @@ def FMad :  DXILOp<46, tertiary> {
   let Doc = "Floating point arithmetic multiply/add operation. fmad(m,a,b) = m "
             "* a + b.";
   let LLVMIntrinsic = int_fmuladd;
-  let arguments = [overloadTy, overloadTy, overloadTy];
-  let result = overloadTy;
+  let arguments = [OverloadTy, OverloadTy, OverloadTy];
+  let result = OverloadTy;
   let overloads =
-      [Overloads<DXIL1_0, [halfTy, floatTy, doubleTy]>];
+      [Overloads<DXIL1_0, [HalfTy, FloatTy, DoubleTy]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -615,10 +620,10 @@ def IMad :  DXILOp<48, tertiary> {
   let Doc = "Signed integer arithmetic multiply/add operation. imad(m,a,b) = m "
             "* a + b.";
   let LLVMIntrinsic = int_dx_imad;
-  let arguments = [overloadTy, overloadTy, overloadTy];
-  let result = overloadTy;
+  let arguments = [OverloadTy, OverloadTy, OverloadTy];
+  let result = OverloadTy;
   let overloads =
-      [Overloads<DXIL1_0, [i16Ty, i32Ty, i64Ty]>];
+      [Overloads<DXIL1_0, [Int16Ty, Int32Ty, Int64Ty]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -627,10 +632,10 @@ def UMad :  DXILOp<49, tertiary> {
   let Doc = "Unsigned integer arithmetic multiply/add operation. umad(m,a, = m "
             "* a + b.";
   let LLVMIntrinsic = int_dx_umad;
-  let arguments = [overloadTy, overloadTy, overloadTy];
-  let result = overloadTy;
+  let arguments = [OverloadTy, OverloadTy, OverloadTy];
+  let result = OverloadTy;
   let overloads =
-      [Overloads<DXIL1_0, [i16Ty, i32Ty, i64Ty]>];
+      [Overloads<DXIL1_0, [Int16Ty, Int32Ty, Int64Ty]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -639,9 +644,9 @@ def Dot2 :  DXILOp<54, dot2> {
   let Doc = "dot product of two float vectors Dot(a,b) = a[0]*b[0] + ... + "
             "a[n]*b[n] where n is between 0 and 1";
   let LLVMIntrinsic = int_dx_dot2;
-  let arguments = !listsplat(overloadTy, 4);
-  let result = overloadTy;
-  let overloads = [Overloads<DXIL1_0, [halfTy, floatTy]>];
+  let arguments = !listsplat(OverloadTy, 4);
+  let result = OverloadTy;
+  let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -650,9 +655,9 @@ def Dot3 :  DXILOp<55, dot3> {
   let Doc = "dot product of two float vectors Dot(a,b) = a[0]*b[0] + ... + "
             "a[n]*b[n] where n is between 0 and 2";
   let LLVMIntrinsic = int_dx_dot3;
-  let arguments = !listsplat(overloadTy, 6);
-  let result = overloadTy;
-  let overloads = [Overloads<DXIL1_0, [halfTy, floatTy]>];
+  let arguments = !listsplat(OverloadTy, 6);
+  let result = OverloadTy;
+  let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -661,9 +666,9 @@ def Dot4 :  DXILOp<56, dot4> {
   let Doc = "dot product of two float vectors Dot(a,b) = a[0]*b[0] + ... + "
             "a[n]*b[n] where n is between 0 and 3";
   let LLVMIntrinsic = int_dx_dot4;
-  let arguments = !listsplat(overloadTy, 8);
-  let result = overloadTy;
-  let overloads = [Overloads<DXIL1_0, [halfTy, floatTy]>];
+  let arguments = !listsplat(OverloadTy, 8);
+  let result = OverloadTy;
+  let overloads = [Overloads<DXIL1_0, [HalfTy, FloatTy]>];
   let stages = [Stages<DXIL1_0, [all_stages]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -671,9 +676,9 @@ def Dot4 :  DXILOp<56, dot4> {
 def ThreadId :  DXILOp<93, threadId> {
   let Doc = "Reads the thread ID";
   let LLVMIntrinsic = int_dx_thread_id;
-  let arguments = [overloadTy];
-  let result = overloadTy;
-  let overloads = [Overloads<DXIL1_0, [i32Ty]>];
+  let arguments = [OverloadTy];
+  let result = OverloadTy;
+  let overloads = [Overloads<DXIL1_0, [Int32Ty]>];
   let stages = [Stages<DXIL1_0, [compute, mesh, amplification, node]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -681,9 +686,9 @@ def ThreadId :  DXILOp<93, threadId> {
 def GroupId :  DXILOp<94, groupId> {
   let Doc = "Reads the group ID (SV_GroupID)";
   let LLVMIntrinsic = int_dx_group_id;
-  let arguments = [overloadTy];
-  let result = overloadTy;
-  let overloads = [Overloads<DXIL1_0, [i32Ty]>];
+  let arguments = [OverloadTy];
+  let result = OverloadTy;
+  let overloads = [Overloads<DXIL1_0, [Int32Ty]>];
   let stages = [Stages<DXIL1_0, [compute, mesh, amplification, node]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -691,9 +696,9 @@ def GroupId :  DXILOp<94, groupId> {
 def ThreadIdInGroup :  DXILOp<95, threadIdInGroup> {
   let Doc = "Reads the thread ID within the group  (SV_GroupThreadID)";
   let LLVMIntrinsic = int_dx_thread_id_in_group;
-  let arguments = [overloadTy];
-  let result = overloadTy;
-  let overloads = [Overloads<DXIL1_0, [i32Ty]>];
+  let arguments = [OverloadTy];
+  let result = OverloadTy;
+  let overloads = [Overloads<DXIL1_0, [Int32Ty]>];
   let stages = [Stages<DXIL1_0, [compute, mesh, amplification, node]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
@@ -702,8 +707,8 @@ def FlattenedThreadIdInGroup :  DXILOp<96, flattenedThreadIdInGroup> {
   let Doc = "Provides a flattened index for a given thread within a given "
             "group (SV_GroupIndex)";
   let LLVMIntrinsic = int_dx_flattened_thread_id_in_group;
-  let result = overloadTy;
-  let overloads = [Overloads<DXIL1_0, [i32Ty]>];
+  let result = OverloadTy;
+  let overloads = [Overloads<DXIL1_0, [Int32Ty]>];
   let stages = [Stages<DXIL1_0, [compute, mesh, amplification, node]>];
   let attributes = [Attributes<DXIL1_0, [ReadNone]>];
 }
diff --git a/llvm/lib/Target/DirectX/DXILConstants.h b/llvm/lib/Target/DirectX/DXILConstants.h
index 0c9c1ac38fdbceb..022cd57795a0633 100644
--- a/llvm/lib/Target/DirectX/DXILConstants.h
+++ b/llvm/lib/Target/DirectX/DXILConstants.h
@@ -25,6 +25,11 @@ enum class OpCodeClass : unsigned {
 #include "DXILOperation.inc"
 };
 
+enum class OpParamType : unsigned {
+#define DXIL_OP_PARAM_TYPE(Name) Name,
+#include "DXILOperation.inc"
+};
+
 } // namespace dxil
 } // namespace llvm
 
diff --git a/llvm/lib/Target/DirectX/DXILOpBuilder.cpp b/llvm/lib/Target/DirectX/DXILOpBuilder.cpp
index 0e2b4601112b5fa..ca372026141fbf3 100644
--- a/llvm/lib/Target/DirectX/DXILOpBuilder.cpp
+++ b/llvm/lib/Target/DirectX/DXILOpBuilder.cpp
@@ -157,10 +157,8 @@ struct OpCodeProperty {
   llvm::SmallVector<OpOverload> Overloads;
   llvm::SmallVector<OpStage> Stages;
   llvm::SmallVector<OpAttribute> Attributes;
-  int OverloadParamIndex;        // parameter index which control the overload.
-                                 // When < 0, should be only 1 overload type.
-  unsigned NumOfParameters;      // Number of parameters include return value.
-  unsigned ParameterTableOffset; // Offset in ParameterTable.
+  int OverloadParamIndex; // parameter index which control the overload.
+                          // When < 0, should be only 1 overload type.
 };
 
 // Include getOpCodeClassName getOpCodeProperty, getOpCodeName and
@@ -211,35 +209,33 @@ static StructType *getHandleType(LLVMContext &Ctx) {
                                Ctx);
 }
 
-static Type *getTypeFromParameterKind(ParameterKind Kind, LLVMContext &Ctx,
-                                      Type *OverloadTy) {
+static Type *getTypeFromOpParamType(OpParamType Kind, LLVMContext &Ctx,
+                                    Type *OverloadTy) {
   switch (Kind) {
-  case ParameterKind::Void:
+  case OpParamType::VoidTy:
     return Type::getVoidTy(Ctx);
-  case ParameterKind::Half:
+  case OpParamType::HalfTy:
     return Type::getHalfTy(Ctx);
-  case ParameterKind::Float:
+  case OpParamType::FloatTy:
     return Type::getFloatTy(Ctx);
-  case ParameterKind::Double:
+  case OpParamType::DoubleTy:
     return Type::getDoubleTy(Ctx);
-  case ParameterKind::I1:
+  case OpParamType::Int1Ty:
     return Type::getInt1Ty(Ctx);
-  case ParameterKind::I8:
+  case OpParamType::Int8Ty:
     return Type::getInt8Ty(Ctx);
-  case ParameterKind::I16:
+  case OpParamType::Int16Ty:
     return Type::getInt16Ty(Ctx);
-  case ParameterKind::I32:
+  case OpParamType::Int32Ty:
     return Type::getInt32Ty(Ctx);
-  case ParameterKind::I64:
+  case OpParamType::Int64Ty:
     return Type::getInt64Ty(Ctx);
-  case ParameterKind::Overload:
+  case OpParamType::OverloadTy:
     return OverloadTy;
-  case ParameterKind::ResourceRet:
+  case OpParamType::ResRetTy:
     return getResRetType(OverloadTy, Ctx);
-  case ParameterKind::DXILHandle:
+  case OpParamType::HandleTy:
     return getHandleType(Ctx);
-  default:
-    break;
   }
   llvm_unreachable("Invalid parameter kind");
   return nullptr;
@@ -284,30 +280,34 @@ static ShaderKind getShaderKindEnum(Triple::EnvironmentType EnvType) {
       "Shader Kind Not Found - Invalid DXIL Environment Specified");
 }
 
+static SmallVector<Type *>
+getArgTypesFromOpParamTypes(ArrayRef<dxil::OpParamType> Types,
+                            LLVMContext &Context, Type *OverloadTy) {
+  SmallVector<Type *> ArgTys;
+  ArgTys.emplace_back(Type::getInt32Ty(Context));
+  for (dxil::OpParamType Ty : Types)
+    ArgTys.emplace_back(getTypeFromOpParamType(Ty, Context, OverloadTy));
+  return ArgTys;
+}
+
 /// Construct DXIL function type. This is the type of a function with
 /// the following prototype
 ///     OverloadType dx.op.<opclass>.<return-type>(int opcode, <param types>)
 /// <param-types> are constructed from types in Prop.
-static FunctionType *getDXILOpFunctionType(const OpCodeProperty *Prop,
+static FunctionType *getDXILOpFunctionType(dxil::OpCode OpCode,
                                            LLVMContext &Context,
                                            Type *OverloadTy) {
-  SmallVector<Type *> ArgTys;
-
-  const ParameterKind *ParamKinds = getOpCodeParameterKind(*Prop);
-
-  assert(Prop->NumOfParameters && "No return type?");
-  // Add return type of the function
-  Type *ReturnTy = getTypeFromParameterKind(ParamKinds[0], Context, OverloadTy);
 
-  // Add DXIL Opcode value type viz., Int32 as first argument
-  ArgTys.emplace_back(Type::getInt32Ty(Context));
-
-  // Add DXIL Operation parameter types as specified in DXIL properties
-  for (unsigned I = 1; I < Prop->NumOfParameters; ++I) {
-    ParameterKind Kind = ParamKinds[I];
-    ArgTys.emplace_back(getTypeFromParameterKind(Kind, Context, OverloadTy));
+  switch (OpCode) {
+#define DXIL_OP_FUNCTION_TYPE(OpCode, RetType, ...)                            \
+  case OpCode:                                                                 \
+    return FunctionType::get(                                                  \
+        getTypeFromOpParamType(RetType, Context, OverloadTy),                  \
+        getArgTypesFromOpParamTypes({__VA_ARGS__}, Context, OverloadTy),       \
+        /*isVarArg=*/false);
+#include "DXILOperation.inc"
   }
-  return FunctionType::get(ReturnTy, ArgTys, /*isVarArg=*/false);
+  llvm_unreachable("Invalid OpCode?");
 }
 
 /// Get index of the property from PropList valid for the most recent
@@ -372,7 +372,7 @@ Expected<CallInst *> DXILOpBuilder::tryCreateOp(dxil::OpCode OpCode,
     OverloadTy = Args[ArgIndex]->getType();
   }
   FunctionType *DXILOpFT =
-      getDXILOpFunctionType(Prop, M.getContext(), OverloadTy);
+      getDXILOpFunctionType(OpCode, M.getContext(), OverloadTy);
 
   std::optional<size_t> OlIndexOrErr =
       getPropIndex(ArrayRef(Prop->Overloads), DXILVersion);
diff --git a/llvm/utils/TableGen/DXILEmitter.cpp b/llvm/utils/TableGen/DXILEmitter.cpp
index 53791618e80fe06..9cc1b5ccb8acb95 100644
--- a/llvm/utils/TableGen/DXILEmitter.cpp
+++ b/llvm/utils/TableGen/DXILEmitter.cpp
@@ -18,7 +18,7 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringSet.h"
-#include "llvm/CodeGenTypes/MachineValueType.h"
+#include "llvm/ADT/StringSwitch.h"
 #include "llvm/Support/DXILABI.h"
 #include "llvm/Support/VersionTuple.h"
 #include "llvm/TableGen/Error.h"
@@ -54,40 +54,6 @@ struct DXILOperationDesc {
 };
 } // end anonymous namespace
 
-/// Return dxil::ParameterKind corresponding to input LLVMType record
-///
-/// \param R TableGen def record of class LLVMType
-/// \return ParameterKind As defined in llvm/Support/DXILABI.h
-
-static ParameterKind getParameterKind(const Record *R) {
-  auto VTRec = R->getValueAsDef("VT");
-  switch (getValueType(VTRec)) {
-  case MVT::isVoid:
-    return ParameterKind::Void;
-  case MVT::f16:
-    return ParameterKind::Half;
-  case MVT::f32:
-    return ParameterKind::Float;
-  case MVT::f64:
-    return ParameterKind::Double;
-  case MVT::i1:
-    return ParameterKind::I1;
-  case MVT::i8:
-    return ParameterKind::I8;
-  case MVT::i16:
-    return ParameterKind::I16;
-  case MVT::i32:
-    return ParameterKind::I32;
-  case MVT::fAny:
-  case MVT::iAny:
-  case MVT::Any:
-    return ParameterKind::Overload;
-  default:
-    llvm_unreachable(
-        "Support for specified parameter type not yet implemented");
-  }
-}
-
 /// In-place sort TableGen records of class with a field
 ///    Version dxil_version
 /// in the ascending version order.
@@ -134,10 +100,9 @@ DXILOperationDesc::DXILOperationDesc(const Record *R) {
   // llvm/IR/Intrinsics.td
   OverloadParamIndex = -1; // A sigil meaning none.
   for (unsigned i = 0; i < ParamTypeRecsSize; i++) {
-    auto TR = ParamTypeRecs[i];
+    Record *TR = ParamTypeRecs[i];
     // Track operation parameter indices of any overload types
-    auto isAny = TR->getValueAsInt("isAny");
-    if (isAny == 1) {
+    if (TR->getValueAsInt("isOverload")) {
       if (OverloadParamIndex != -1) {
         assert(TR == ParamTypeRecs[OverloadParamIndex] &&
                "Specification of multiple differing overload parameter types "
@@ -148,8 +113,6 @@ DXILOperationDesc::DXILOperationDesc(const Record *R) {
       if (OverloadParamIndex <= 0)
         OverloadParamIndex = i;
     }
-    if (TR->isAnonymous())
-      PrintFatalError(TR, "Only concrete types are allowed here");
     OpTypes.emplace_back(TR);
   }
 
@@ -208,71 +171,23 @@ DXILOperationDesc::DXILOperationDesc(const Record *R) {
   }
 }
 
-/// Return a string representation of ParameterKind enum
-/// \param Kind Parameter Kind enum value
-/// \return std::string string representation of input Kind
-static std::string getParameterKindStr(ParameterKind Kind) {
-  switch (Kind) {
-  case ParameterKind::Invalid:
-    return "Invalid";
-  case ParameterKind::Void:
-    return "Void";
-  case ParameterKind::Half:
-    return "Half";
-  case ParameterKind::Float:
-    return "Float";
-  case ParameterKind::Double:
-    return "Double";
-  case ParameterKind::I1:
-    return "I1";
-  case ParameterKind::I8:
-    return "I8";
-  case ParameterKind::I16:
-    return "I16";
-  case ParameterKind::I32:
-    return "I32";
-  case ParameterKind::I64:
-    return "I64";
-  case ParameterKind::Overload:
-    return "Overload";
-  case ParameterKind::CBufferRet:
-    return "CBufferRet";
-  case ParameterKind::ResourceRet:
-    return "ResourceRet";
-  case ParameterKind::DXILHandle:
-    return "DXILHandle";
-  }
-  llvm_unreachable("Unknown llvm::dxil::ParameterKind enum");
-}
-
 /// Return a string representation of OverloadKind enum that maps to
 /// input LLVMType record
 /// \param R TableGen def record of class LLVMType
 /// \return std::string string representation of OverloadKind
 
-static std::string getOverloadKindStr(const Record *R) {
-  Record *VTRec = R->getValueAsDef("VT");
-  switch (getValueType(VTRec)) {
-  case MVT::f16:
-    return "OverloadKind::HALF";
-  case MVT::f32:
-    return "OverloadKind::FLOAT";
-  case MVT::f64:
-    return "OverloadKind::DOUBLE";
-  case MVT::i1:
-    return "OverloadKind::I1";
-  case MVT::i8:
-    return "OverloadKind::I8";
-  case MVT::i16:
-    return "OverloadKind::I16";
-  case MVT::i32:
-    return "OverloadKind::I32";
-  case MVT::i64:
-    return "OverloadKind::I64";
-  default:
-    llvm_unreachable("Support for specified fixed type option for overload "
-                     "type not supported");
-  }
+static StringRef getOverloadKindStr(const Record *R) {
+  // TODO: This is a hack. We need to rework how we're handling the set of
+  // overloads to avoid this business with the separate OverloadKind enum.
+  return StringSwitch<StringRef>(R->getName())
+      .Case("HalfTy", "OverloadKind::HALF")
+      .Case("FloatTy", "OverloadKind::FLOAT")
+      .Case("DoubleTy", "OverloadKind::DOUBLE")
+      .Case("Int1Ty", "OverloadKind::I1")
+      .Case("Int8Ty", "OverloadKind::I8")
+      .Case("Int16Ty", "OverloadKind::I16")
+      .Case("Int32Ty", "OverloadKind::I32")
+      .Case("Int64Ty", "OverloadKind::I64");
 }
 
 /// Return a string representation of valid overload information denoted
@@ -417,8 +332,7 @@ static void emitDXILOpCodes(std::vector<DXILOperationDesc> &Ops,
 }
 
 /// Emit a list of DXIL op classes
-static void emitDXILOpClasses(RecordKeeper &Records,
-                              raw_ostream &OS) {
+static void emitDXILOpClasses(RecordKeeper &Records, raw_ostream &OS) {
   OS << "#ifdef DXIL_OPCLASS\n";
   std::vector<Record *> OpClasses =
       Records.getAllDerivedDefinitions("DXILOpClass");
@@ -428,6 +342,35 @@ static void emitDXILOpClasses(RecordKeeper &Records,
   OS << "#endif\n\n";
 }
 
+/// Emit a list of DXIL op parameter types
+static void emitDXILOpParamTypes(RecordKeeper &Records, raw_ostream &OS) {
+  OS << "#ifdef DXIL_OP_PARAM_TYPE\n";
+  std::vector<Record *> OpClasses =
+      Records.getAllDerivedDefinitions("DXILOpParamType");
+  for (Record *OpClass : OpClasses)
+    OS << "DXIL_OP_PARAM_TYPE(" << OpClass->getName() << ")\n";
+  OS << "#undef DXIL_OP_PARAM_TYPE\n";
+  OS << "#endif\n\n";
+}
+
+/// Emit a list of DXIL op function types
+static void emitDXILOpFunctionTypes(ArrayRef<DXILOperationDesc> Ops,
+                                    raw_ostream &OS) {
+  OS << "#ifndef DXIL_OP_FUNCTION_TYPE\n";
+  OS << "#define DXIL_OP_FUNCTION_TYPE(OpCode, RetType, ...)\n";
+  OS << "#endif\n";
+  for (const DXILOperationDesc &Op : Ops) {
+    OS << "DXIL_OP_FUNCTION_TYPE(dxil::OpCode::" << Op.OpName;
+    for (const Record *Rec : Op.OpTypes)
+      OS << ", dxil::OpParamType::" << Rec->getName();
+    // If there are no arguments, we need an empty comma for the varargs
+    if (Op.OpTypes.size() == 1)
+      OS << ", ";
+    OS << ")\n";
+  }
+  OS << "#undef DXIL_OP_FUNCTION_TYPE\n";
+}
+
 /// Emit map of DXIL operation to LLVM or DirectX intrinsic
 /// \param A vector of DXIL Ops
 /// \param Output stream
@@ -454,9 +397,7 @@ static void emitDXILOperationTable(std::vector<DXILOperationDesc> &Ops,
   // Collect Names.
   SequenceToOffsetTable<std::string> OpClassStrings;
   SequenceToOffsetTable<std::string> OpStrings;
-  SequenceToOffsetTable<SmallVector<ParameterKind>> Parameters;
 
-  StringMap<SmallVector<ParameterKind>> ParameterMap;
   StringSet<> ClassSet;
   for (auto &Op : Ops) {
     OpStrings.add(Op.OpName);
@@ -465,18 +406,11 @@ static void emitDXILOperationTable(std::vector<DXILOperationDesc> &Ops,
       continue;
     ClassSet.insert(Op.OpClass);
     OpClassStrings.add(Op.OpClass.data());
-    SmallVector<ParameterKind> ParamKindVec;
-    for (unsigned i = 0; i < Op.OpTypes.size(); i++) {
-      ParamKindVec.emplace_back(getParameterKind(Op.OpTypes[i]));
-    }
-    ParameterMap[Op.OpClass] = ParamKindVec;
-    Parameters.add(ParamKindVec);
   }
 
   // Layout names.
   OpStrings.layout();
   OpClassStrings.layout();
-  Parameters.layout();
 
   // Emit access function getOpcodeProperty() that embeds DXIL Operation table
   // with entries of type struct OpcodeProperty.
@@ -492,8 +426,7 @@ static void emitDXILOperationTable(std::vector<DXILOperationDesc> &Ops,
        << getOverloadMaskString(Op.OverloadRecs) << ", "
        << getStageMaskString(Op.StageRecs) << ", "
        << getAttributeMaskString(Op.AttrRecs) << ", " << Op.OverloadParamIndex
-       << ", " << Op.OpTypes.size() << ", "
-       << Parameters.get(ParameterMap[Op.OpClass]) << " }";
+       << " }";
     Prefix = ",\n";
   }
   OS << "  };\n";
@@ -531,21 +464,6 @@ static void emitDXILOperationTable(std::vector<DXILOperationDesc> &Ops,
 
   OS << "  unsigned Index = Prop.OpCodeClassNameOffset;\n";
   OS << "  return DXILOpCodeClassNameTable + Index;\n";
-  OS << "}\n ";
-
-  OS << "static const ParameterKind *getOpCodeParameterKind(const "
-        "OpCodeProperty &Prop) "
-        "{\n\n";
-  OS << "  static const ParameterKind DXILOpParameterKindTable[] = {\n";
-  Parameters.emit(
-      OS,
-      [](raw_ostream &ParamOS, ParameterKind Kind) {
-        ParamOS << "ParameterKind::" << getParameterKindStr(Kind);
-      },
-      "ParameterKind::Invalid");
-  OS << "  };\n\n";
-  OS << "  unsigned Index = Prop.ParameterTableOffset;\n";
-  OS << "  return DXILOpParameterKindTable + Index;\n";
   OS << "}\n\n";
 }
 
@@ -611,6 +529,8 @@ static void EmitDXILOperation(RecordKeeper &Records, raw_ostream &OS) {
 
   emitDXILOpCodes(DXILOps, OS);
   emitDXILOpClasses(Records, OS);
+  emitDXILOpParamTypes(Records, OS);
+  emitDXILOpFunctionTypes(DXILOps, OS);
   emitDXILIntrinsicMap(DXILOps, OS);
   OS << "#ifdef DXIL_OP_OPERATION_TABLE\n\n";
   emitDXILOperationTableDataStructs(Records, OS);

From 340fb6597116acf026a8ac49edca4d993fb1d7f3 Mon Sep 17 00:00:00 2001
From: Rahul Joshi <rjoshi@nvidia.com>
Date: Mon, 19 Aug 2024 10:15:52 -0700
Subject: [PATCH 389/441] [TableGen] Detect invalid -D arguments and fail
 (#102813)

- Detect invalid macro names specified on command line and fail if one
found.
- Specifically, -DXYZ=1 for example, will fail instead is being silently
accepted.
---
 llvm/lib/TableGen/TGLexer.cpp                 | 44 +++++++++++++------
 .../invalid-macro-name-command-line.td        |  9 ++++
 2 files changed, 40 insertions(+), 13 deletions(-)
 create mode 100644 llvm/test/TableGen/invalid-macro-name-command-line.td

diff --git a/llvm/lib/TableGen/TGLexer.cpp b/llvm/lib/TableGen/TGLexer.cpp
index 0554f0cf5788310..c1fae91e1fd9f2d 100644
--- a/llvm/lib/TableGen/TGLexer.cpp
+++ b/llvm/lib/TableGen/TGLexer.cpp
@@ -44,6 +44,25 @@ constexpr PreprocessorDir PreprocessorDirs[] = {{tgtok::Ifdef, "ifdef"},
                                                 {tgtok::Endif, "endif"},
                                                 {tgtok::Define, "define"}};
 
+// Returns a pointer past the end of a valid macro name at the start of `Str`.
+// Valid macro names match the regular expression [a-zA-Z_][0-9a-zA-Z_]*.
+static const char *lexMacroName(StringRef Str) {
+  assert(!Str.empty());
+
+  // Macro names start with [a-zA-Z_].
+  const char *Next = Str.begin();
+  if (*Next != '_' && !isalpha(*Next))
+    return Next;
+  // Eat the first character of the name.
+  ++Next;
+
+  // Match the rest of the identifier regex: [0-9a-zA-Z_]*
+  const char *End = Str.end();
+  while (Next != End && (isalpha(*Next) || isdigit(*Next) || *Next == '_'))
+    ++Next;
+  return Next;
+}
+
 TGLexer::TGLexer(SourceMgr &SM, ArrayRef<std::string> Macros) : SrcMgr(SM) {
   CurBuffer = SrcMgr.getMainFileID();
   CurBuf = SrcMgr.getMemoryBuffer(CurBuffer)->getBuffer();
@@ -54,9 +73,16 @@ TGLexer::TGLexer(SourceMgr &SM, ArrayRef<std::string> Macros) : SrcMgr(SM) {
   PrepIncludeStack.push_back(
       std::make_unique<std::vector<PreprocessorControlDesc>>());
 
-  // Put all macros defined in the command line into the DefinedMacros set.
-  for (const std::string &MacroName : Macros)
+  // Add all macros defined on the command line to the DefinedMacros set.
+  // Check invalid macro names and print fatal error if we find one.
+  for (StringRef MacroName : Macros) {
+    const char *End = lexMacroName(MacroName);
+    if (End != MacroName.end())
+      PrintFatalError("Invalid macro name `" + MacroName +
+                      "` specified on command line");
+
     DefinedMacros.insert(MacroName);
+  }
 }
 
 SMLoc TGLexer::getLoc() const {
@@ -699,9 +725,8 @@ bool TGLexer::prepEatPreprocessorDirective(tgtok::TokKind Kind) {
   return false;
 }
 
-tgtok::TokKind TGLexer::lexPreprocessor(
-    tgtok::TokKind Kind, bool ReturnNextLiveToken) {
-
+tgtok::TokKind TGLexer::lexPreprocessor(tgtok::TokKind Kind,
+                                        bool ReturnNextLiveToken) {
   // We must be looking at a preprocessing directive.  Eat it!
   if (!prepEatPreprocessorDirective(Kind))
     PrintFatalError("lexPreprocessor() called for unknown "
@@ -901,14 +926,7 @@ StringRef TGLexer::prepLexMacroName() {
     ++CurPtr;
 
   TokStart = CurPtr;
-  // Macro names start with [a-zA-Z_].
-  if (*CurPtr != '_' && !isalpha(*CurPtr))
-    return "";
-
-  // Match the rest of the identifier regex: [0-9a-zA-Z_]*
-  while (isalpha(*CurPtr) || isdigit(*CurPtr) || *CurPtr == '_')
-    ++CurPtr;
-
+  CurPtr = lexMacroName(StringRef(CurPtr, CurBuf.end() - CurPtr));
   return StringRef(TokStart, CurPtr - TokStart);
 }
 
diff --git a/llvm/test/TableGen/invalid-macro-name-command-line.td b/llvm/test/TableGen/invalid-macro-name-command-line.td
new file mode 100644
index 000000000000000..0d2307997ebe545
--- /dev/null
+++ b/llvm/test/TableGen/invalid-macro-name-command-line.td
@@ -0,0 +1,9 @@
+// RUN: not llvm-tblgen %s -DMACRO=1 2>&1 | FileCheck %s --check-prefix=CHECK-TEST-1
+// RUN: not llvm-tblgen %s -D0MAC 2>&1 | FileCheck %s --check-prefix=CHECK-TEST-2
+// RUN: not llvm-tblgen %s -D_MAC# 2>&1 | FileCheck %s --check-prefix=CHECK-TEST-3
+// RUN: not llvm-tblgen %s -D 2>&1 | FileCheck %s --check-prefix=CHECK-TEST-4
+
+// CHECK-TEST-1: error: Invalid macro name `MACRO=1` specified on command line
+// CHECK-TEST-2: error: Invalid macro name `0MAC` specified on command line
+// CHECK-TEST-3: error: Invalid macro name `_MAC#` specified on command line
+// CHECK-TEST-4: for the -D option: requires a value!

From 2258bc429b6f76ade78059f3c4f6c7c77f93a996 Mon Sep 17 00:00:00 2001
From: Jay Foad <jay.foad@amd.com>
Date: Mon, 19 Aug 2024 18:22:06 +0100
Subject: [PATCH 390/441] [AMDGPU] Simplify, fix and improve known bits for
 mbcnt (#104768)

Simplify by using KnownBits::add.

Fix GlobalISel path which was ignoring the known bits of src1.

Improve analysis of mbcnt.hi which adds at most 31 even in wave64.
---
 llvm/lib/Target/AMDGPU/SIISelLowering.cpp     | 31 ++++++++++---------
 llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mbcnt.ll | 23 ++++++++++----
 2 files changed, 34 insertions(+), 20 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 09fe4ac33c3267b..ae55b56fbf43fb7 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -15758,16 +15758,12 @@ void SITargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
     case Intrinsic::amdgcn_mbcnt_hi: {
       const GCNSubtarget &ST =
           DAG.getMachineFunction().getSubtarget<GCNSubtarget>();
-      // These return at most the (wavefront size - 1) + src1
-      // As long as src1 is an immediate we can calc known bits
-      KnownBits Src1Known = DAG.computeKnownBits(Op.getOperand(2), Depth + 1);
-      unsigned Src1ValBits = Src1Known.countMaxActiveBits();
-      unsigned MaxActiveBits = std::max(Src1ValBits, ST.getWavefrontSizeLog2());
-      // Cater for potential carry
-      MaxActiveBits += Src1ValBits ? 1 : 0;
-      unsigned Size = Op.getValueType().getSizeInBits();
-      if (MaxActiveBits < Size)
-        Known.Zero.setHighBits(Size - MaxActiveBits);
+      // Wave64 mbcnt_lo returns at most 32 + src1. Otherwise these return at
+      // most 31 + src1.
+      Known.Zero.setBitsFrom(
+          IID == Intrinsic::amdgcn_mbcnt_lo ? ST.getWavefrontSizeLog2() : 5);
+      KnownBits Known2 = DAG.computeKnownBits(Op.getOperand(2), Depth + 1);
+      Known = KnownBits::add(Known, Known2);
       return;
     }
     }
@@ -15802,7 +15798,8 @@ void SITargetLowering::computeKnownBitsForTargetInstr(
   switch (MI->getOpcode()) {
   case AMDGPU::G_INTRINSIC:
   case AMDGPU::G_INTRINSIC_CONVERGENT: {
-    switch (cast<GIntrinsic>(MI)->getIntrinsicID()) {
+    Intrinsic::ID IID = cast<GIntrinsic>(MI)->getIntrinsicID();
+    switch (IID) {
     case Intrinsic::amdgcn_workitem_id_x:
       knownBitsForWorkitemID(*getSubtarget(), KB, Known, 0);
       break;
@@ -15814,9 +15811,15 @@ void SITargetLowering::computeKnownBitsForTargetInstr(
       break;
     case Intrinsic::amdgcn_mbcnt_lo:
     case Intrinsic::amdgcn_mbcnt_hi: {
-      // These return at most the wavefront size - 1.
-      unsigned Size = MRI.getType(R).getSizeInBits();
-      Known.Zero.setHighBits(Size - getSubtarget()->getWavefrontSizeLog2());
+      // Wave64 mbcnt_lo returns at most 32 + src1. Otherwise these return at
+      // most 31 + src1.
+      Known.Zero.setBitsFrom(IID == Intrinsic::amdgcn_mbcnt_lo
+                                 ? getSubtarget()->getWavefrontSizeLog2()
+                                 : 5);
+      KnownBits Known2;
+      KB.computeKnownBitsImpl(MI->getOperand(3).getReg(), Known2, DemandedElts,
+                              Depth + 1);
+      Known = KnownBits::add(Known, Known2);
       break;
     }
     case Intrinsic::amdgcn_groupstaticsize: {
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mbcnt.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mbcnt.ll
index 4ac9dc8565ff40e..fd1b99a9f30eb7c 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mbcnt.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mbcnt.ll
@@ -1,10 +1,9 @@
-; RUN: llc -mtriple=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SI %s
-; RUN: llc -mtriple=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=verde < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=verde < %s | FileCheck -check-prefix=GCN %s
 
 ; GCN-LABEL: {{^}}mbcnt_intrinsics:
-; GCN: v_mbcnt_lo_u32_b32{{(_e64)*}} [[LO:v[0-9]+]], -1, 0
-; SI: v_mbcnt_hi_u32_b32_e32 {{v[0-9]+}}, -1, [[LO]]
-; VI: v_mbcnt_hi_u32_b32 {{v[0-9]+}}, -1, [[LO]]
+; GCN: v_mbcnt_lo_u32_b32_e64 [[LO:v[0-9]+]], -1, 0
+; GCN: v_mbcnt_hi_u32_b32_e32 {{v[0-9]+}}, -1, [[LO]]
 define amdgpu_ps void @mbcnt_intrinsics(ptr addrspace(4) inreg %arg, ptr addrspace(4) inreg %arg1, ptr addrspace(4) inreg %arg2, i32 inreg %arg3) {
 main_body:
   %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0) #0
@@ -80,13 +79,25 @@ define i32 @mbcnt_hi_known_bits_3(i32 %x) #0 {
 
 ; GCN-LABEL: {{^}}mbcnt_hi_known_bits_4:
 ; GCN: v_mbcnt_hi_u32_b32
-; GCN: v_and_b32_e32
+; GCN-NOT: v_and_b32_e32
 define i32 @mbcnt_hi_known_bits_4(i32 %x) #0 {
   %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 %x, i32 15)
   %mask = and i32 %hi, 63
   ret i32 %mask
 }
 
+; TODO: Special case mbcnt.lo feeding into mbcnt.hi to remove this AND.
+; GCN-LABEL: {{^}}mbcnt_lo_hi_known_bits_1:
+; GCN: v_mbcnt_lo_u32_b32
+; GCN: v_mbcnt_hi_u32_b32
+; GCN: v_and_b32_e32
+define i32 @mbcnt_lo_hi_known_bits_1(i32 %x) #0 {
+  %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 %x, i32 0)
+  %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 %x, i32 %lo)
+  %mask = and i32 %hi, 63
+  ret i32 %mask
+}
+
 declare i32 @llvm.amdgcn.mbcnt.lo(i32, i32) #0
 declare i32 @llvm.amdgcn.mbcnt.hi(i32, i32) #0
 declare void @llvm.amdgcn.exp.f32(i32, i32, float, float, float, float, i1, i1) #1

From 564bd206588d330211914ba2e94bd13caa4e0431 Mon Sep 17 00:00:00 2001
From: Jay Foad <jay.foad@amd.com>
Date: Mon, 19 Aug 2024 18:22:29 +0100
Subject: [PATCH 391/441] [AMDGPU][GlobalISel] Save a copy in one case of
 addrspacecast (#104789)

Refactor legalization of addrspacecast local/private -> flat to avoid
building a copy in the nonnull case.
---
 .../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 23 +++++++++++--------
 .../GlobalISel/legalize-addrspacecast.mir     |  6 ++---
 2 files changed, 16 insertions(+), 13 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 65198b4285a1e44..4fd917f5ea7fa80 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -2368,25 +2368,30 @@ bool AMDGPULegalizerInfo::legalizeAddrSpaceCast(
   if (DestAS == AMDGPUAS::FLAT_ADDRESS &&
       (SrcAS == AMDGPUAS::LOCAL_ADDRESS ||
        SrcAS == AMDGPUAS::PRIVATE_ADDRESS)) {
-    Register ApertureReg = getSegmentAperture(SrcAS, MRI, B);
-    if (!ApertureReg.isValid())
-      return false;
+    auto castLocalOrPrivateToFlat = [&](const DstOp &Dst) -> Register {
+      Register ApertureReg = getSegmentAperture(SrcAS, MRI, B);
+      if (!ApertureReg.isValid())
+        return false;
 
-    // Coerce the type of the low half of the result so we can use merge_values.
-    Register SrcAsInt = B.buildPtrToInt(S32, Src).getReg(0);
+      // Coerce the type of the low half of the result so we can use
+      // merge_values.
+      Register SrcAsInt = B.buildPtrToInt(S32, Src).getReg(0);
 
-    // TODO: Should we allow mismatched types but matching sizes in merges to
-    // avoid the ptrtoint?
-    auto BuildPtr = B.buildMergeLikeInstr(DstTy, {SrcAsInt, ApertureReg});
+      // TODO: Should we allow mismatched types but matching sizes in merges to
+      // avoid the ptrtoint?
+      return B.buildMergeLikeInstr(Dst, {SrcAsInt, ApertureReg}).getReg(0);
+    };
 
     // For llvm.amdgcn.addrspacecast.nonnull we can always assume non-null, for
     // G_ADDRSPACE_CAST we need to guess.
     if (isa<GIntrinsic>(MI) || isKnownNonNull(Src, MRI, TM, SrcAS)) {
-      B.buildCopy(Dst, BuildPtr);
+      castLocalOrPrivateToFlat(Dst);
       MI.eraseFromParent();
       return true;
     }
 
+    Register BuildPtr = castLocalOrPrivateToFlat(DstTy);
+
     auto SegmentNull = B.buildConstant(SrcTy, TM.getNullPointerValue(SrcAS));
     auto FlatNull = B.buildConstant(DstTy, TM.getNullPointerValue(DestAS));
 
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-addrspacecast.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-addrspacecast.mir
index 24c4d668885f439..0b3b4288556bfef 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-addrspacecast.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-addrspacecast.mir
@@ -509,16 +509,14 @@ body: |
     ; SIVI-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load (s32), addrspace 4)
     ; SIVI-NEXT: [[PTRTOINT:%[0-9]+]]:_(s32) = G_PTRTOINT [[FRAME_INDEX]](p5)
     ; SIVI-NEXT: [[MV:%[0-9]+]]:_(p0) = G_MERGE_VALUES [[PTRTOINT]](s32), [[LOAD]](s32)
-    ; SIVI-NEXT: [[COPY2:%[0-9]+]]:_(p0) = COPY [[MV]](p0)
-    ; SIVI-NEXT: $vgpr0_vgpr1 = COPY [[COPY2]](p0)
+    ; SIVI-NEXT: $vgpr0_vgpr1 = COPY [[MV]](p0)
     ; GFX9-LABEL: name: test_addrspacecast_p5_fi_to_p0
     ; GFX9: [[FRAME_INDEX:%[0-9]+]]:_(p5) = G_FRAME_INDEX %stack.0
     ; GFX9-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64(s64) = S_MOV_B64 $src_private_base
     ; GFX9-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[S_MOV_B64_]](s64)
     ; GFX9-NEXT: [[PTRTOINT:%[0-9]+]]:_(s32) = G_PTRTOINT [[FRAME_INDEX]](p5)
     ; GFX9-NEXT: [[MV:%[0-9]+]]:_(p0) = G_MERGE_VALUES [[PTRTOINT]](s32), [[UV1]](s32)
-    ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY [[MV]](p0)
-    ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[COPY]](p0)
+    ; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[MV]](p0)
     %0:_(p5) = G_FRAME_INDEX %stack.0
     %1:_(p0) = G_ADDRSPACE_CAST %0
     $vgpr0_vgpr1 = COPY %1

From 2575ea6e17889e6d83198fc1c80963f81485bfcc Mon Sep 17 00:00:00 2001
From: Samira Bazuzi <bazuzi@google.com>
Date: Mon, 19 Aug 2024 13:25:49 -0400
Subject: [PATCH 392/441] =?UTF-8?q?[clang][dataflow]=20Collect=20local=20v?=
 =?UTF-8?q?ariables=20referenced=20within=20a=20functio=E2=80=A6=20(#10445?=
 =?UTF-8?q?9)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…n/statement.

We don't need these for the same in-tree purposes as the other sets,
i.e. for making sure we model these Decls that are declared outside the
function, but we have an out-of-tree use for these sets that would
benefit from this simple addition and would avoid duplicating so much of
this code.
---
 .../clang/Analysis/FlowSensitive/ASTOps.h     |  3 +++
 clang/lib/Analysis/FlowSensitive/ASTOps.cpp   |  9 ++++++++
 .../Analysis/FlowSensitive/ASTOpsTest.cpp     | 22 +++++++++++++++++++
 3 files changed, 34 insertions(+)

diff --git a/clang/include/clang/Analysis/FlowSensitive/ASTOps.h b/clang/include/clang/Analysis/FlowSensitive/ASTOps.h
index f9c923a36ad2296..ec4d041254877fd 100644
--- a/clang/include/clang/Analysis/FlowSensitive/ASTOps.h
+++ b/clang/include/clang/Analysis/FlowSensitive/ASTOps.h
@@ -139,6 +139,9 @@ struct ReferencedDecls {
   /// All variables with static storage duration, notably including static
   /// member variables and static variables declared within a function.
   llvm::DenseSet<const VarDecl *> Globals;
+  /// Local variables, not including parameters or static variables declared
+  /// within a function.
+  llvm::DenseSet<const VarDecl *> Locals;
   /// Free functions and member functions which are referenced (but not
   /// necessarily called).
   llvm::DenseSet<const FunctionDecl *> Functions;
diff --git a/clang/lib/Analysis/FlowSensitive/ASTOps.cpp b/clang/lib/Analysis/FlowSensitive/ASTOps.cpp
index 27d42a7b508562e..fdba139628d8ff0 100644
--- a/clang/lib/Analysis/FlowSensitive/ASTOps.cpp
+++ b/clang/lib/Analysis/FlowSensitive/ASTOps.cpp
@@ -170,6 +170,13 @@ static void insertIfGlobal(const Decl &D,
       Globals.insert(V);
 }
 
+static void insertIfLocal(const Decl &D,
+                          llvm::DenseSet<const VarDecl *> &Locals) {
+  if (auto *V = dyn_cast<VarDecl>(&D))
+    if (V->hasLocalStorage() && !isa<ParmVarDecl>(V))
+      Locals.insert(V);
+}
+
 static void insertIfFunction(const Decl &D,
                              llvm::DenseSet<const FunctionDecl *> &Funcs) {
   if (auto *FD = dyn_cast<FunctionDecl>(&D))
@@ -220,12 +227,14 @@ class ReferencedDeclsVisitor
 
   bool VisitDecl(Decl *D) {
     insertIfGlobal(*D, Referenced.Globals);
+    insertIfLocal(*D, Referenced.Locals);
     insertIfFunction(*D, Referenced.Functions);
     return true;
   }
 
   bool VisitDeclRefExpr(DeclRefExpr *E) {
     insertIfGlobal(*E->getDecl(), Referenced.Globals);
+    insertIfLocal(*E->getDecl(), Referenced.Locals);
     insertIfFunction(*E->getDecl(), Referenced.Functions);
     return true;
   }
diff --git a/clang/unittests/Analysis/FlowSensitive/ASTOpsTest.cpp b/clang/unittests/Analysis/FlowSensitive/ASTOpsTest.cpp
index cd1c076ab09e6b4..834aa7f4c2ac2e1 100644
--- a/clang/unittests/Analysis/FlowSensitive/ASTOpsTest.cpp
+++ b/clang/unittests/Analysis/FlowSensitive/ASTOpsTest.cpp
@@ -85,4 +85,26 @@ TEST(ASTOpsTest, ReferencedDeclsOnUnionInitList) {
               UnorderedElementsAre(IDecl));
 }
 
+TEST(ASTOpsTest, ReferencedDeclsLocalsNotParamsOrStatics) {
+  std::string Code = R"cc(
+    void func(int Param) {
+      static int Static = 0;
+      int Local = Param;
+      Local = Static;
+    }
+  )cc";
+  std::unique_ptr<ASTUnit> Unit =
+      tooling::buildASTFromCodeWithArgs(Code, {"-fsyntax-only", "-std=c++17"});
+  auto &ASTCtx = Unit->getASTContext();
+
+  ASSERT_EQ(ASTCtx.getDiagnostics().getClient()->getNumErrors(), 0U);
+
+  auto *Func = cast<FunctionDecl>(findValueDecl(ASTCtx, "func"));
+  ASSERT_NE(Func, nullptr);
+  auto *LocalDecl = cast<VarDecl>(findValueDecl(ASTCtx, "Local"));
+
+  EXPECT_THAT(getReferencedDecls(*Func).Locals,
+              UnorderedElementsAre(LocalDecl));
+}
+
 } // namespace

From 7f968e3aea41264a952692a4bf32e5e2b612d44e Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Mon, 19 Aug 2024 10:28:49 -0700
Subject: [PATCH 393/441] [RISCV] Improve BCLRITwoBitsMaskHigh SDNodeXForm. NFC

Use getZExtValue instead of getSExtValue since we are going to
overwrite the sign extension on RV32. getZExtValue should be cheaper than
getSExtValue.

Use maskLeadingOnes to improve readability.
---
 llvm/lib/Target/RISCV/RISCVInstrInfoZb.td | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td
index f27039b1c658d20..8e6f281027695a2 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td
@@ -124,9 +124,9 @@ def BCLRITwoBitsMaskLow : SDNodeXForm<imm, [{
 }]>;
 
 def BCLRITwoBitsMaskHigh : SDNodeXForm<imm, [{
-  uint64_t I = N->getSExtValue();
+  uint64_t I = N->getZExtValue();
   if (!Subtarget->is64Bit())
-    I |= 0xffffffffull << 32;
+    I |= maskLeadingOnes<uint64_t>(32);
   return CurDAG->getTargetConstant(llvm::Log2_64(~I), SDLoc(N),
                                    N->getValueType(0));
 }]>;

From 865952bbbb97655410d2c34c21938134059c4076 Mon Sep 17 00:00:00 2001
From: Sergey Kozub <skozub@nvidia.com>
Date: Mon, 19 Aug 2024 19:34:22 +0200
Subject: [PATCH 394/441] [NVPTX] Add conversion intrinsics from/to fp8 types
 (e4m3, e5m2) (#102969)

PTX ISA 8.1 supports FP8 conversions:

https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-cvt

This PR adds the support for:

- cvt.rn.satfinite{.relu}.f8x2type.f32 d, a, b;
- cvt.rn.satfinite{.relu}.f8x2type.f16x2 d, a;
- cvt.rn.{.relu}.f16x2.f8x2type d, a;

where .f8x2type = { .e4m3x2, .e5m2x2 };
---
 clang/include/clang/Basic/BuiltinsNVPTX.def | 15 ++++
 clang/test/CodeGen/builtins-nvptx.c         | 36 +++++++++
 llvm/include/llvm/IR/IntrinsicsNVVM.td      | 27 +++++++
 llvm/lib/Target/NVPTX/NVPTXInstrInfo.td     | 29 +++++++
 llvm/lib/Target/NVPTX/NVPTXIntrinsics.td    | 27 +++++++
 llvm/test/CodeGen/NVPTX/convert-sm89.ll     | 86 +++++++++++++++++++++
 6 files changed, 220 insertions(+)
 create mode 100644 llvm/test/CodeGen/NVPTX/convert-sm89.ll

diff --git a/clang/include/clang/Basic/BuiltinsNVPTX.def b/clang/include/clang/Basic/BuiltinsNVPTX.def
index 504314d8d96e91e..c11970c279c4bba 100644
--- a/clang/include/clang/Basic/BuiltinsNVPTX.def
+++ b/clang/include/clang/Basic/BuiltinsNVPTX.def
@@ -584,6 +584,21 @@ TARGET_BUILTIN(__nvvm_f2bf16_rz_relu, "yf", "", AND(SM_80,PTX70))
 
 TARGET_BUILTIN(__nvvm_f2tf32_rna, "ZUif", "", AND(SM_80,PTX70))
 
+TARGET_BUILTIN(__nvvm_ff_to_e4m3x2_rn, "sff", "", AND(SM_89,PTX81))
+TARGET_BUILTIN(__nvvm_ff_to_e4m3x2_rn_relu, "sff", "", AND(SM_89,PTX81))
+TARGET_BUILTIN(__nvvm_ff_to_e5m2x2_rn, "sff", "", AND(SM_89,PTX81))
+TARGET_BUILTIN(__nvvm_ff_to_e5m2x2_rn_relu, "sff", "", AND(SM_89,PTX81))
+
+TARGET_BUILTIN(__nvvm_f16x2_to_e4m3x2_rn, "sV2h", "", AND(SM_89,PTX81))
+TARGET_BUILTIN(__nvvm_f16x2_to_e4m3x2_rn_relu, "sV2h", "", AND(SM_89,PTX81))
+TARGET_BUILTIN(__nvvm_f16x2_to_e5m2x2_rn, "sV2h", "", AND(SM_89,PTX81))
+TARGET_BUILTIN(__nvvm_f16x2_to_e5m2x2_rn_relu, "sV2h", "", AND(SM_89,PTX81))
+
+TARGET_BUILTIN(__nvvm_e4m3x2_to_f16x2_rn, "V2hs", "", AND(SM_89,PTX81))
+TARGET_BUILTIN(__nvvm_e4m3x2_to_f16x2_rn_relu, "V2hs", "", AND(SM_89,PTX81))
+TARGET_BUILTIN(__nvvm_e5m2x2_to_f16x2_rn, "V2hs", "", AND(SM_89,PTX81))
+TARGET_BUILTIN(__nvvm_e5m2x2_to_f16x2_rn_relu, "V2hs", "", AND(SM_89,PTX81))
+
 // Bitcast
 
 BUILTIN(__nvvm_bitcast_f2i, "if", "")
diff --git a/clang/test/CodeGen/builtins-nvptx.c b/clang/test/CodeGen/builtins-nvptx.c
index 75b9d6d1fe1902c..20399b73e637575 100644
--- a/clang/test/CodeGen/builtins-nvptx.c
+++ b/clang/test/CodeGen/builtins-nvptx.c
@@ -22,6 +22,9 @@
 // RUN: %clang_cc1 -ffp-contract=off -triple nvptx64-unknown-unknown -target-cpu sm_86 -target-feature +ptx72 \
 // RUN:            -fcuda-is-device -emit-llvm -o - -x cuda %s \
 // RUN:   | FileCheck -check-prefix=CHECK -check-prefix=CHECK_PTX72_SM86 -check-prefix=LP64 %s
+// RUN: %clang_cc1 -ffp-contract=off -triple nvptx64-unknown-unknown -target-cpu sm_89 -target-feature +ptx81 \
+// RUN:            -fcuda-is-device -emit-llvm -o - -x cuda %s \
+// RUN:   | FileCheck -check-prefix=CHECK -check-prefix=CHECK_PTX81_SM89 %s
 
 #define __device__ __attribute__((device))
 #define __global__ __attribute__((global))
@@ -968,6 +971,39 @@ __device__ void nvvm_cvt_sm80() {
   // CHECK: ret void
 }
 
+// CHECK-LABEL: nvvm_cvt_sm89
+__device__ void nvvm_cvt_sm89() {
+#if __CUDA_ARCH__ >= 890
+  // CHECK_PTX81_SM89: call i16 @llvm.nvvm.ff.to.e4m3x2.rn(float 1.000000e+00, float 1.000000e+00)
+  __nvvm_ff_to_e4m3x2_rn(1.0f, 1.0f);
+  // CHECK_PTX81_SM89: call i16 @llvm.nvvm.ff.to.e4m3x2.rn.relu(float 1.000000e+00, float 1.000000e+00)
+  __nvvm_ff_to_e4m3x2_rn_relu(1.0f, 1.0f);
+  // CHECK_PTX81_SM89: call i16 @llvm.nvvm.ff.to.e5m2x2.rn(float 1.000000e+00, float 1.000000e+00)
+  __nvvm_ff_to_e5m2x2_rn(1.0f, 1.0f);
+  // CHECK_PTX81_SM89: call i16 @llvm.nvvm.ff.to.e5m2x2.rn.relu(float 1.000000e+00, float 1.000000e+00)
+  __nvvm_ff_to_e5m2x2_rn_relu(1.0f, 1.0f);
+
+  // CHECK_PTX81_SM89: call i16 @llvm.nvvm.f16x2.to.e4m3x2.rn(<2 x half> <half 0xH3C00, half 0xH3C00>)
+  __nvvm_f16x2_to_e4m3x2_rn({1.0f16, 1.0f16});
+  // CHECK_PTX81_SM89: call i16 @llvm.nvvm.f16x2.to.e4m3x2.rn.relu(<2 x half> <half 0xH3C00, half 0xH3C00>)
+  __nvvm_f16x2_to_e4m3x2_rn_relu({1.0f16, 1.0f16});
+  // CHECK_PTX81_SM89: call i16 @llvm.nvvm.f16x2.to.e5m2x2.rn(<2 x half> <half 0xH3C00, half 0xH3C00>)
+  __nvvm_f16x2_to_e5m2x2_rn({1.0f16, 1.0f16});
+  // CHECK_PTX81_SM89: call i16 @llvm.nvvm.f16x2.to.e5m2x2.rn.relu(<2 x half> <half 0xH3C00, half 0xH3C00>)
+  __nvvm_f16x2_to_e5m2x2_rn_relu({1.0f16, 1.0f16});
+
+  // CHECK_PTX81_SM89: call <2 x half> @llvm.nvvm.e4m3x2.to.f16x2.rn(i16 18504)
+  __nvvm_e4m3x2_to_f16x2_rn(0x4848);
+  // CHECK_PTX81_SM89: call <2 x half> @llvm.nvvm.e4m3x2.to.f16x2.rn.relu(i16 18504)
+  __nvvm_e4m3x2_to_f16x2_rn_relu(0x4848);
+  // CHECK_PTX81_SM89: call <2 x half> @llvm.nvvm.e5m2x2.to.f16x2.rn(i16 19532)
+  __nvvm_e5m2x2_to_f16x2_rn(0x4c4c);
+  // CHECK_PTX81_SM89: call <2 x half> @llvm.nvvm.e5m2x2.to.f16x2.rn.relu(i16 19532)
+  __nvvm_e5m2x2_to_f16x2_rn_relu(0x4c4c);
+#endif
+  // CHECK: ret void
+}
+
 #define NAN32 0x7FBFFFFF
 #define NAN16 (__bf16)0x7FBF
 #define BF16 (__bf16)0.1f
diff --git a/llvm/include/llvm/IR/IntrinsicsNVVM.td b/llvm/include/llvm/IR/IntrinsicsNVVM.td
index 40fd86d3063a169..8a333e7a559fe28 100644
--- a/llvm/include/llvm/IR/IntrinsicsNVVM.td
+++ b/llvm/include/llvm/IR/IntrinsicsNVVM.td
@@ -1312,6 +1312,33 @@ let TargetPrefix = "nvvm" in {
   def int_nvvm_f2tf32_rna : ClangBuiltin<"__nvvm_f2tf32_rna">,
       Intrinsic<[llvm_i32_ty], [llvm_float_ty], [IntrNoMem, IntrNoCallback]>;
 
+  def int_nvvm_ff_to_e4m3x2_rn : ClangBuiltin<"__nvvm_ff_to_e4m3x2_rn">,
+      Intrinsic<[llvm_i16_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem, IntrNoCallback]>;
+  def int_nvvm_ff_to_e4m3x2_rn_relu : ClangBuiltin<"__nvvm_ff_to_e4m3x2_rn_relu">,
+      Intrinsic<[llvm_i16_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem, IntrNoCallback]>;
+  def int_nvvm_ff_to_e5m2x2_rn : ClangBuiltin<"__nvvm_ff_to_e5m2x2_rn">,
+      Intrinsic<[llvm_i16_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem, IntrNoCallback]>;
+  def int_nvvm_ff_to_e5m2x2_rn_relu : ClangBuiltin<"__nvvm_ff_to_e5m2x2_rn_relu">,
+      Intrinsic<[llvm_i16_ty], [llvm_float_ty, llvm_float_ty], [IntrNoMem, IntrNoCallback]>;
+
+  def int_nvvm_f16x2_to_e4m3x2_rn : ClangBuiltin<"__nvvm_f16x2_to_e4m3x2_rn">,
+      Intrinsic<[llvm_i16_ty], [llvm_v2f16_ty], [IntrNoMem, IntrNoCallback]>;
+  def int_nvvm_f16x2_to_e4m3x2_rn_relu : ClangBuiltin<"__nvvm_f16x2_to_e4m3x2_rn_relu">,
+      Intrinsic<[llvm_i16_ty], [llvm_v2f16_ty], [IntrNoMem, IntrNoCallback]>;
+  def int_nvvm_f16x2_to_e5m2x2_rn : ClangBuiltin<"__nvvm_f16x2_to_e5m2x2_rn">,
+      Intrinsic<[llvm_i16_ty], [llvm_v2f16_ty], [IntrNoMem, IntrNoCallback]>;
+  def int_nvvm_f16x2_to_e5m2x2_rn_relu : ClangBuiltin<"__nvvm_f16x2_to_e5m2x2_rn_relu">,
+      Intrinsic<[llvm_i16_ty], [llvm_v2f16_ty], [IntrNoMem, IntrNoCallback]>;
+
+  def int_nvvm_e4m3x2_to_f16x2_rn : ClangBuiltin<"__nvvm_e4m3x2_to_f16x2_rn">,
+      Intrinsic<[llvm_v2f16_ty], [llvm_i16_ty], [IntrNoMem, IntrNoCallback]>;
+  def int_nvvm_e4m3x2_to_f16x2_rn_relu : ClangBuiltin<"__nvvm_e4m3x2_to_f16x2_rn_relu">,
+      Intrinsic<[llvm_v2f16_ty], [llvm_i16_ty], [IntrNoMem, IntrNoCallback]>;
+  def int_nvvm_e5m2x2_to_f16x2_rn : ClangBuiltin<"__nvvm_e5m2x2_to_f16x2_rn">,
+      Intrinsic<[llvm_v2f16_ty], [llvm_i16_ty], [IntrNoMem, IntrNoCallback]>;
+  def int_nvvm_e5m2x2_to_f16x2_rn_relu : ClangBuiltin<"__nvvm_e5m2x2_to_f16x2_rn_relu">,
+      Intrinsic<[llvm_v2f16_ty], [llvm_i16_ty], [IntrNoMem, IntrNoCallback]>;
+
 //
 // Bitcast
 //
diff --git a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
index 1c116c496658dfb..a6dfb704e38d2e2 100644
--- a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
+++ b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
@@ -723,6 +723,35 @@ let hasSideEffects = false in {
 
   defm CVT_f16x2 : CVT_FROM_FLOAT_V2_SM80<"f16x2", Int32Regs>;
   defm CVT_bf16x2 : CVT_FROM_FLOAT_V2_SM80<"bf16x2", Int32Regs>;
+
+  // FP8 conversions.
+  multiclass CVT_TO_F8X2<string F8Name> {
+    def _f32 :
+      NVPTXInst<(outs Int16Regs:$dst),
+                (ins Float32Regs:$src1, Float32Regs:$src2, CvtMode:$mode),
+                !strconcat("cvt${mode:base}.satfinite${mode:relu}.",
+                F8Name, "x2.f32 \t$dst, $src1, $src2;"), []>,
+      Requires<[hasPTX<81>, hasSM<89>]>;
+    def _f16x2 :
+      NVPTXInst<(outs Int16Regs:$dst),
+                (ins Int32Regs:$src, CvtMode:$mode),
+                !strconcat("cvt${mode:base}.satfinite${mode:relu}.",
+                F8Name, "x2.f16x2 \t$dst, $src;"), []>,
+      Requires<[hasPTX<81>, hasSM<89>]>;
+  }
+
+  defm CVT_e4m3x2 : CVT_TO_F8X2<"e4m3">;
+  defm CVT_e5m2x2 : CVT_TO_F8X2<"e5m2">;
+
+  class CVT_f16x2_fp8<string F8Name> :
+    NVPTXInst<(outs Int32Regs:$dst),
+              (ins Int16Regs:$src, CvtMode:$mode),
+              !strconcat("cvt${mode:base}${mode:relu}.f16x2.",
+              F8Name, "x2 \t$dst, $src;"), []>,
+    Requires<[hasPTX<81>, hasSM<89>]>;
+
+  def CVT_f16x2_e4m3x2 : CVT_f16x2_fp8<"e4m3">;
+  def CVT_f16x2_e5m2x2 : CVT_f16x2_fp8<"e5m2">;
 }
 
 //-----------------------------------
diff --git a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
index 887951b55fb3b70..a9116e15c3671be 100644
--- a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -1524,6 +1524,33 @@ def : Pat<(int_nvvm_f2h_rn_ftz Float32Regs:$a),
 def : Pat<(int_nvvm_f2h_rn Float32Regs:$a),
           (CVT_f16_f32 Float32Regs:$a, CvtRN)>;
 
+def : Pat<(int_nvvm_ff_to_e4m3x2_rn Float32Regs:$a, Float32Regs:$b),
+          (CVT_e4m3x2_f32 Float32Regs:$a, Float32Regs:$b, CvtRN)>;
+def : Pat<(int_nvvm_ff_to_e4m3x2_rn_relu Float32Regs:$a, Float32Regs:$b),
+          (CVT_e4m3x2_f32 Float32Regs:$a, Float32Regs:$b, CvtRN_RELU)>;
+def : Pat<(int_nvvm_ff_to_e5m2x2_rn Float32Regs:$a, Float32Regs:$b),
+          (CVT_e5m2x2_f32 Float32Regs:$a, Float32Regs:$b, CvtRN)>;
+def : Pat<(int_nvvm_ff_to_e5m2x2_rn_relu Float32Regs:$a, Float32Regs:$b),
+          (CVT_e5m2x2_f32 Float32Regs:$a, Float32Regs:$b, CvtRN_RELU)>;
+
+def : Pat<(int_nvvm_f16x2_to_e4m3x2_rn Int32Regs:$a),
+          (CVT_e4m3x2_f16x2 Int32Regs:$a, CvtRN)>;
+def : Pat<(int_nvvm_f16x2_to_e4m3x2_rn_relu Int32Regs:$a),
+          (CVT_e4m3x2_f16x2 Int32Regs:$a, CvtRN_RELU)>;
+def : Pat<(int_nvvm_f16x2_to_e5m2x2_rn Int32Regs:$a),
+          (CVT_e5m2x2_f16x2 Int32Regs:$a, CvtRN)>;
+def : Pat<(int_nvvm_f16x2_to_e5m2x2_rn_relu Int32Regs:$a),
+          (CVT_e5m2x2_f16x2 Int32Regs:$a, CvtRN_RELU)>;
+
+def : Pat<(int_nvvm_e4m3x2_to_f16x2_rn Int16Regs:$a),
+          (CVT_f16x2_e4m3x2 Int16Regs:$a, CvtRN)>;
+def : Pat<(int_nvvm_e4m3x2_to_f16x2_rn_relu Int16Regs:$a),
+          (CVT_f16x2_e4m3x2 Int16Regs:$a, CvtRN_RELU)>;
+def : Pat<(int_nvvm_e5m2x2_to_f16x2_rn Int16Regs:$a),
+          (CVT_f16x2_e5m2x2 Int16Regs:$a, CvtRN)>;
+def : Pat<(int_nvvm_e5m2x2_to_f16x2_rn_relu Int16Regs:$a),
+          (CVT_f16x2_e5m2x2 Int16Regs:$a, CvtRN_RELU)>;
+
 //
 // Bitcast
 //
diff --git a/llvm/test/CodeGen/NVPTX/convert-sm89.ll b/llvm/test/CodeGen/NVPTX/convert-sm89.ll
new file mode 100644
index 000000000000000..5a1a640afc1cf9c
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/convert-sm89.ll
@@ -0,0 +1,86 @@
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_89 -mattr=+ptx81 | FileCheck %s
+; RUN: %if ptxas-12.1 %{ llc < %s -march=nvptx64 -mcpu=sm_89 -mattr=+ptx81 | %ptxas-verify -arch=sm_89 %}
+
+; CHECK-LABEL: cvt_rn_e4m3x2_f32
+define i16 @cvt_rn_e4m3x2_f32(float %f1, float %f2) {
+; CHECK: cvt.rn.satfinite.e4m3x2.f32
+  %val = call i16 @llvm.nvvm.ff.to.e4m3x2.rn(float %f1, float %f2);
+  ret i16 %val
+}
+
+; CHECK-LABEL: cvt_rn_relu_e4m3x2_f32
+define i16 @cvt_rn_relu_e4m3x2_f32(float %f1, float %f2) {
+; CHECK: cvt.rn.satfinite.relu.e4m3x2.f32
+  %val = call i16 @llvm.nvvm.ff.to.e4m3x2.rn.relu(float %f1, float %f2);
+  ret i16 %val
+}
+
+; CHECK-LABEL: cvt_rn_e5m2x2_f32
+define i16 @cvt_rn_e5m2x2_f32(float %f1, float %f2) {
+; CHECK: cvt.rn.satfinite.e5m2x2.f32
+  %val = call i16 @llvm.nvvm.ff.to.e5m2x2.rn(float %f1, float %f2);
+  ret i16 %val
+}
+
+; CHECK-LABEL: cvt_rn_relu_e5m2x2_f32
+define i16 @cvt_rn_relu_e5m2x2_f32(float %f1, float %f2) {
+; CHECK: cvt.rn.satfinite.relu.e5m2x2.f32
+  %val = call i16 @llvm.nvvm.ff.to.e5m2x2.rn.relu(float %f1, float %f2);
+  ret i16 %val
+}
+
+; CHECK-LABEL: cvt_rn_e4m3x2_f16x2
+define i16 @cvt_rn_e4m3x2_f16x2(<2 x half> %in) {
+; CHECK: cvt.rn.satfinite.e4m3x2.f16x2
+  %val = call i16 @llvm.nvvm.f16x2.to.e4m3x2.rn(<2 x half> %in);
+  ret i16 %val
+}
+
+; CHECK-LABEL: cvt_rn_relu_e4m3x2_f16x2
+define i16 @cvt_rn_relu_e4m3x2_f16x2(<2 x half> %in) {
+; CHECK: cvt.rn.satfinite.relu.e4m3x2.f16x2
+  %val = call i16 @llvm.nvvm.f16x2.to.e4m3x2.rn.relu(<2 x half> %in);
+  ret i16 %val
+}
+
+; CHECK-LABEL: cvt_rn_e5m2x2_f16x2
+define i16 @cvt_rn_e5m2x2_f16x2(<2 x half> %in) {
+; CHECK: cvt.rn.satfinite.e5m2x2.f16x2
+  %val = call i16 @llvm.nvvm.f16x2.to.e5m2x2.rn(<2 x half> %in);
+  ret i16 %val
+}
+
+; CHECK-LABEL: cvt_rn_relu_e5m2x2_f16x2
+define i16 @cvt_rn_relu_e5m2x2_f16x2(<2 x half> %in) {
+; CHECK: cvt.rn.satfinite.relu.e5m2x2.f16x2
+  %val = call i16 @llvm.nvvm.f16x2.to.e5m2x2.rn.relu(<2 x half> %in);
+  ret i16 %val
+}
+
+; CHECK-LABEL: cvt_rn_f16x2_e4m3x2
+define <2 x half> @cvt_rn_f16x2_e4m3x2(i16 %in) {
+; CHECK: cvt.rn.f16x2.e4m3x2
+  %val = call <2 x half> @llvm.nvvm.e4m3x2.to.f16x2.rn(i16 %in);
+  ret <2 x half> %val
+}
+
+; CHECK-LABEL: cvt_rn_relu_f16x2_e4m3x2
+define <2 x half> @cvt_rn_relu_f16x2_e4m3x2(i16 %in) {
+; CHECK: cvt.rn.relu.f16x2.e4m3x2
+  %val = call <2 x half> @llvm.nvvm.e4m3x2.to.f16x2.rn.relu(i16 %in);
+  ret <2 x half> %val
+}
+
+; CHECK-LABEL: cvt_rn_f16x2_e5m2x2
+define <2 x half> @cvt_rn_f16x2_e5m2x2(i16 %in) {
+; CHECK: cvt.rn.f16x2.e5m2x2
+  %val = call <2 x half> @llvm.nvvm.e5m2x2.to.f16x2.rn(i16 %in);
+  ret <2 x half> %val
+}
+
+; CHECK-LABEL: cvt_rn_relu_f16x2_e5m2x2
+define <2 x half> @cvt_rn_relu_f16x2_e5m2x2(i16 %in) {
+; CHECK: cvt.rn.relu.f16x2.e5m2x2
+  %val = call <2 x half> @llvm.nvvm.e5m2x2.to.f16x2.rn.relu(i16 %in);
+  ret <2 x half> %val
+}

From 4e332bba2f3a51966817088cb4dc4a9188955b64 Mon Sep 17 00:00:00 2001
From: Thurston Dang <thurston@google.com>
Date: Mon, 19 Aug 2024 10:40:17 -0700
Subject: [PATCH 395/441] [asan] Change Apple back to fixed allocator base
 address (#104818)

This partially reverts https://github.com/llvm/llvm-project/pull/98511
- specifically, the SANITIZER_APPLE case - because of a suspected
breakage for clang on Mac
(https://g-issues.chromium.org/issues/360160858;
https://github.com/llvm/llvm-project/pull/98511#issuecomment-2296749757).
---
 compiler-rt/lib/asan/asan_allocator.h | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/compiler-rt/lib/asan/asan_allocator.h b/compiler-rt/lib/asan/asan_allocator.h
index b52cdeb73089239..db8dc3bebfc6206 100644
--- a/compiler-rt/lib/asan/asan_allocator.h
+++ b/compiler-rt/lib/asan/asan_allocator.h
@@ -189,7 +189,13 @@ const uptr kAllocatorSize = 0x40000000000ULL;  // 4T.
 typedef DefaultSizeClassMap SizeClassMap;
 #    endif  // SANITIZER_RISCV64
 #  else     // SANITIZER_FUCHSIA
+
+#    if SANITIZER_APPLE
+const uptr kAllocatorSpace = 0x600000000000ULL;
+#    else   // SANITIZER_APPLE
 const uptr kAllocatorSpace = ~(uptr)0;
+#    endif  // SANITIZER_APPLE
+
 #    if defined(__powerpc64__)
 const uptr kAllocatorSize  =  0x20000000000ULL;  // 2T.
 typedef DefaultSizeClassMap SizeClassMap;

From e306db0ff0d348da49b44bd0bcae3df5bf8f02a9 Mon Sep 17 00:00:00 2001
From: Daniel Chen <cdchen@ca.ibm.com>
Date: Mon, 19 Aug 2024 13:43:54 -0400
Subject: [PATCH 396/441] [Flang] Fix test case for AIX(big-endian) system for
 issuing an extra message. (#104792)

The 10-byte FP on big-endian system such as AIX is not represented as a
valid number in FE, so Flang issues an extra WARNING message as
`underflow on REAL(10) to REAL(4) conversion` for `_10` for such a
number.
This PR is to fix the test case.
---
 flang/test/Semantics/kinds04_q16.f90 | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/flang/test/Semantics/kinds04_q16.f90 b/flang/test/Semantics/kinds04_q16.f90
index 885ca8c8392b688..d440fa4d1402d27 100644
--- a/flang/test/Semantics/kinds04_q16.f90
+++ b/flang/test/Semantics/kinds04_q16.f90
@@ -1,4 +1,5 @@
-! RUN: %python %S/test_errors.py %s %flang_fc1
+! RUN: not %flang_fc1 %s 2>%t.stderr
+! RUN: FileCheck %s --input-file=%t.stderr --check-prefixes=%if system-aix %{"PORTABILITY","WARNING","ERROR","AIX_WARNING"%} %else %{"PORTABILITY","WARNING","ERROR"%}
 ! C716 If both kind-param and exponent-letter appear, exponent-letter
 ! shall be E. (As an extension we also allow an exponent-letter which matches
 ! the kind-param)
@@ -15,10 +16,12 @@ subroutine s(var)
   !PORTABILITY: Explicit kind parameter together with non-'E' exponent letter is not standard
   real :: realvar4 = 4.0D6_8
   !WARNING: Explicit kind parameter on real constant disagrees with exponent letter 'q'
+  !AIX_WARNING: underflow on REAL(10) to REAL(4) conversion
   real :: realvar5 = 4.0Q6_10
   !PORTABILITY: Explicit kind parameter together with non-'E' exponent letter is not standard
   real :: realvar6 = 4.0Q6_16
   real :: realvar7 = 4.0E6_8
+  !AIX_WARNING: underflow on REAL(10) to REAL(4) conversion
   real :: realvar8 = 4.0E6_10
   real :: realvar9 = 4.0E6_16
   !ERROR: Unsupported REAL(KIND=32)
@@ -32,6 +35,7 @@ subroutine s(var)
   !PORTABILITY: Explicit kind parameter together with non-'E' exponent letter is not standard
   double precision :: doublevar5 = 4.0Q6_16
   double precision :: doublevar6 = 4.0E6_8
+  !AIX_WARNING: underflow on REAL(10) to REAL(8) conversion
   double precision :: doublevar7 = 4.0E6_10
   double precision :: doublevar8 = 4.0E6_16
   !ERROR: Unsupported REAL(KIND=32)

From d5617ada36705e871f329a7b0efb19ce6e6e2a1f Mon Sep 17 00:00:00 2001
From: Alexander Shaposhnikov <ashaposhnikov@google.com>
Date: Mon, 19 Aug 2024 10:44:53 -0700
Subject: [PATCH 397/441] [compiler-rt][nsan] Add more tests for shadow memory
 (#100906)

Add more tests for shadow memory.
---
 compiler-rt/test/nsan/stable_sort.cpp | 56 +++++++++++++++++++++++++++
 compiler-rt/test/nsan/swap.cpp        | 46 ++++++++++++++++++++++
 2 files changed, 102 insertions(+)
 create mode 100644 compiler-rt/test/nsan/stable_sort.cpp
 create mode 100644 compiler-rt/test/nsan/swap.cpp

diff --git a/compiler-rt/test/nsan/stable_sort.cpp b/compiler-rt/test/nsan/stable_sort.cpp
new file mode 100644
index 000000000000000..d280eedf0ebf7b6
--- /dev/null
+++ b/compiler-rt/test/nsan/stable_sort.cpp
@@ -0,0 +1,56 @@
+// RUN: %clangxx_nsan -O2 -g %s -o %t
+// RUN: %run %t 2>&1 | FileCheck %s
+
+// Check compilation mode that is required to call memcpy/memmove.
+// RUN: %clangxx_nsan -fno-builtin -O2 -g %s -o %t
+// RUN: %run %t 2>&1 | FileCheck %s
+
+// This tests a particularaly hard case of memory tracking. stable_sort does
+// conditional swaps of pairs of elements with mixed types (int/double).
+
+#include <algorithm>
+#include <cstddef>
+#include <cstdio>
+#include <utility>
+#include <vector>
+
+extern "C" void __nsan_dump_shadow_mem(const char *addr, size_t size_bytes,
+                                       size_t bytes_per_line, size_t reserved);
+
+__attribute__((noinline)) void Run(std::vector<int> &indices,
+                                   std::vector<double> &values) {
+  const auto num_entries = indices.size();
+  std::vector<std::pair<int, double>> entries;
+  entries.reserve(num_entries);
+  for (size_t i = 0; i < num_entries; ++i)
+    entries.emplace_back(indices[i], values[i]);
+
+  __nsan_dump_shadow_mem((const char *)&entries[0].second, sizeof(double),
+                         sizeof(double), 0);
+  __nsan_dump_shadow_mem((const char *)&entries[1].second, sizeof(double),
+                         sizeof(double), 0);
+  // CHECK: {{.*}}: d0 d1 d2 d3 d4 d5 d6 d7 (1.02800000000000002487)
+  // CHECK-NEXT: {{.*}}: d0 d1 d2 d3 d4 d5 d6 d7 (7.95099999999999962341)
+  std::stable_sort(
+      entries.begin(), entries.end(),
+      [](const std::pair<int, double> &a, const std::pair<int, double> &b) {
+        return a.first < b.first;
+      });
+  __nsan_dump_shadow_mem((const char *)&entries[0].second, sizeof(double),
+                         sizeof(double), 0);
+  __nsan_dump_shadow_mem((const char *)&entries[1].second, sizeof(double),
+                         sizeof(double), 0);
+  // We make sure that the shadow values have been swapped correctly.
+  // CHECK-NEXT: {{.*}}: d0 d1 d2 d3 d4 d5 d6 d7 (7.95099999999999962341)
+  // CHECK-NEXT: {{.*}}: d0 d1 d2 d3 d4 d5 d6 d7 (1.02800000000000002487)
+}
+
+int main() {
+  std::vector<int> indices;
+  std::vector<double> values;
+  indices.push_back(75);
+  values.push_back(1.028);
+  indices.push_back(74);
+  values.push_back(7.951);
+  Run(indices, values);
+}
diff --git a/compiler-rt/test/nsan/swap.cpp b/compiler-rt/test/nsan/swap.cpp
new file mode 100644
index 000000000000000..0e1d9c8a880c02a
--- /dev/null
+++ b/compiler-rt/test/nsan/swap.cpp
@@ -0,0 +1,46 @@
+// RUN: %clangxx_nsan -O2 %s -o %t
+// RUN: %run %t 2>&1 | FileCheck %s
+
+// RUN: %clangxx_nsan -fno-builtin -O2 %s -o %t
+// RUN: %run %t 2>&1 | FileCheck %s
+
+// This verifies that shadow memory is tracked correcty across typed and
+// bitcasted swaps.
+
+#include <cassert>
+#include <cstddef>
+#include <cstdint>
+#include <utility>
+
+extern "C" void __nsan_dump_shadow_mem(const char *addr, size_t size_bytes,
+                                       size_t bytes_per_line, size_t reserved);
+
+__attribute__((noinline)) void SwapFT(double &a, double &b) {
+  // LLVM typically optimizes this to an untyped swap (through i64) anyway.
+  std::swap(a, b);
+}
+
+__attribute__((noinline)) void SwapBitcasted(uint64_t &a, uint64_t &b) {
+  std::swap(a, b);
+}
+
+int main() {
+  double a = 1.0, b = 2.0;
+  __nsan_dump_shadow_mem((const char *)&a, sizeof(a), sizeof(a), 0);
+  __nsan_dump_shadow_mem((const char *)&b, sizeof(b), sizeof(b), 0);
+  SwapFT(a, b);
+  __nsan_dump_shadow_mem((const char *)&a, sizeof(a), sizeof(a), 0);
+  __nsan_dump_shadow_mem((const char *)&b, sizeof(b), sizeof(b), 0);
+  assert(a == 2.0 && b == 1.0);
+  SwapBitcasted(*reinterpret_cast<uint64_t *>(&a),
+                *reinterpret_cast<uint64_t *>(&b));
+  __nsan_dump_shadow_mem((const char *)&a, sizeof(a), sizeof(a), 0);
+  __nsan_dump_shadow_mem((const char *)&b, sizeof(b), sizeof(b), 0);
+  assert(a == 1.0 && b == 2.0);
+  // CHECK: 0x{{[a-f0-9]*}}:    d0 d1 d2 d3 d4 d5 d6 d7   (1.0{{.*}}
+  // CHECK-NEXT: 0x{{[a-f0-9]*}}:    d0 d1 d2 d3 d4 d5 d6 d7   (2.0{{.*}}
+  // CHECK-NEXT: 0x{{[a-f0-9]*}}:    d0 d1 d2 d3 d4 d5 d6 d7   (2.0{{.*}}
+  // CHECK-NEXT: 0x{{[a-f0-9]*}}:    d0 d1 d2 d3 d4 d5 d6 d7   (1.0{{.*}}
+  // CHECK-NEXT: 0x{{[a-f0-9]*}}:    d0 d1 d2 d3 d4 d5 d6 d7   (1.0{{.*}}
+  // CHECK-NEXT: 0x{{[a-f0-9]*}}:    d0 d1 d2 d3 d4 d5 d6 d7   (2.0{{.*}}
+}

From 05d17a1c705e1053f95b90aa37d91ce4f94a9287 Mon Sep 17 00:00:00 2001
From: David Green <david.green@arm.com>
Date: Mon, 19 Aug 2024 18:50:47 +0100
Subject: [PATCH 398/441] [GlobalISel] Bail out early for big-endian (#103310)

If we continue through the function we can currently hit crashes. We can
bail out early and fall back to SDAG.

Fixes #103032
---
 llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp  |  1 +
 llvm/lib/Target/ARM/ARMCallLowering.cpp       |  9 ++++++++
 llvm/lib/Target/ARM/ARMCallLowering.h         |  2 ++
 .../AArch64/GlobalISel/endian_fallback.ll     | 21 +++++++++++++++++++
 .../ARM/GlobalISel/arm-irtranslator.ll        |  2 +-
 .../ARM/GlobalISel/arm-param-lowering.ll      |  2 +-
 6 files changed, 35 insertions(+), 2 deletions(-)
 create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/endian_fallback.ll

diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 8ceb729f0c156e9..f44af78cded46de 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -3889,6 +3889,7 @@ bool IRTranslator::runOnMachineFunction(MachineFunction &CurMF) {
                                F.getSubprogram(), &F.getEntryBlock());
     R << "unable to translate in big endian mode";
     reportTranslationError(*MF, *TPC, *ORE, R);
+    return false;
   }
 
   // Release the per-function state when we return, whether we succeeded or not.
diff --git a/llvm/lib/Target/ARM/ARMCallLowering.cpp b/llvm/lib/Target/ARM/ARMCallLowering.cpp
index 9cc162d041f48b8..883808ae981f5e4 100644
--- a/llvm/lib/Target/ARM/ARMCallLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMCallLowering.cpp
@@ -50,6 +50,13 @@
 
 using namespace llvm;
 
+// Whether Big-endian GISel is enabled, defaults to off, can be enabled for
+// testing.
+static cl::opt<bool>
+    EnableGISelBigEndian("enable-arm-gisel-bigendian", cl::Hidden,
+                         cl::init(false),
+                         cl::desc("Enable Global-ISel Big Endian Lowering"));
+
 ARMCallLowering::ARMCallLowering(const ARMTargetLowering &TLI)
     : CallLowering(&TLI) {}
 
@@ -539,3 +546,5 @@ bool ARMCallLowering::lowerCall(MachineIRBuilder &MIRBuilder, CallLoweringInfo &
 
   return true;
 }
+
+bool ARMCallLowering::enableBigEndian() const { return EnableGISelBigEndian; }
\ No newline at end of file
diff --git a/llvm/lib/Target/ARM/ARMCallLowering.h b/llvm/lib/Target/ARM/ARMCallLowering.h
index 38095617fb4f354..32c95a044d7b7ea 100644
--- a/llvm/lib/Target/ARM/ARMCallLowering.h
+++ b/llvm/lib/Target/ARM/ARMCallLowering.h
@@ -42,6 +42,8 @@ class ARMCallLowering : public CallLowering {
   bool lowerCall(MachineIRBuilder &MIRBuilder,
                  CallLoweringInfo &Info) const override;
 
+  bool enableBigEndian() const override;
+
 private:
   bool lowerReturnVal(MachineIRBuilder &MIRBuilder, const Value *Val,
                       ArrayRef<Register> VRegs,
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/endian_fallback.ll b/llvm/test/CodeGen/AArch64/GlobalISel/endian_fallback.ll
new file mode 100644
index 000000000000000..6c27b4dd85f9b09
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/endian_fallback.ll
@@ -0,0 +1,21 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple aarch64-unknown-linux-musl -O0 -global-isel-abort=2 < %s 2>&1 | FileCheck %s --check-prefix=CHECK-LE
+; RUN: llc -mtriple aarch64_be-unknown-linux-musl -O0 -global-isel-abort=2 < %s 2>&1 | FileCheck %s --check-prefix=CHECK-BE
+
+; Make sure we fall-back to SDAG for BE targets.
+
+; CHECK-LE-NOT: warning: Instruction selection used fallback path for foo
+; CHECK-BE: warning: Instruction selection used fallback path for foo
+
+define <4 x i6> @foo(float %0, <4 x i6> %1) {
+; CHECK-LE-LABEL: foo:
+; CHECK-LE:       // %bb.0:
+; CHECK-LE-NEXT:    fmov d0, d1
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: foo:
+; CHECK-BE:       // %bb.0:
+; CHECK-BE-NEXT:    fmov d0, d1
+; CHECK-BE-NEXT:    ret
+  ret <4 x i6> %1
+}
diff --git a/llvm/test/CodeGen/ARM/GlobalISel/arm-irtranslator.ll b/llvm/test/CodeGen/ARM/GlobalISel/arm-irtranslator.ll
index 411cf78b621f8bf..dc1d4b289c2ab81 100644
--- a/llvm/test/CodeGen/ARM/GlobalISel/arm-irtranslator.ll
+++ b/llvm/test/CodeGen/ARM/GlobalISel/arm-irtranslator.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -mtriple arm-unknown -mattr=+vfp2,+v4t -global-isel -stop-after=irtranslator -verify-machineinstrs %s -o - | FileCheck %s -check-prefix=CHECK -check-prefix=LITTLE
-; RUN: llc -mtriple armeb-unknown -mattr=+vfp2,+v4t -global-isel -global-isel-abort=0 -stop-after=irtranslator -verify-machineinstrs %s -o - | FileCheck %s -check-prefix=CHECK -check-prefix=BIG
+; RUN: llc -mtriple armeb-unknown -mattr=+vfp2,+v4t -global-isel -global-isel-abort=0 -enable-arm-gisel-bigendian -stop-after=irtranslator -verify-machineinstrs %s -o - | FileCheck %s -check-prefix=CHECK -check-prefix=BIG
 
 define void @test_void_return() {
 ; CHECK-LABEL: name: test_void_return
diff --git a/llvm/test/CodeGen/ARM/GlobalISel/arm-param-lowering.ll b/llvm/test/CodeGen/ARM/GlobalISel/arm-param-lowering.ll
index e8cd182196b6221..65586f72c7c198b 100644
--- a/llvm/test/CodeGen/ARM/GlobalISel/arm-param-lowering.ll
+++ b/llvm/test/CodeGen/ARM/GlobalISel/arm-param-lowering.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -O0 -mtriple arm-unknown -mattr=+vfp2,+v4t -global-isel -stop-after=irtranslator -verify-machineinstrs %s -o - | FileCheck %s -check-prefix=CHECK -check-prefix=ARM -check-prefix=LITTLE
-; RUN: llc -O0 -mtriple armeb-unknown -mattr=+vfp2,+v4t -global-isel -global-isel-abort=0 -stop-after=irtranslator -verify-machineinstrs %s -o - | FileCheck %s -check-prefix=CHECK -check-prefix=ARM -check-prefix=BIG
+; RUN: llc -O0 -mtriple armeb-unknown -mattr=+vfp2,+v4t -global-isel -global-isel-abort=0 -enable-arm-gisel-bigendian -stop-after=irtranslator -verify-machineinstrs %s -o - | FileCheck %s -check-prefix=CHECK -check-prefix=ARM -check-prefix=BIG
 ; RUN: llc -O0 -mtriple thumb-unknown -mattr=+vfp2,+v6t2 -global-isel -stop-after=irtranslator -verify-machineinstrs %s -o - | FileCheck %s -check-prefix=CHECK -check-prefix=LITTLE -check-prefix=THUMB
 
 declare arm_aapcscc ptr @simple_reg_params_target(i32, ptr)

From dde8280ded53f7167dcd2e8f6c50d7c9b5796a10 Mon Sep 17 00:00:00 2001
From: Angel Zhang <angel.zhang@amd.com>
Date: Mon, 19 Aug 2024 13:52:18 -0400
Subject: [PATCH 399/441] [mlir][spirv] Add `GroupNonUniformBallotFindLSB` and
 `GroupNonUniformBallotFindMSB` ops (#104791)

---
 .../mlir/Dialect/SPIRV/IR/SPIRVBase.td        | 495 +++++++++---------
 .../Dialect/SPIRV/IR/SPIRVNonUniformOps.td    | 112 ++++
 mlir/lib/Dialect/SPIRV/IR/GroupOps.cpp        |  24 +
 .../Dialect/SPIRV/IR/non-uniform-ops.mlir     |  56 ++
 4 files changed, 440 insertions(+), 247 deletions(-)

diff --git a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVBase.td b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVBase.td
index af0b2624feb327f..3b7da9b44a08fbf 100644
--- a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVBase.td
+++ b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVBase.td
@@ -4243,230 +4243,231 @@ class SPIRV_OpCode<string name, int val> {
 
 // Begin opcode section. Generated from SPIR-V spec; DO NOT MODIFY!
 
-def SPIRV_OC_OpNop                        : I32EnumAttrCase<"OpNop", 0>;
-def SPIRV_OC_OpUndef                      : I32EnumAttrCase<"OpUndef", 1>;
-def SPIRV_OC_OpSourceContinued            : I32EnumAttrCase<"OpSourceContinued", 2>;
-def SPIRV_OC_OpSource                     : I32EnumAttrCase<"OpSource", 3>;
-def SPIRV_OC_OpSourceExtension            : I32EnumAttrCase<"OpSourceExtension", 4>;
-def SPIRV_OC_OpName                       : I32EnumAttrCase<"OpName", 5>;
-def SPIRV_OC_OpMemberName                 : I32EnumAttrCase<"OpMemberName", 6>;
-def SPIRV_OC_OpString                     : I32EnumAttrCase<"OpString", 7>;
-def SPIRV_OC_OpLine                       : I32EnumAttrCase<"OpLine", 8>;
-def SPIRV_OC_OpExtension                  : I32EnumAttrCase<"OpExtension", 10>;
-def SPIRV_OC_OpExtInstImport              : I32EnumAttrCase<"OpExtInstImport", 11>;
-def SPIRV_OC_OpExtInst                    : I32EnumAttrCase<"OpExtInst", 12>;
-def SPIRV_OC_OpMemoryModel                : I32EnumAttrCase<"OpMemoryModel", 14>;
-def SPIRV_OC_OpEntryPoint                 : I32EnumAttrCase<"OpEntryPoint", 15>;
-def SPIRV_OC_OpExecutionMode              : I32EnumAttrCase<"OpExecutionMode", 16>;
-def SPIRV_OC_OpCapability                 : I32EnumAttrCase<"OpCapability", 17>;
-def SPIRV_OC_OpTypeVoid                   : I32EnumAttrCase<"OpTypeVoid", 19>;
-def SPIRV_OC_OpTypeBool                   : I32EnumAttrCase<"OpTypeBool", 20>;
-def SPIRV_OC_OpTypeInt                    : I32EnumAttrCase<"OpTypeInt", 21>;
-def SPIRV_OC_OpTypeFloat                  : I32EnumAttrCase<"OpTypeFloat", 22>;
-def SPIRV_OC_OpTypeVector                 : I32EnumAttrCase<"OpTypeVector", 23>;
-def SPIRV_OC_OpTypeMatrix                 : I32EnumAttrCase<"OpTypeMatrix", 24>;
-def SPIRV_OC_OpTypeImage                  : I32EnumAttrCase<"OpTypeImage", 25>;
-def SPIRV_OC_OpTypeSampledImage           : I32EnumAttrCase<"OpTypeSampledImage", 27>;
-def SPIRV_OC_OpTypeArray                  : I32EnumAttrCase<"OpTypeArray", 28>;
-def SPIRV_OC_OpTypeRuntimeArray           : I32EnumAttrCase<"OpTypeRuntimeArray", 29>;
-def SPIRV_OC_OpTypeStruct                 : I32EnumAttrCase<"OpTypeStruct", 30>;
-def SPIRV_OC_OpTypePointer                : I32EnumAttrCase<"OpTypePointer", 32>;
-def SPIRV_OC_OpTypeFunction               : I32EnumAttrCase<"OpTypeFunction", 33>;
-def SPIRV_OC_OpTypeForwardPointer         : I32EnumAttrCase<"OpTypeForwardPointer", 39>;
-def SPIRV_OC_OpConstantTrue               : I32EnumAttrCase<"OpConstantTrue", 41>;
-def SPIRV_OC_OpConstantFalse              : I32EnumAttrCase<"OpConstantFalse", 42>;
-def SPIRV_OC_OpConstant                   : I32EnumAttrCase<"OpConstant", 43>;
-def SPIRV_OC_OpConstantComposite          : I32EnumAttrCase<"OpConstantComposite", 44>;
-def SPIRV_OC_OpConstantNull               : I32EnumAttrCase<"OpConstantNull", 46>;
-def SPIRV_OC_OpSpecConstantTrue           : I32EnumAttrCase<"OpSpecConstantTrue", 48>;
-def SPIRV_OC_OpSpecConstantFalse          : I32EnumAttrCase<"OpSpecConstantFalse", 49>;
-def SPIRV_OC_OpSpecConstant               : I32EnumAttrCase<"OpSpecConstant", 50>;
-def SPIRV_OC_OpSpecConstantComposite      : I32EnumAttrCase<"OpSpecConstantComposite", 51>;
-def SPIRV_OC_OpSpecConstantOp             : I32EnumAttrCase<"OpSpecConstantOp", 52>;
-def SPIRV_OC_OpFunction                   : I32EnumAttrCase<"OpFunction", 54>;
-def SPIRV_OC_OpFunctionParameter          : I32EnumAttrCase<"OpFunctionParameter", 55>;
-def SPIRV_OC_OpFunctionEnd                : I32EnumAttrCase<"OpFunctionEnd", 56>;
-def SPIRV_OC_OpFunctionCall               : I32EnumAttrCase<"OpFunctionCall", 57>;
-def SPIRV_OC_OpVariable                   : I32EnumAttrCase<"OpVariable", 59>;
-def SPIRV_OC_OpLoad                       : I32EnumAttrCase<"OpLoad", 61>;
-def SPIRV_OC_OpStore                      : I32EnumAttrCase<"OpStore", 62>;
-def SPIRV_OC_OpCopyMemory                 : I32EnumAttrCase<"OpCopyMemory", 63>;
-def SPIRV_OC_OpAccessChain                : I32EnumAttrCase<"OpAccessChain", 65>;
-def SPIRV_OC_OpPtrAccessChain             : I32EnumAttrCase<"OpPtrAccessChain", 67>;
-def SPIRV_OC_OpInBoundsPtrAccessChain     : I32EnumAttrCase<"OpInBoundsPtrAccessChain", 70>;
-def SPIRV_OC_OpDecorate                   : I32EnumAttrCase<"OpDecorate", 71>;
-def SPIRV_OC_OpMemberDecorate             : I32EnumAttrCase<"OpMemberDecorate", 72>;
-def SPIRV_OC_OpVectorExtractDynamic       : I32EnumAttrCase<"OpVectorExtractDynamic", 77>;
-def SPIRV_OC_OpVectorInsertDynamic        : I32EnumAttrCase<"OpVectorInsertDynamic", 78>;
-def SPIRV_OC_OpVectorShuffle              : I32EnumAttrCase<"OpVectorShuffle", 79>;
-def SPIRV_OC_OpCompositeConstruct         : I32EnumAttrCase<"OpCompositeConstruct", 80>;
-def SPIRV_OC_OpCompositeExtract           : I32EnumAttrCase<"OpCompositeExtract", 81>;
-def SPIRV_OC_OpCompositeInsert            : I32EnumAttrCase<"OpCompositeInsert", 82>;
-def SPIRV_OC_OpTranspose                  : I32EnumAttrCase<"OpTranspose", 84>;
-def SPIRV_OC_OpImageDrefGather            : I32EnumAttrCase<"OpImageDrefGather", 97>;
-def SPIRV_OC_OpImage                      : I32EnumAttrCase<"OpImage", 100>;
-def SPIRV_OC_OpImageQuerySize             : I32EnumAttrCase<"OpImageQuerySize", 104>;
-def SPIRV_OC_OpConvertFToU                : I32EnumAttrCase<"OpConvertFToU", 109>;
-def SPIRV_OC_OpConvertFToS                : I32EnumAttrCase<"OpConvertFToS", 110>;
-def SPIRV_OC_OpConvertSToF                : I32EnumAttrCase<"OpConvertSToF", 111>;
-def SPIRV_OC_OpConvertUToF                : I32EnumAttrCase<"OpConvertUToF", 112>;
-def SPIRV_OC_OpUConvert                   : I32EnumAttrCase<"OpUConvert", 113>;
-def SPIRV_OC_OpSConvert                   : I32EnumAttrCase<"OpSConvert", 114>;
-def SPIRV_OC_OpFConvert                   : I32EnumAttrCase<"OpFConvert", 115>;
-def SPIRV_OC_OpConvertPtrToU              : I32EnumAttrCase<"OpConvertPtrToU", 117>;
-def SPIRV_OC_OpConvertUToPtr              : I32EnumAttrCase<"OpConvertUToPtr", 120>;
-def SPIRV_OC_OpPtrCastToGeneric           : I32EnumAttrCase<"OpPtrCastToGeneric", 121>;
-def SPIRV_OC_OpGenericCastToPtr           : I32EnumAttrCase<"OpGenericCastToPtr", 122>;
-def SPIRV_OC_OpGenericCastToPtrExplicit   : I32EnumAttrCase<"OpGenericCastToPtrExplicit", 123>;
-def SPIRV_OC_OpBitcast                    : I32EnumAttrCase<"OpBitcast", 124>;
-def SPIRV_OC_OpSNegate                    : I32EnumAttrCase<"OpSNegate", 126>;
-def SPIRV_OC_OpFNegate                    : I32EnumAttrCase<"OpFNegate", 127>;
-def SPIRV_OC_OpIAdd                       : I32EnumAttrCase<"OpIAdd", 128>;
-def SPIRV_OC_OpFAdd                       : I32EnumAttrCase<"OpFAdd", 129>;
-def SPIRV_OC_OpISub                       : I32EnumAttrCase<"OpISub", 130>;
-def SPIRV_OC_OpFSub                       : I32EnumAttrCase<"OpFSub", 131>;
-def SPIRV_OC_OpIMul                       : I32EnumAttrCase<"OpIMul", 132>;
-def SPIRV_OC_OpFMul                       : I32EnumAttrCase<"OpFMul", 133>;
-def SPIRV_OC_OpUDiv                       : I32EnumAttrCase<"OpUDiv", 134>;
-def SPIRV_OC_OpSDiv                       : I32EnumAttrCase<"OpSDiv", 135>;
-def SPIRV_OC_OpFDiv                       : I32EnumAttrCase<"OpFDiv", 136>;
-def SPIRV_OC_OpUMod                       : I32EnumAttrCase<"OpUMod", 137>;
-def SPIRV_OC_OpSRem                       : I32EnumAttrCase<"OpSRem", 138>;
-def SPIRV_OC_OpSMod                       : I32EnumAttrCase<"OpSMod", 139>;
-def SPIRV_OC_OpFRem                       : I32EnumAttrCase<"OpFRem", 140>;
-def SPIRV_OC_OpFMod                       : I32EnumAttrCase<"OpFMod", 141>;
-def SPIRV_OC_OpVectorTimesScalar          : I32EnumAttrCase<"OpVectorTimesScalar", 142>;
-def SPIRV_OC_OpMatrixTimesScalar          : I32EnumAttrCase<"OpMatrixTimesScalar", 143>;
-def SPIRV_OC_OpMatrixTimesMatrix          : I32EnumAttrCase<"OpMatrixTimesMatrix", 146>;
-def SPIRV_OC_OpDot                        : I32EnumAttrCase<"OpDot", 148>;
-def SPIRV_OC_OpIAddCarry                  : I32EnumAttrCase<"OpIAddCarry", 149>;
-def SPIRV_OC_OpISubBorrow                 : I32EnumAttrCase<"OpISubBorrow", 150>;
-def SPIRV_OC_OpUMulExtended               : I32EnumAttrCase<"OpUMulExtended", 151>;
-def SPIRV_OC_OpSMulExtended               : I32EnumAttrCase<"OpSMulExtended", 152>;
-def SPIRV_OC_OpIsNan                      : I32EnumAttrCase<"OpIsNan", 156>;
-def SPIRV_OC_OpIsInf                      : I32EnumAttrCase<"OpIsInf", 157>;
-def SPIRV_OC_OpOrdered                    : I32EnumAttrCase<"OpOrdered", 162>;
-def SPIRV_OC_OpUnordered                  : I32EnumAttrCase<"OpUnordered", 163>;
-def SPIRV_OC_OpLogicalEqual               : I32EnumAttrCase<"OpLogicalEqual", 164>;
-def SPIRV_OC_OpLogicalNotEqual            : I32EnumAttrCase<"OpLogicalNotEqual", 165>;
-def SPIRV_OC_OpLogicalOr                  : I32EnumAttrCase<"OpLogicalOr", 166>;
-def SPIRV_OC_OpLogicalAnd                 : I32EnumAttrCase<"OpLogicalAnd", 167>;
-def SPIRV_OC_OpLogicalNot                 : I32EnumAttrCase<"OpLogicalNot", 168>;
-def SPIRV_OC_OpSelect                     : I32EnumAttrCase<"OpSelect", 169>;
-def SPIRV_OC_OpIEqual                     : I32EnumAttrCase<"OpIEqual", 170>;
-def SPIRV_OC_OpINotEqual                  : I32EnumAttrCase<"OpINotEqual", 171>;
-def SPIRV_OC_OpUGreaterThan               : I32EnumAttrCase<"OpUGreaterThan", 172>;
-def SPIRV_OC_OpSGreaterThan               : I32EnumAttrCase<"OpSGreaterThan", 173>;
-def SPIRV_OC_OpUGreaterThanEqual          : I32EnumAttrCase<"OpUGreaterThanEqual", 174>;
-def SPIRV_OC_OpSGreaterThanEqual          : I32EnumAttrCase<"OpSGreaterThanEqual", 175>;
-def SPIRV_OC_OpULessThan                  : I32EnumAttrCase<"OpULessThan", 176>;
-def SPIRV_OC_OpSLessThan                  : I32EnumAttrCase<"OpSLessThan", 177>;
-def SPIRV_OC_OpULessThanEqual             : I32EnumAttrCase<"OpULessThanEqual", 178>;
-def SPIRV_OC_OpSLessThanEqual             : I32EnumAttrCase<"OpSLessThanEqual", 179>;
-def SPIRV_OC_OpFOrdEqual                  : I32EnumAttrCase<"OpFOrdEqual", 180>;
-def SPIRV_OC_OpFUnordEqual                : I32EnumAttrCase<"OpFUnordEqual", 181>;
-def SPIRV_OC_OpFOrdNotEqual               : I32EnumAttrCase<"OpFOrdNotEqual", 182>;
-def SPIRV_OC_OpFUnordNotEqual             : I32EnumAttrCase<"OpFUnordNotEqual", 183>;
-def SPIRV_OC_OpFOrdLessThan               : I32EnumAttrCase<"OpFOrdLessThan", 184>;
-def SPIRV_OC_OpFUnordLessThan             : I32EnumAttrCase<"OpFUnordLessThan", 185>;
-def SPIRV_OC_OpFOrdGreaterThan            : I32EnumAttrCase<"OpFOrdGreaterThan", 186>;
-def SPIRV_OC_OpFUnordGreaterThan          : I32EnumAttrCase<"OpFUnordGreaterThan", 187>;
-def SPIRV_OC_OpFOrdLessThanEqual          : I32EnumAttrCase<"OpFOrdLessThanEqual", 188>;
-def SPIRV_OC_OpFUnordLessThanEqual        : I32EnumAttrCase<"OpFUnordLessThanEqual", 189>;
-def SPIRV_OC_OpFOrdGreaterThanEqual       : I32EnumAttrCase<"OpFOrdGreaterThanEqual", 190>;
-def SPIRV_OC_OpFUnordGreaterThanEqual     : I32EnumAttrCase<"OpFUnordGreaterThanEqual", 191>;
-def SPIRV_OC_OpShiftRightLogical          : I32EnumAttrCase<"OpShiftRightLogical", 194>;
-def SPIRV_OC_OpShiftRightArithmetic       : I32EnumAttrCase<"OpShiftRightArithmetic", 195>;
-def SPIRV_OC_OpShiftLeftLogical           : I32EnumAttrCase<"OpShiftLeftLogical", 196>;
-def SPIRV_OC_OpBitwiseOr                  : I32EnumAttrCase<"OpBitwiseOr", 197>;
-def SPIRV_OC_OpBitwiseXor                 : I32EnumAttrCase<"OpBitwiseXor", 198>;
-def SPIRV_OC_OpBitwiseAnd                 : I32EnumAttrCase<"OpBitwiseAnd", 199>;
-def SPIRV_OC_OpNot                        : I32EnumAttrCase<"OpNot", 200>;
-def SPIRV_OC_OpBitFieldInsert             : I32EnumAttrCase<"OpBitFieldInsert", 201>;
-def SPIRV_OC_OpBitFieldSExtract           : I32EnumAttrCase<"OpBitFieldSExtract", 202>;
-def SPIRV_OC_OpBitFieldUExtract           : I32EnumAttrCase<"OpBitFieldUExtract", 203>;
-def SPIRV_OC_OpBitReverse                 : I32EnumAttrCase<"OpBitReverse", 204>;
-def SPIRV_OC_OpBitCount                   : I32EnumAttrCase<"OpBitCount", 205>;
-def SPIRV_OC_OpControlBarrier             : I32EnumAttrCase<"OpControlBarrier", 224>;
-def SPIRV_OC_OpMemoryBarrier              : I32EnumAttrCase<"OpMemoryBarrier", 225>;
-def SPIRV_OC_OpAtomicExchange             : I32EnumAttrCase<"OpAtomicExchange", 229>;
-def SPIRV_OC_OpAtomicCompareExchange      : I32EnumAttrCase<"OpAtomicCompareExchange", 230>;
-def SPIRV_OC_OpAtomicCompareExchangeWeak  : I32EnumAttrCase<"OpAtomicCompareExchangeWeak", 231>;
-def SPIRV_OC_OpAtomicIIncrement           : I32EnumAttrCase<"OpAtomicIIncrement", 232>;
-def SPIRV_OC_OpAtomicIDecrement           : I32EnumAttrCase<"OpAtomicIDecrement", 233>;
-def SPIRV_OC_OpAtomicIAdd                 : I32EnumAttrCase<"OpAtomicIAdd", 234>;
-def SPIRV_OC_OpAtomicISub                 : I32EnumAttrCase<"OpAtomicISub", 235>;
-def SPIRV_OC_OpAtomicSMin                 : I32EnumAttrCase<"OpAtomicSMin", 236>;
-def SPIRV_OC_OpAtomicUMin                 : I32EnumAttrCase<"OpAtomicUMin", 237>;
-def SPIRV_OC_OpAtomicSMax                 : I32EnumAttrCase<"OpAtomicSMax", 238>;
-def SPIRV_OC_OpAtomicUMax                 : I32EnumAttrCase<"OpAtomicUMax", 239>;
-def SPIRV_OC_OpAtomicAnd                  : I32EnumAttrCase<"OpAtomicAnd", 240>;
-def SPIRV_OC_OpAtomicOr                   : I32EnumAttrCase<"OpAtomicOr", 241>;
-def SPIRV_OC_OpAtomicXor                  : I32EnumAttrCase<"OpAtomicXor", 242>;
-def SPIRV_OC_OpPhi                        : I32EnumAttrCase<"OpPhi", 245>;
-def SPIRV_OC_OpLoopMerge                  : I32EnumAttrCase<"OpLoopMerge", 246>;
-def SPIRV_OC_OpSelectionMerge             : I32EnumAttrCase<"OpSelectionMerge", 247>;
-def SPIRV_OC_OpLabel                      : I32EnumAttrCase<"OpLabel", 248>;
-def SPIRV_OC_OpBranch                     : I32EnumAttrCase<"OpBranch", 249>;
-def SPIRV_OC_OpBranchConditional          : I32EnumAttrCase<"OpBranchConditional", 250>;
-def SPIRV_OC_OpReturn                     : I32EnumAttrCase<"OpReturn", 253>;
-def SPIRV_OC_OpReturnValue                : I32EnumAttrCase<"OpReturnValue", 254>;
-def SPIRV_OC_OpUnreachable                : I32EnumAttrCase<"OpUnreachable", 255>;
-def SPIRV_OC_OpGroupBroadcast             : I32EnumAttrCase<"OpGroupBroadcast", 263>;
-def SPIRV_OC_OpGroupIAdd                  : I32EnumAttrCase<"OpGroupIAdd", 264>;
-def SPIRV_OC_OpGroupFAdd                  : I32EnumAttrCase<"OpGroupFAdd", 265>;
-def SPIRV_OC_OpGroupFMin                  : I32EnumAttrCase<"OpGroupFMin", 266>;
-def SPIRV_OC_OpGroupUMin                  : I32EnumAttrCase<"OpGroupUMin", 267>;
-def SPIRV_OC_OpGroupSMin                  : I32EnumAttrCase<"OpGroupSMin", 268>;
-def SPIRV_OC_OpGroupFMax                  : I32EnumAttrCase<"OpGroupFMax", 269>;
-def SPIRV_OC_OpGroupUMax                  : I32EnumAttrCase<"OpGroupUMax", 270>;
-def SPIRV_OC_OpGroupSMax                  : I32EnumAttrCase<"OpGroupSMax", 271>;
-def SPIRV_OC_OpNoLine                     : I32EnumAttrCase<"OpNoLine", 317>;
-def SPIRV_OC_OpModuleProcessed            : I32EnumAttrCase<"OpModuleProcessed", 330>;
-def SPIRV_OC_OpGroupNonUniformElect       : I32EnumAttrCase<"OpGroupNonUniformElect", 333>;
-def SPIRV_OC_OpGroupNonUniformBroadcast   : I32EnumAttrCase<"OpGroupNonUniformBroadcast", 337>;
-def SPIRV_OC_OpGroupNonUniformBallot      : I32EnumAttrCase<"OpGroupNonUniformBallot", 339>;
-def SPIRV_OC_OpGroupNonUniformShuffle     : I32EnumAttrCase<"OpGroupNonUniformShuffle", 345>;
-def SPIRV_OC_OpGroupNonUniformShuffleXor  : I32EnumAttrCase<"OpGroupNonUniformShuffleXor", 346>;
-def SPIRV_OC_OpGroupNonUniformShuffleUp   : I32EnumAttrCase<"OpGroupNonUniformShuffleUp", 347>;
-def SPIRV_OC_OpGroupNonUniformShuffleDown : I32EnumAttrCase<"OpGroupNonUniformShuffleDown", 348>;
-def SPIRV_OC_OpGroupNonUniformIAdd        : I32EnumAttrCase<"OpGroupNonUniformIAdd", 349>;
-def SPIRV_OC_OpGroupNonUniformFAdd        : I32EnumAttrCase<"OpGroupNonUniformFAdd", 350>;
-def SPIRV_OC_OpGroupNonUniformIMul        : I32EnumAttrCase<"OpGroupNonUniformIMul", 351>;
-def SPIRV_OC_OpGroupNonUniformFMul        : I32EnumAttrCase<"OpGroupNonUniformFMul", 352>;
-def SPIRV_OC_OpGroupNonUniformSMin        : I32EnumAttrCase<"OpGroupNonUniformSMin", 353>;
-def SPIRV_OC_OpGroupNonUniformUMin        : I32EnumAttrCase<"OpGroupNonUniformUMin", 354>;
-def SPIRV_OC_OpGroupNonUniformFMin        : I32EnumAttrCase<"OpGroupNonUniformFMin", 355>;
-def SPIRV_OC_OpGroupNonUniformSMax        : I32EnumAttrCase<"OpGroupNonUniformSMax", 356>;
-def SPIRV_OC_OpGroupNonUniformUMax        : I32EnumAttrCase<"OpGroupNonUniformUMax", 357>;
-def SPIRV_OC_OpGroupNonUniformFMax        : I32EnumAttrCase<"OpGroupNonUniformFMax", 358>;
-def SPIRV_OC_OpGroupNonUniformBitwiseAnd  : I32EnumAttrCase<"OpGroupNonUniformBitwiseAnd", 359>;
-def SPIRV_OC_OpGroupNonUniformBitwiseOr   : I32EnumAttrCase<"OpGroupNonUniformBitwiseOr", 360>;
-def SPIRV_OC_OpGroupNonUniformBitwiseXor  : I32EnumAttrCase<"OpGroupNonUniformBitwiseXor", 361>;
-def SPIRV_OC_OpGroupNonUniformLogicalAnd  : I32EnumAttrCase<"OpGroupNonUniformLogicalAnd", 362>;
-def SPIRV_OC_OpGroupNonUniformLogicalOr   : I32EnumAttrCase<"OpGroupNonUniformLogicalOr", 363>;
-def SPIRV_OC_OpGroupNonUniformLogicalXor  : I32EnumAttrCase<"OpGroupNonUniformLogicalXor", 364>;
-def SPIRV_OC_OpSubgroupBallotKHR          : I32EnumAttrCase<"OpSubgroupBallotKHR", 4421>;
-def SPIRV_OC_OpSDot                       : I32EnumAttrCase<"OpSDot", 4450>;
-def SPIRV_OC_OpUDot                       : I32EnumAttrCase<"OpUDot", 4451>;
-def SPIRV_OC_OpSUDot                      : I32EnumAttrCase<"OpSUDot", 4452>;
-def SPIRV_OC_OpSDotAccSat                 : I32EnumAttrCase<"OpSDotAccSat", 4453>;
-def SPIRV_OC_OpUDotAccSat                 : I32EnumAttrCase<"OpUDotAccSat", 4454>;
-def SPIRV_OC_OpSUDotAccSat                : I32EnumAttrCase<"OpSUDotAccSat", 4455>;
-def SPIRV_OC_OpTypeCooperativeMatrixKHR   : I32EnumAttrCase<"OpTypeCooperativeMatrixKHR", 4456>;
-def SPIRV_OC_OpCooperativeMatrixLoadKHR   : I32EnumAttrCase<"OpCooperativeMatrixLoadKHR", 4457>;
-def SPIRV_OC_OpCooperativeMatrixStoreKHR  : I32EnumAttrCase<"OpCooperativeMatrixStoreKHR", 4458>;
-def SPIRV_OC_OpCooperativeMatrixMulAddKHR : I32EnumAttrCase<"OpCooperativeMatrixMulAddKHR", 4459>;
-def SPIRV_OC_OpCooperativeMatrixLengthKHR : I32EnumAttrCase<"OpCooperativeMatrixLengthKHR", 4460>;
-def SPIRV_OC_OpSubgroupBlockReadINTEL     : I32EnumAttrCase<"OpSubgroupBlockReadINTEL", 5575>;
-def SPIRV_OC_OpSubgroupBlockWriteINTEL    : I32EnumAttrCase<"OpSubgroupBlockWriteINTEL", 5576>;
-def SPIRV_OC_OpAssumeTrueKHR              : I32EnumAttrCase<"OpAssumeTrueKHR", 5630>;
-def SPIRV_OC_OpAtomicFAddEXT              : I32EnumAttrCase<"OpAtomicFAddEXT", 6035>;
-def SPIRV_OC_OpGroupIMulKHR               : I32EnumAttrCase<"OpGroupIMulKHR", 6401>;
-def SPIRV_OC_OpGroupFMulKHR               : I32EnumAttrCase<"OpGroupFMulKHR", 6402>;
-
-def SPIRV_OC_OpConvertFToBF16INTEL        : I32EnumAttrCase<"OpConvertFToBF16INTEL", 6116>;
-def SPIRV_OC_OpConvertBF16ToFINTEL        : I32EnumAttrCase<"OpConvertBF16ToFINTEL", 6117>;
+def SPIRV_OC_OpNop                          : I32EnumAttrCase<"OpNop", 0>;
+def SPIRV_OC_OpUndef                        : I32EnumAttrCase<"OpUndef", 1>;
+def SPIRV_OC_OpSourceContinued              : I32EnumAttrCase<"OpSourceContinued", 2>;
+def SPIRV_OC_OpSource                       : I32EnumAttrCase<"OpSource", 3>;
+def SPIRV_OC_OpSourceExtension              : I32EnumAttrCase<"OpSourceExtension", 4>;
+def SPIRV_OC_OpName                         : I32EnumAttrCase<"OpName", 5>;
+def SPIRV_OC_OpMemberName                   : I32EnumAttrCase<"OpMemberName", 6>;
+def SPIRV_OC_OpString                       : I32EnumAttrCase<"OpString", 7>;
+def SPIRV_OC_OpLine                         : I32EnumAttrCase<"OpLine", 8>;
+def SPIRV_OC_OpExtension                    : I32EnumAttrCase<"OpExtension", 10>;
+def SPIRV_OC_OpExtInstImport                : I32EnumAttrCase<"OpExtInstImport", 11>;
+def SPIRV_OC_OpExtInst                      : I32EnumAttrCase<"OpExtInst", 12>;
+def SPIRV_OC_OpMemoryModel                  : I32EnumAttrCase<"OpMemoryModel", 14>;
+def SPIRV_OC_OpEntryPoint                   : I32EnumAttrCase<"OpEntryPoint", 15>;
+def SPIRV_OC_OpExecutionMode                : I32EnumAttrCase<"OpExecutionMode", 16>;
+def SPIRV_OC_OpCapability                   : I32EnumAttrCase<"OpCapability", 17>;
+def SPIRV_OC_OpTypeVoid                     : I32EnumAttrCase<"OpTypeVoid", 19>;
+def SPIRV_OC_OpTypeBool                     : I32EnumAttrCase<"OpTypeBool", 20>;
+def SPIRV_OC_OpTypeInt                      : I32EnumAttrCase<"OpTypeInt", 21>;
+def SPIRV_OC_OpTypeFloat                    : I32EnumAttrCase<"OpTypeFloat", 22>;
+def SPIRV_OC_OpTypeVector                   : I32EnumAttrCase<"OpTypeVector", 23>;
+def SPIRV_OC_OpTypeMatrix                   : I32EnumAttrCase<"OpTypeMatrix", 24>;
+def SPIRV_OC_OpTypeImage                    : I32EnumAttrCase<"OpTypeImage", 25>;
+def SPIRV_OC_OpTypeSampledImage             : I32EnumAttrCase<"OpTypeSampledImage", 27>;
+def SPIRV_OC_OpTypeArray                    : I32EnumAttrCase<"OpTypeArray", 28>;
+def SPIRV_OC_OpTypeRuntimeArray             : I32EnumAttrCase<"OpTypeRuntimeArray", 29>;
+def SPIRV_OC_OpTypeStruct                   : I32EnumAttrCase<"OpTypeStruct", 30>;
+def SPIRV_OC_OpTypePointer                  : I32EnumAttrCase<"OpTypePointer", 32>;
+def SPIRV_OC_OpTypeFunction                 : I32EnumAttrCase<"OpTypeFunction", 33>;
+def SPIRV_OC_OpTypeForwardPointer           : I32EnumAttrCase<"OpTypeForwardPointer", 39>;
+def SPIRV_OC_OpConstantTrue                 : I32EnumAttrCase<"OpConstantTrue", 41>;
+def SPIRV_OC_OpConstantFalse                : I32EnumAttrCase<"OpConstantFalse", 42>;
+def SPIRV_OC_OpConstant                     : I32EnumAttrCase<"OpConstant", 43>;
+def SPIRV_OC_OpConstantComposite            : I32EnumAttrCase<"OpConstantComposite", 44>;
+def SPIRV_OC_OpConstantNull                 : I32EnumAttrCase<"OpConstantNull", 46>;
+def SPIRV_OC_OpSpecConstantTrue             : I32EnumAttrCase<"OpSpecConstantTrue", 48>;
+def SPIRV_OC_OpSpecConstantFalse            : I32EnumAttrCase<"OpSpecConstantFalse", 49>;
+def SPIRV_OC_OpSpecConstant                 : I32EnumAttrCase<"OpSpecConstant", 50>;
+def SPIRV_OC_OpSpecConstantComposite        : I32EnumAttrCase<"OpSpecConstantComposite", 51>;
+def SPIRV_OC_OpSpecConstantOp               : I32EnumAttrCase<"OpSpecConstantOp", 52>;
+def SPIRV_OC_OpFunction                     : I32EnumAttrCase<"OpFunction", 54>;
+def SPIRV_OC_OpFunctionParameter            : I32EnumAttrCase<"OpFunctionParameter", 55>;
+def SPIRV_OC_OpFunctionEnd                  : I32EnumAttrCase<"OpFunctionEnd", 56>;
+def SPIRV_OC_OpFunctionCall                 : I32EnumAttrCase<"OpFunctionCall", 57>;
+def SPIRV_OC_OpVariable                     : I32EnumAttrCase<"OpVariable", 59>;
+def SPIRV_OC_OpLoad                         : I32EnumAttrCase<"OpLoad", 61>;
+def SPIRV_OC_OpStore                        : I32EnumAttrCase<"OpStore", 62>;
+def SPIRV_OC_OpCopyMemory                   : I32EnumAttrCase<"OpCopyMemory", 63>;
+def SPIRV_OC_OpAccessChain                  : I32EnumAttrCase<"OpAccessChain", 65>;
+def SPIRV_OC_OpPtrAccessChain               : I32EnumAttrCase<"OpPtrAccessChain", 67>;
+def SPIRV_OC_OpInBoundsPtrAccessChain       : I32EnumAttrCase<"OpInBoundsPtrAccessChain", 70>;
+def SPIRV_OC_OpDecorate                     : I32EnumAttrCase<"OpDecorate", 71>;
+def SPIRV_OC_OpMemberDecorate               : I32EnumAttrCase<"OpMemberDecorate", 72>;
+def SPIRV_OC_OpVectorExtractDynamic         : I32EnumAttrCase<"OpVectorExtractDynamic", 77>;
+def SPIRV_OC_OpVectorInsertDynamic          : I32EnumAttrCase<"OpVectorInsertDynamic", 78>;
+def SPIRV_OC_OpVectorShuffle                : I32EnumAttrCase<"OpVectorShuffle", 79>;
+def SPIRV_OC_OpCompositeConstruct           : I32EnumAttrCase<"OpCompositeConstruct", 80>;
+def SPIRV_OC_OpCompositeExtract             : I32EnumAttrCase<"OpCompositeExtract", 81>;
+def SPIRV_OC_OpCompositeInsert              : I32EnumAttrCase<"OpCompositeInsert", 82>;
+def SPIRV_OC_OpTranspose                    : I32EnumAttrCase<"OpTranspose", 84>;
+def SPIRV_OC_OpImageDrefGather              : I32EnumAttrCase<"OpImageDrefGather", 97>;
+def SPIRV_OC_OpImage                        : I32EnumAttrCase<"OpImage", 100>;
+def SPIRV_OC_OpImageQuerySize               : I32EnumAttrCase<"OpImageQuerySize", 104>;
+def SPIRV_OC_OpConvertFToU                  : I32EnumAttrCase<"OpConvertFToU", 109>;
+def SPIRV_OC_OpConvertFToS                  : I32EnumAttrCase<"OpConvertFToS", 110>;
+def SPIRV_OC_OpConvertSToF                  : I32EnumAttrCase<"OpConvertSToF", 111>;
+def SPIRV_OC_OpConvertUToF                  : I32EnumAttrCase<"OpConvertUToF", 112>;
+def SPIRV_OC_OpUConvert                     : I32EnumAttrCase<"OpUConvert", 113>;
+def SPIRV_OC_OpSConvert                     : I32EnumAttrCase<"OpSConvert", 114>;
+def SPIRV_OC_OpFConvert                     : I32EnumAttrCase<"OpFConvert", 115>;
+def SPIRV_OC_OpConvertPtrToU                : I32EnumAttrCase<"OpConvertPtrToU", 117>;
+def SPIRV_OC_OpConvertUToPtr                : I32EnumAttrCase<"OpConvertUToPtr", 120>;
+def SPIRV_OC_OpPtrCastToGeneric             : I32EnumAttrCase<"OpPtrCastToGeneric", 121>;
+def SPIRV_OC_OpGenericCastToPtr             : I32EnumAttrCase<"OpGenericCastToPtr", 122>;
+def SPIRV_OC_OpGenericCastToPtrExplicit     : I32EnumAttrCase<"OpGenericCastToPtrExplicit", 123>;
+def SPIRV_OC_OpBitcast                      : I32EnumAttrCase<"OpBitcast", 124>;
+def SPIRV_OC_OpSNegate                      : I32EnumAttrCase<"OpSNegate", 126>;
+def SPIRV_OC_OpFNegate                      : I32EnumAttrCase<"OpFNegate", 127>;
+def SPIRV_OC_OpIAdd                         : I32EnumAttrCase<"OpIAdd", 128>;
+def SPIRV_OC_OpFAdd                         : I32EnumAttrCase<"OpFAdd", 129>;
+def SPIRV_OC_OpISub                         : I32EnumAttrCase<"OpISub", 130>;
+def SPIRV_OC_OpFSub                         : I32EnumAttrCase<"OpFSub", 131>;
+def SPIRV_OC_OpIMul                         : I32EnumAttrCase<"OpIMul", 132>;
+def SPIRV_OC_OpFMul                         : I32EnumAttrCase<"OpFMul", 133>;
+def SPIRV_OC_OpUDiv                         : I32EnumAttrCase<"OpUDiv", 134>;
+def SPIRV_OC_OpSDiv                         : I32EnumAttrCase<"OpSDiv", 135>;
+def SPIRV_OC_OpFDiv                         : I32EnumAttrCase<"OpFDiv", 136>;
+def SPIRV_OC_OpUMod                         : I32EnumAttrCase<"OpUMod", 137>;
+def SPIRV_OC_OpSRem                         : I32EnumAttrCase<"OpSRem", 138>;
+def SPIRV_OC_OpSMod                         : I32EnumAttrCase<"OpSMod", 139>;
+def SPIRV_OC_OpFRem                         : I32EnumAttrCase<"OpFRem", 140>;
+def SPIRV_OC_OpFMod                         : I32EnumAttrCase<"OpFMod", 141>;
+def SPIRV_OC_OpVectorTimesScalar            : I32EnumAttrCase<"OpVectorTimesScalar", 142>;
+def SPIRV_OC_OpMatrixTimesScalar            : I32EnumAttrCase<"OpMatrixTimesScalar", 143>;
+def SPIRV_OC_OpMatrixTimesMatrix            : I32EnumAttrCase<"OpMatrixTimesMatrix", 146>;
+def SPIRV_OC_OpDot                          : I32EnumAttrCase<"OpDot", 148>;
+def SPIRV_OC_OpIAddCarry                    : I32EnumAttrCase<"OpIAddCarry", 149>;
+def SPIRV_OC_OpISubBorrow                   : I32EnumAttrCase<"OpISubBorrow", 150>;
+def SPIRV_OC_OpUMulExtended                 : I32EnumAttrCase<"OpUMulExtended", 151>;
+def SPIRV_OC_OpSMulExtended                 : I32EnumAttrCase<"OpSMulExtended", 152>;
+def SPIRV_OC_OpIsNan                        : I32EnumAttrCase<"OpIsNan", 156>;
+def SPIRV_OC_OpIsInf                        : I32EnumAttrCase<"OpIsInf", 157>;
+def SPIRV_OC_OpOrdered                      : I32EnumAttrCase<"OpOrdered", 162>;
+def SPIRV_OC_OpUnordered                    : I32EnumAttrCase<"OpUnordered", 163>;
+def SPIRV_OC_OpLogicalEqual                 : I32EnumAttrCase<"OpLogicalEqual", 164>;
+def SPIRV_OC_OpLogicalNotEqual              : I32EnumAttrCase<"OpLogicalNotEqual", 165>;
+def SPIRV_OC_OpLogicalOr                    : I32EnumAttrCase<"OpLogicalOr", 166>;
+def SPIRV_OC_OpLogicalAnd                   : I32EnumAttrCase<"OpLogicalAnd", 167>;
+def SPIRV_OC_OpLogicalNot                   : I32EnumAttrCase<"OpLogicalNot", 168>;
+def SPIRV_OC_OpSelect                       : I32EnumAttrCase<"OpSelect", 169>;
+def SPIRV_OC_OpIEqual                       : I32EnumAttrCase<"OpIEqual", 170>;
+def SPIRV_OC_OpINotEqual                    : I32EnumAttrCase<"OpINotEqual", 171>;
+def SPIRV_OC_OpUGreaterThan                 : I32EnumAttrCase<"OpUGreaterThan", 172>;
+def SPIRV_OC_OpSGreaterThan                 : I32EnumAttrCase<"OpSGreaterThan", 173>;
+def SPIRV_OC_OpUGreaterThanEqual            : I32EnumAttrCase<"OpUGreaterThanEqual", 174>;
+def SPIRV_OC_OpSGreaterThanEqual            : I32EnumAttrCase<"OpSGreaterThanEqual", 175>;
+def SPIRV_OC_OpULessThan                    : I32EnumAttrCase<"OpULessThan", 176>;
+def SPIRV_OC_OpSLessThan                    : I32EnumAttrCase<"OpSLessThan", 177>;
+def SPIRV_OC_OpULessThanEqual               : I32EnumAttrCase<"OpULessThanEqual", 178>;
+def SPIRV_OC_OpSLessThanEqual               : I32EnumAttrCase<"OpSLessThanEqual", 179>;
+def SPIRV_OC_OpFOrdEqual                    : I32EnumAttrCase<"OpFOrdEqual", 180>;
+def SPIRV_OC_OpFUnordEqual                  : I32EnumAttrCase<"OpFUnordEqual", 181>;
+def SPIRV_OC_OpFOrdNotEqual                 : I32EnumAttrCase<"OpFOrdNotEqual", 182>;
+def SPIRV_OC_OpFUnordNotEqual               : I32EnumAttrCase<"OpFUnordNotEqual", 183>;
+def SPIRV_OC_OpFOrdLessThan                 : I32EnumAttrCase<"OpFOrdLessThan", 184>;
+def SPIRV_OC_OpFUnordLessThan               : I32EnumAttrCase<"OpFUnordLessThan", 185>;
+def SPIRV_OC_OpFOrdGreaterThan              : I32EnumAttrCase<"OpFOrdGreaterThan", 186>;
+def SPIRV_OC_OpFUnordGreaterThan            : I32EnumAttrCase<"OpFUnordGreaterThan", 187>;
+def SPIRV_OC_OpFOrdLessThanEqual            : I32EnumAttrCase<"OpFOrdLessThanEqual", 188>;
+def SPIRV_OC_OpFUnordLessThanEqual          : I32EnumAttrCase<"OpFUnordLessThanEqual", 189>;
+def SPIRV_OC_OpFOrdGreaterThanEqual         : I32EnumAttrCase<"OpFOrdGreaterThanEqual", 190>;
+def SPIRV_OC_OpFUnordGreaterThanEqual       : I32EnumAttrCase<"OpFUnordGreaterThanEqual", 191>;
+def SPIRV_OC_OpShiftRightLogical            : I32EnumAttrCase<"OpShiftRightLogical", 194>;
+def SPIRV_OC_OpShiftRightArithmetic         : I32EnumAttrCase<"OpShiftRightArithmetic", 195>;
+def SPIRV_OC_OpShiftLeftLogical             : I32EnumAttrCase<"OpShiftLeftLogical", 196>;
+def SPIRV_OC_OpBitwiseOr                    : I32EnumAttrCase<"OpBitwiseOr", 197>;
+def SPIRV_OC_OpBitwiseXor                   : I32EnumAttrCase<"OpBitwiseXor", 198>;
+def SPIRV_OC_OpBitwiseAnd                   : I32EnumAttrCase<"OpBitwiseAnd", 199>;
+def SPIRV_OC_OpNot                          : I32EnumAttrCase<"OpNot", 200>;
+def SPIRV_OC_OpBitFieldInsert               : I32EnumAttrCase<"OpBitFieldInsert", 201>;
+def SPIRV_OC_OpBitFieldSExtract             : I32EnumAttrCase<"OpBitFieldSExtract", 202>;
+def SPIRV_OC_OpBitFieldUExtract             : I32EnumAttrCase<"OpBitFieldUExtract", 203>;
+def SPIRV_OC_OpBitReverse                   : I32EnumAttrCase<"OpBitReverse", 204>;
+def SPIRV_OC_OpBitCount                     : I32EnumAttrCase<"OpBitCount", 205>;
+def SPIRV_OC_OpControlBarrier               : I32EnumAttrCase<"OpControlBarrier", 224>;
+def SPIRV_OC_OpMemoryBarrier                : I32EnumAttrCase<"OpMemoryBarrier", 225>;
+def SPIRV_OC_OpAtomicExchange               : I32EnumAttrCase<"OpAtomicExchange", 229>;
+def SPIRV_OC_OpAtomicCompareExchange        : I32EnumAttrCase<"OpAtomicCompareExchange", 230>;
+def SPIRV_OC_OpAtomicCompareExchangeWeak    : I32EnumAttrCase<"OpAtomicCompareExchangeWeak", 231>;
+def SPIRV_OC_OpAtomicIIncrement             : I32EnumAttrCase<"OpAtomicIIncrement", 232>;
+def SPIRV_OC_OpAtomicIDecrement             : I32EnumAttrCase<"OpAtomicIDecrement", 233>;
+def SPIRV_OC_OpAtomicIAdd                   : I32EnumAttrCase<"OpAtomicIAdd", 234>;
+def SPIRV_OC_OpAtomicISub                   : I32EnumAttrCase<"OpAtomicISub", 235>;
+def SPIRV_OC_OpAtomicSMin                   : I32EnumAttrCase<"OpAtomicSMin", 236>;
+def SPIRV_OC_OpAtomicUMin                   : I32EnumAttrCase<"OpAtomicUMin", 237>;
+def SPIRV_OC_OpAtomicSMax                   : I32EnumAttrCase<"OpAtomicSMax", 238>;
+def SPIRV_OC_OpAtomicUMax                   : I32EnumAttrCase<"OpAtomicUMax", 239>;
+def SPIRV_OC_OpAtomicAnd                    : I32EnumAttrCase<"OpAtomicAnd", 240>;
+def SPIRV_OC_OpAtomicOr                     : I32EnumAttrCase<"OpAtomicOr", 241>;
+def SPIRV_OC_OpAtomicXor                    : I32EnumAttrCase<"OpAtomicXor", 242>;
+def SPIRV_OC_OpPhi                          : I32EnumAttrCase<"OpPhi", 245>;
+def SPIRV_OC_OpLoopMerge                    : I32EnumAttrCase<"OpLoopMerge", 246>;
+def SPIRV_OC_OpSelectionMerge               : I32EnumAttrCase<"OpSelectionMerge", 247>;
+def SPIRV_OC_OpLabel                        : I32EnumAttrCase<"OpLabel", 248>;
+def SPIRV_OC_OpBranch                       : I32EnumAttrCase<"OpBranch", 249>;
+def SPIRV_OC_OpBranchConditional            : I32EnumAttrCase<"OpBranchConditional", 250>;
+def SPIRV_OC_OpReturn                       : I32EnumAttrCase<"OpReturn", 253>;
+def SPIRV_OC_OpReturnValue                  : I32EnumAttrCase<"OpReturnValue", 254>;
+def SPIRV_OC_OpUnreachable                  : I32EnumAttrCase<"OpUnreachable", 255>;
+def SPIRV_OC_OpGroupBroadcast               : I32EnumAttrCase<"OpGroupBroadcast", 263>;
+def SPIRV_OC_OpGroupIAdd                    : I32EnumAttrCase<"OpGroupIAdd", 264>;
+def SPIRV_OC_OpGroupFAdd                    : I32EnumAttrCase<"OpGroupFAdd", 265>;
+def SPIRV_OC_OpGroupFMin                    : I32EnumAttrCase<"OpGroupFMin", 266>;
+def SPIRV_OC_OpGroupUMin                    : I32EnumAttrCase<"OpGroupUMin", 267>;
+def SPIRV_OC_OpGroupSMin                    : I32EnumAttrCase<"OpGroupSMin", 268>;
+def SPIRV_OC_OpGroupFMax                    : I32EnumAttrCase<"OpGroupFMax", 269>;
+def SPIRV_OC_OpGroupUMax                    : I32EnumAttrCase<"OpGroupUMax", 270>;
+def SPIRV_OC_OpGroupSMax                    : I32EnumAttrCase<"OpGroupSMax", 271>;
+def SPIRV_OC_OpNoLine                       : I32EnumAttrCase<"OpNoLine", 317>;
+def SPIRV_OC_OpModuleProcessed              : I32EnumAttrCase<"OpModuleProcessed", 330>;
+def SPIRV_OC_OpGroupNonUniformElect         : I32EnumAttrCase<"OpGroupNonUniformElect", 333>;
+def SPIRV_OC_OpGroupNonUniformBroadcast     : I32EnumAttrCase<"OpGroupNonUniformBroadcast", 337>;
+def SPIRV_OC_OpGroupNonUniformBallot        : I32EnumAttrCase<"OpGroupNonUniformBallot", 339>;
+def SPIRV_OC_OpGroupNonUniformBallotFindLSB : I32EnumAttrCase<"OpGroupNonUniformBallotFindLSB", 343>;
+def SPIRV_OC_OpGroupNonUniformBallotFindMSB : I32EnumAttrCase<"OpGroupNonUniformBallotFindMSB", 344>;
+def SPIRV_OC_OpGroupNonUniformShuffle       : I32EnumAttrCase<"OpGroupNonUniformShuffle", 345>;
+def SPIRV_OC_OpGroupNonUniformShuffleXor    : I32EnumAttrCase<"OpGroupNonUniformShuffleXor", 346>;
+def SPIRV_OC_OpGroupNonUniformShuffleUp     : I32EnumAttrCase<"OpGroupNonUniformShuffleUp", 347>;
+def SPIRV_OC_OpGroupNonUniformShuffleDown   : I32EnumAttrCase<"OpGroupNonUniformShuffleDown", 348>;
+def SPIRV_OC_OpGroupNonUniformIAdd          : I32EnumAttrCase<"OpGroupNonUniformIAdd", 349>;
+def SPIRV_OC_OpGroupNonUniformFAdd          : I32EnumAttrCase<"OpGroupNonUniformFAdd", 350>;
+def SPIRV_OC_OpGroupNonUniformIMul          : I32EnumAttrCase<"OpGroupNonUniformIMul", 351>;
+def SPIRV_OC_OpGroupNonUniformFMul          : I32EnumAttrCase<"OpGroupNonUniformFMul", 352>;
+def SPIRV_OC_OpGroupNonUniformSMin          : I32EnumAttrCase<"OpGroupNonUniformSMin", 353>;
+def SPIRV_OC_OpGroupNonUniformUMin          : I32EnumAttrCase<"OpGroupNonUniformUMin", 354>;
+def SPIRV_OC_OpGroupNonUniformFMin          : I32EnumAttrCase<"OpGroupNonUniformFMin", 355>;
+def SPIRV_OC_OpGroupNonUniformSMax          : I32EnumAttrCase<"OpGroupNonUniformSMax", 356>;
+def SPIRV_OC_OpGroupNonUniformUMax          : I32EnumAttrCase<"OpGroupNonUniformUMax", 357>;
+def SPIRV_OC_OpGroupNonUniformFMax          : I32EnumAttrCase<"OpGroupNonUniformFMax", 358>;
+def SPIRV_OC_OpGroupNonUniformBitwiseAnd    : I32EnumAttrCase<"OpGroupNonUniformBitwiseAnd", 359>;
+def SPIRV_OC_OpGroupNonUniformBitwiseOr     : I32EnumAttrCase<"OpGroupNonUniformBitwiseOr", 360>;
+def SPIRV_OC_OpGroupNonUniformBitwiseXor    : I32EnumAttrCase<"OpGroupNonUniformBitwiseXor", 361>;
+def SPIRV_OC_OpGroupNonUniformLogicalAnd    : I32EnumAttrCase<"OpGroupNonUniformLogicalAnd", 362>;
+def SPIRV_OC_OpGroupNonUniformLogicalOr     : I32EnumAttrCase<"OpGroupNonUniformLogicalOr", 363>;
+def SPIRV_OC_OpGroupNonUniformLogicalXor    : I32EnumAttrCase<"OpGroupNonUniformLogicalXor", 364>;
+def SPIRV_OC_OpSubgroupBallotKHR            : I32EnumAttrCase<"OpSubgroupBallotKHR", 4421>;
+def SPIRV_OC_OpSDot                         : I32EnumAttrCase<"OpSDot", 4450>;
+def SPIRV_OC_OpUDot                         : I32EnumAttrCase<"OpUDot", 4451>;
+def SPIRV_OC_OpSUDot                        : I32EnumAttrCase<"OpSUDot", 4452>;
+def SPIRV_OC_OpSDotAccSat                   : I32EnumAttrCase<"OpSDotAccSat", 4453>;
+def SPIRV_OC_OpUDotAccSat                   : I32EnumAttrCase<"OpUDotAccSat", 4454>;
+def SPIRV_OC_OpSUDotAccSat                  : I32EnumAttrCase<"OpSUDotAccSat", 4455>;
+def SPIRV_OC_OpTypeCooperativeMatrixKHR     : I32EnumAttrCase<"OpTypeCooperativeMatrixKHR", 4456>;
+def SPIRV_OC_OpCooperativeMatrixLoadKHR     : I32EnumAttrCase<"OpCooperativeMatrixLoadKHR", 4457>;
+def SPIRV_OC_OpCooperativeMatrixStoreKHR    : I32EnumAttrCase<"OpCooperativeMatrixStoreKHR", 4458>;
+def SPIRV_OC_OpCooperativeMatrixMulAddKHR   : I32EnumAttrCase<"OpCooperativeMatrixMulAddKHR", 4459>;
+def SPIRV_OC_OpCooperativeMatrixLengthKHR   : I32EnumAttrCase<"OpCooperativeMatrixLengthKHR", 4460>;
+def SPIRV_OC_OpSubgroupBlockReadINTEL       : I32EnumAttrCase<"OpSubgroupBlockReadINTEL", 5575>;
+def SPIRV_OC_OpSubgroupBlockWriteINTEL      : I32EnumAttrCase<"OpSubgroupBlockWriteINTEL", 5576>;
+def SPIRV_OC_OpAssumeTrueKHR                : I32EnumAttrCase<"OpAssumeTrueKHR", 5630>;
+def SPIRV_OC_OpAtomicFAddEXT                : I32EnumAttrCase<"OpAtomicFAddEXT", 6035>;
+def SPIRV_OC_OpConvertFToBF16INTEL          : I32EnumAttrCase<"OpConvertFToBF16INTEL", 6116>;
+def SPIRV_OC_OpConvertBF16ToFINTEL          : I32EnumAttrCase<"OpConvertBF16ToFINTEL", 6117>;
+def SPIRV_OC_OpGroupIMulKHR                 : I32EnumAttrCase<"OpGroupIMulKHR", 6401>;
+def SPIRV_OC_OpGroupFMulKHR                 : I32EnumAttrCase<"OpGroupFMulKHR", 6402>;
 
 def SPIRV_OpcodeAttr :
     SPIRV_I32EnumAttr<"Opcode", "valid SPIR-V instructions", "opcode", [
@@ -4503,10 +4504,10 @@ def SPIRV_OpcodeAttr :
       SPIRV_OC_OpFSub, SPIRV_OC_OpIMul, SPIRV_OC_OpFMul, SPIRV_OC_OpUDiv,
       SPIRV_OC_OpSDiv, SPIRV_OC_OpFDiv, SPIRV_OC_OpUMod, SPIRV_OC_OpSRem,
       SPIRV_OC_OpSMod, SPIRV_OC_OpFRem, SPIRV_OC_OpFMod,
-      SPIRV_OC_OpVectorTimesScalar, SPIRV_OC_OpMatrixTimesScalar, SPIRV_OC_OpDot,
-      SPIRV_OC_OpMatrixTimesMatrix, SPIRV_OC_OpIAddCarry, SPIRV_OC_OpISubBorrow,
-      SPIRV_OC_OpUMulExtended, SPIRV_OC_OpSMulExtended, SPIRV_OC_OpIsNan,
-      SPIRV_OC_OpIsInf, SPIRV_OC_OpOrdered, SPIRV_OC_OpUnordered,
+      SPIRV_OC_OpVectorTimesScalar, SPIRV_OC_OpMatrixTimesScalar,
+      SPIRV_OC_OpMatrixTimesMatrix, SPIRV_OC_OpDot, SPIRV_OC_OpIAddCarry,
+      SPIRV_OC_OpISubBorrow, SPIRV_OC_OpUMulExtended, SPIRV_OC_OpSMulExtended,
+      SPIRV_OC_OpIsNan, SPIRV_OC_OpIsInf, SPIRV_OC_OpOrdered, SPIRV_OC_OpUnordered,
       SPIRV_OC_OpLogicalEqual, SPIRV_OC_OpLogicalNotEqual, SPIRV_OC_OpLogicalOr,
       SPIRV_OC_OpLogicalAnd, SPIRV_OC_OpLogicalNot, SPIRV_OC_OpSelect,
       SPIRV_OC_OpIEqual, SPIRV_OC_OpINotEqual, SPIRV_OC_OpUGreaterThan,
@@ -4536,27 +4537,27 @@ def SPIRV_OpcodeAttr :
       SPIRV_OC_OpGroupFMax, SPIRV_OC_OpGroupUMax, SPIRV_OC_OpGroupSMax,
       SPIRV_OC_OpNoLine, SPIRV_OC_OpModuleProcessed, SPIRV_OC_OpGroupNonUniformElect,
       SPIRV_OC_OpGroupNonUniformBroadcast, SPIRV_OC_OpGroupNonUniformBallot,
-      SPIRV_OC_OpGroupNonUniformShuffle, SPIRV_OC_OpGroupNonUniformShuffleXor,
-      SPIRV_OC_OpGroupNonUniformShuffleUp, SPIRV_OC_OpGroupNonUniformShuffleDown,
-      SPIRV_OC_OpGroupNonUniformIAdd, SPIRV_OC_OpGroupNonUniformFAdd,
-      SPIRV_OC_OpGroupNonUniformIMul, SPIRV_OC_OpGroupNonUniformFMul,
-      SPIRV_OC_OpGroupNonUniformSMin, SPIRV_OC_OpGroupNonUniformUMin,
-      SPIRV_OC_OpGroupNonUniformFMin, SPIRV_OC_OpGroupNonUniformSMax,
-      SPIRV_OC_OpGroupNonUniformUMax, SPIRV_OC_OpGroupNonUniformFMax,
-      SPIRV_OC_OpGroupNonUniformBitwiseAnd, SPIRV_OC_OpGroupNonUniformBitwiseOr,
-      SPIRV_OC_OpGroupNonUniformBitwiseXor, SPIRV_OC_OpGroupNonUniformLogicalAnd,
-      SPIRV_OC_OpGroupNonUniformLogicalOr, SPIRV_OC_OpGroupNonUniformLogicalXor,
-      SPIRV_OC_OpSubgroupBallotKHR, SPIRV_OC_OpSDot, SPIRV_OC_OpUDot,
-      SPIRV_OC_OpSUDot, SPIRV_OC_OpSDotAccSat, SPIRV_OC_OpUDotAccSat,
-      SPIRV_OC_OpSUDotAccSat,
+      SPIRV_OC_OpGroupNonUniformBallotFindLSB,
+      SPIRV_OC_OpGroupNonUniformBallotFindMSB, SPIRV_OC_OpGroupNonUniformShuffle,
+      SPIRV_OC_OpGroupNonUniformShuffleXor, SPIRV_OC_OpGroupNonUniformShuffleUp,
+      SPIRV_OC_OpGroupNonUniformShuffleDown, SPIRV_OC_OpGroupNonUniformIAdd,
+      SPIRV_OC_OpGroupNonUniformFAdd, SPIRV_OC_OpGroupNonUniformIMul,
+      SPIRV_OC_OpGroupNonUniformFMul, SPIRV_OC_OpGroupNonUniformSMin,
+      SPIRV_OC_OpGroupNonUniformUMin, SPIRV_OC_OpGroupNonUniformFMin,
+      SPIRV_OC_OpGroupNonUniformSMax, SPIRV_OC_OpGroupNonUniformUMax,
+      SPIRV_OC_OpGroupNonUniformFMax, SPIRV_OC_OpGroupNonUniformBitwiseAnd,
+      SPIRV_OC_OpGroupNonUniformBitwiseOr, SPIRV_OC_OpGroupNonUniformBitwiseXor,
+      SPIRV_OC_OpGroupNonUniformLogicalAnd, SPIRV_OC_OpGroupNonUniformLogicalOr,
+      SPIRV_OC_OpGroupNonUniformLogicalXor, SPIRV_OC_OpSubgroupBallotKHR,
+      SPIRV_OC_OpSDot, SPIRV_OC_OpUDot, SPIRV_OC_OpSUDot, SPIRV_OC_OpSDotAccSat,
+      SPIRV_OC_OpUDotAccSat, SPIRV_OC_OpSUDotAccSat,
       SPIRV_OC_OpTypeCooperativeMatrixKHR, SPIRV_OC_OpCooperativeMatrixLoadKHR,
       SPIRV_OC_OpCooperativeMatrixStoreKHR, SPIRV_OC_OpCooperativeMatrixMulAddKHR,
-      SPIRV_OC_OpCooperativeMatrixLengthKHR,
-      SPIRV_OC_OpSubgroupBlockReadINTEL, SPIRV_OC_OpSubgroupBlockWriteINTEL,
-      SPIRV_OC_OpAssumeTrueKHR, SPIRV_OC_OpAtomicFAddEXT, SPIRV_OC_OpGroupIMulKHR,
-      SPIRV_OC_OpGroupFMulKHR,
-
-      SPIRV_OC_OpConvertFToBF16INTEL, SPIRV_OC_OpConvertBF16ToFINTEL
+      SPIRV_OC_OpCooperativeMatrixLengthKHR, SPIRV_OC_OpSubgroupBlockReadINTEL,
+      SPIRV_OC_OpSubgroupBlockWriteINTEL, SPIRV_OC_OpAssumeTrueKHR,
+      SPIRV_OC_OpAtomicFAddEXT, SPIRV_OC_OpConvertFToBF16INTEL,
+      SPIRV_OC_OpConvertBF16ToFINTEL, SPIRV_OC_OpGroupIMulKHR,
+      SPIRV_OC_OpGroupFMulKHR
     ]>;
 
 // End opcode section. Generated from SPIR-V spec; DO NOT MODIFY!
diff --git a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVNonUniformOps.td b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVNonUniformOps.td
index c144dc52d4ea0c3..a32f625ae821127 100644
--- a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVNonUniformOps.td
+++ b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVNonUniformOps.td
@@ -83,6 +83,118 @@ def SPIRV_GroupNonUniformBallotOp : SPIRV_Op<"GroupNonUniformBallot", []> {
 
 // -----
 
+def SPIRV_GroupNonUniformBallotFindLSBOp : SPIRV_Op<"GroupNonUniformBallotFindLSB", []> {
+  let summary = [{
+    Find the least significant bit set to 1 in Value, considering only the
+    bits in Value required to represent all bits of the group's invocations.
+    If none of the considered bits is set to 1, the resulting value is
+    undefined.
+  }];
+
+  let description = [{
+    Result Type must be a scalar of integer type, whose Signedness operand
+    is 0.
+
+    Execution is a Scope that identifies the group of invocations affected
+    by this command. It must be Subgroup.
+
+    Value must be a vector of four components of integer type scalar, whose
+    Width operand is 32 and whose Signedness operand is 0.
+
+    Value is a set of bitfields where the first invocation is represented in
+    the lowest bit of the first vector component and the last (up to the
+    size of the group) is the higher bit number of the last bitmask needed
+    to represent all bits of the group invocations.
+
+    <!-- End of AutoGen section -->
+
+    #### Example:
+
+    ```mlir
+    %vector = ... : vector<4xi32>
+    %0 = spirv.GroupNonUniformBallotFindLSB <Subgroup> %vector : vector<4xi32>, i32
+    ```
+  }];
+
+  let availability = [
+    MinVersion<SPIRV_V_1_3>,
+    MaxVersion<SPIRV_V_1_6>,
+    Extension<[]>,
+    Capability<[SPIRV_C_GroupNonUniformBallot]>
+  ];
+
+  let arguments = (ins
+    SPIRV_ScopeAttr:$execution_scope,
+    SPIRV_IOrUIVec4:$value
+  );
+
+  let results = (outs
+    SPIRV_SignlessOrUnsignedInt:$result
+  );
+
+  let assemblyFormat = [{
+    $execution_scope $value attr-dict `:` type($value) `,` type($result)
+  }];
+}
+
+// -----
+
+def SPIRV_GroupNonUniformBallotFindMSBOp : SPIRV_Op<"GroupNonUniformBallotFindMSB", []> {
+  let summary = [{
+    Find the most significant bit set to 1 in Value, considering only the
+    bits in Value required to represent all bits of the group's invocations.
+    If none of the considered bits is set to 1, the resulting value is
+    undefined.
+  }];
+
+  let description = [{
+    Result Type must be a scalar of integer type, whose Signedness operand
+    is 0.
+
+    Execution is a Scope that identifies the group of invocations affected
+    by this command. It must be Subgroup.
+
+    Value must be a vector of four components of integer type scalar, whose
+    Width operand is 32 and whose Signedness operand is 0.
+
+    Value is a set of bitfields where the first invocation is represented in
+    the lowest bit of the first vector component and the last (up to the
+    size of the group) is the higher bit number of the last bitmask needed
+    to represent all bits of the group invocations.
+
+    <!-- End of AutoGen section -->
+
+    #### Example:
+
+    ```mlir
+    %vector = ... : vector<4xi32>
+    %0 = spirv.GroupNonUniformBallotFindMSB <Subgroup> %vector : vector<4xi32>, i32
+    ```
+  }];
+
+  let availability = [
+    MinVersion<SPIRV_V_1_3>,
+    MaxVersion<SPIRV_V_1_6>,
+    Extension<[]>,
+    Capability<[SPIRV_C_GroupNonUniformBallot]>
+  ];
+
+  let arguments = (ins
+    SPIRV_ScopeAttr:$execution_scope,
+    SPIRV_IOrUIVec4:$value
+  );
+
+  let results = (outs
+    SPIRV_SignlessOrUnsignedInt:$result
+  );
+
+  let assemblyFormat = [{
+    $execution_scope $value attr-dict `:` type($value) `,` type($result)
+  }];
+}
+
+// -----
+
 def SPIRV_GroupNonUniformBroadcastOp : SPIRV_Op<"GroupNonUniformBroadcast",
   [Pure, AllTypesMatch<["value", "result"]>]> {
   let summary = [{
diff --git a/mlir/lib/Dialect/SPIRV/IR/GroupOps.cpp b/mlir/lib/Dialect/SPIRV/IR/GroupOps.cpp
index fcbef2be75f9a09..2e5a2aab52a1606 100644
--- a/mlir/lib/Dialect/SPIRV/IR/GroupOps.cpp
+++ b/mlir/lib/Dialect/SPIRV/IR/GroupOps.cpp
@@ -150,6 +150,30 @@ LogicalResult GroupNonUniformBallotOp::verify() {
   return success();
 }
 
+//===----------------------------------------------------------------------===//
+// spirv.GroupNonUniformBallotFindLSBOp
+//===----------------------------------------------------------------------===//
+
+LogicalResult GroupNonUniformBallotFindLSBOp::verify() {
+  spirv::Scope scope = getExecutionScope();
+  if (scope != spirv::Scope::Workgroup && scope != spirv::Scope::Subgroup)
+    return emitOpError("execution scope must be 'Workgroup' or 'Subgroup'");
+
+  return success();
+}
+
+//===----------------------------------------------------------------------===//
+// spirv.GroupNonUniformBallotFindLSBOp
+//===----------------------------------------------------------------------===//
+
+LogicalResult GroupNonUniformBallotFindMSBOp::verify() {
+  spirv::Scope scope = getExecutionScope();
+  if (scope != spirv::Scope::Workgroup && scope != spirv::Scope::Subgroup)
+    return emitOpError("execution scope must be 'Workgroup' or 'Subgroup'");
+
+  return success();
+}
+
 //===----------------------------------------------------------------------===//
 // spirv.GroupNonUniformBroadcast
 //===----------------------------------------------------------------------===//
diff --git a/mlir/test/Dialect/SPIRV/IR/non-uniform-ops.mlir b/mlir/test/Dialect/SPIRV/IR/non-uniform-ops.mlir
index f7fd05b36bae062..d8a26c71d12f91d 100644
--- a/mlir/test/Dialect/SPIRV/IR/non-uniform-ops.mlir
+++ b/mlir/test/Dialect/SPIRV/IR/non-uniform-ops.mlir
@@ -28,6 +28,62 @@ func.func @group_non_uniform_ballot(%predicate: i1) -> vector<4xsi32> {
 
 // -----
 
+//===----------------------------------------------------------------------===//
+// spirv.GroupNonUniformBallotFindLSB
+//===----------------------------------------------------------------------===//
+
+func.func @group_non_uniform_ballot_find_lsb(%value : vector<4xi32>) -> i32 {
+  // CHECK: %{{.*}} = spirv.GroupNonUniformBallotFindLSB <Subgroup> %{{.*}}: vector<4xi32>, i32
+  %0 = spirv.GroupNonUniformBallotFindLSB <Subgroup> %value : vector<4xi32>, i32
+  return %0: i32
+}
+
+// -----
+
+func.func @group_non_uniform_ballot_find_lsb(%value : vector<4xi32>) -> i32 {
+  // expected-error @+1 {{execution scope must be 'Workgroup' or 'Subgroup'}}
+  %0 = spirv.GroupNonUniformBallotFindLSB <Device> %value : vector<4xi32>, i32
+  return %0: i32
+}
+
+// -----
+
+func.func @group_non_uniform_ballot_find_lsb(%value : vector<4xi32>) -> si32 {
+  // expected-error @+1 {{op result #0 must be 8/16/32/64-bit signless/unsigned integer, but got 'si32'}}
+  %0 = spirv.GroupNonUniformBallotFindLSB <Subgroup> %value : vector<4xi32>, si32
+  return %0: si32
+}
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// spirv.GroupNonUniformBallotFindLSB
+//===----------------------------------------------------------------------===//
+
+func.func @group_non_uniform_ballot_find_msb(%value : vector<4xi32>) -> i32 {
+  // CHECK: %{{.*}} = spirv.GroupNonUniformBallotFindMSB <Subgroup> %{{.*}}: vector<4xi32>, i32
+  %0 = spirv.GroupNonUniformBallotFindMSB <Subgroup> %value : vector<4xi32>, i32
+  return %0: i32
+}
+
+// -----
+
+func.func @group_non_uniform_ballot_find_msb(%value : vector<4xi32>) -> i32 {
+  // expected-error @+1 {{execution scope must be 'Workgroup' or 'Subgroup'}}
+  %0 = spirv.GroupNonUniformBallotFindMSB <Device> %value : vector<4xi32>, i32
+  return %0: i32
+}
+
+// -----
+
+func.func @group_non_uniform_ballot_find_msb(%value : vector<4xi32>) -> si32 {
+  // expected-error @+1 {{op result #0 must be 8/16/32/64-bit signless/unsigned integer, but got 'si32'}}
+  %0 = spirv.GroupNonUniformBallotFindMSB <Subgroup> %value : vector<4xi32>, si32
+  return %0: si32
+}
+
+// -----
+
 //===----------------------------------------------------------------------===//
 // spirv.NonUniformGroupBroadcast
 //===----------------------------------------------------------------------===//

From 51785856ec5fb6c3f17834b212086d9fc7183868 Mon Sep 17 00:00:00 2001
From: Vitaly Buka <vitalybuka@google.com>
Date: Mon, 19 Aug 2024 11:24:42 -0700
Subject: [PATCH 400/441] [NFC][Instrumentation] Use `Twine` in
 `createPrivateGlobalForString` (#104726)

---
 llvm/include/llvm/Transforms/Instrumentation.h          | 2 +-
 llvm/lib/Transforms/Instrumentation/Instrumentation.cpp | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/llvm/include/llvm/Transforms/Instrumentation.h b/llvm/include/llvm/Transforms/Instrumentation.h
index 885d3249fb4f833..1a4824a806dc6e0 100644
--- a/llvm/include/llvm/Transforms/Instrumentation.h
+++ b/llvm/include/llvm/Transforms/Instrumentation.h
@@ -46,7 +46,7 @@ BasicBlock::iterator PrepareToSplitEntryBlock(BasicBlock &BB,
 // Create a constant for Str so that we can pass it to the run-time lib.
 GlobalVariable *createPrivateGlobalForString(Module &M, StringRef Str,
                                              bool AllowMerging,
-                                             const char *NamePrefix = "");
+                                             Twine NamePrefix = "");
 
 // Returns F.getComdat() if it exists.
 // Otherwise creates a new comdat, sets F's comdat, and returns it.
diff --git a/llvm/lib/Transforms/Instrumentation/Instrumentation.cpp b/llvm/lib/Transforms/Instrumentation/Instrumentation.cpp
index 19f4d8064fc27d2..305bae0e2010d10 100644
--- a/llvm/lib/Transforms/Instrumentation/Instrumentation.cpp
+++ b/llvm/lib/Transforms/Instrumentation/Instrumentation.cpp
@@ -91,7 +91,7 @@ BasicBlock::iterator llvm::PrepareToSplitEntryBlock(BasicBlock &BB,
 // Create a constant for Str so that we can pass it to the run-time lib.
 GlobalVariable *llvm::createPrivateGlobalForString(Module &M, StringRef Str,
                                                    bool AllowMerging,
-                                                   const char *NamePrefix) {
+                                                   Twine NamePrefix) {
   Constant *StrConst = ConstantDataArray::getString(M.getContext(), Str);
   // We use private linkage for module-local strings. If they can be merged
   // with another one, we set the unnamed_addr attribute.

From f76b9b7e9480d1bb116075593495db287f4c668c Mon Sep 17 00:00:00 2001
From: Vitaly Buka <vitalybuka@google.com>
Date: Mon, 19 Aug 2024 11:25:17 -0700
Subject: [PATCH 401/441] [NFC][asan] Don't `cd` after `split-file` (#104727)

Helps with copy-pasting of command line
with error.
---
 .../AddressSanitizer/global_metadata_array.ll        | 12 ++++++------
 .../AddressSanitizer/global_with_comdat.ll           |  8 ++++----
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/llvm/test/Instrumentation/AddressSanitizer/global_metadata_array.ll b/llvm/test/Instrumentation/AddressSanitizer/global_metadata_array.ll
index 1617bf9b67aa7c5..17b14164c3dc07a 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/global_metadata_array.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/global_metadata_array.ll
@@ -1,10 +1,10 @@
-; RUN: rm -rf %t && split-file %s %t && cd %t
-; RUN: opt < a.ll -passes=asan -asan-globals-live-support=0 -mtriple=x86_64-unknown-linux-gnu -S | FileCheck --check-prefix=CHECK %s
-; RUN: opt < a.ll -passes=asan -asan-globals-live-support=0 -mtriple=x86_64-apple-macosx10.11.0 -S | FileCheck --check-prefix=CHECK %s
-; RUN: opt < a.ll -passes=asan -asan-globals-live-support=0 -mtriple=x86_64-pc-windows-msvc19.0.24215 -S | FileCheck --check-prefix=CHECK %s
-; RUN: opt < a.ll -passes=asan -asan-globals-live-support=0 -asan-mapping-scale=5 -mtriple=x86_64-unknown-linux-gnu -S | FileCheck --check-prefixes=CHECK,CHECK-S5 %s
+; RUN: rm -rf %t && split-file %s %t
+; RUN: opt < %t/a.ll -passes=asan -asan-globals-live-support=0 -mtriple=x86_64-unknown-linux-gnu -S | FileCheck --check-prefix=CHECK %s
+; RUN: opt < %t/a.ll -passes=asan -asan-globals-live-support=0 -mtriple=x86_64-apple-macosx10.11.0 -S | FileCheck --check-prefix=CHECK %s
+; RUN: opt < %t/a.ll -passes=asan -asan-globals-live-support=0 -mtriple=x86_64-pc-windows-msvc19.0.24215 -S | FileCheck --check-prefix=CHECK %s
+; RUN: opt < %t/a.ll -passes=asan -asan-globals-live-support=0 -asan-mapping-scale=5 -mtriple=x86_64-unknown-linux-gnu -S | FileCheck --check-prefixes=CHECK,CHECK-S5 %s
 
-; RUN: opt < empty.ll -passes=asan -asan-globals-live-support=0 -mtriple=x86_64-unknown-linux-gnu -S | FileCheck --check-prefix=ELF-NOGC %s
+; RUN: opt < %t/empty.ll -passes=asan -asan-globals-live-support=0 -mtriple=x86_64-unknown-linux-gnu -S | FileCheck --check-prefix=ELF-NOGC %s
 
 ;--- a.ll
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Instrumentation/AddressSanitizer/global_with_comdat.ll b/llvm/test/Instrumentation/AddressSanitizer/global_with_comdat.ll
index f04eeb4b7aa4031..e61c0d8ce1f1b92 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/global_with_comdat.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/global_with_comdat.ll
@@ -4,12 +4,12 @@
 ; We keep using comdats for garbage collection if odr indicators are
 ; enabled as indicator symbols will cause link time odr violations.
 ; This is to fix PR 47925.
-; RUN: rm -rf %t && split-file %s %t && cd %t
-; RUN: opt < a.ll -passes=asan -asan-globals-live-support=1 -asan-use-odr-indicator=0 -S | FileCheck %s --check-prefixes=CHECK,NOCOMDAT
+; RUN: rm -rf %t && split-file %s %t
+; RUN: opt < %t/a.ll -passes=asan -asan-globals-live-support=1 -asan-use-odr-indicator=0 -S | FileCheck %s --check-prefixes=CHECK,NOCOMDAT
 ; Check that enabling odr indicators enables comdat for globals.
-; RUN: opt < a.ll -passes=asan -asan-globals-live-support=1 -S | FileCheck %s --check-prefixes=CHECK,COMDAT
+; RUN: opt < %t/a.ll -passes=asan -asan-globals-live-support=1 -S | FileCheck %s --check-prefixes=CHECK,COMDAT
 
-; RUN: opt < no_module_id.ll -passes=asan -S | FileCheck %s --check-prefix=NOMODULEID
+; RUN: opt < %t/no_module_id.ll -passes=asan -S | FileCheck %s --check-prefix=NOMODULEID
 
 ;--- a.ll
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"

From 5e8f4618ce9502190fac96eb8a856146acce2a3a Mon Sep 17 00:00:00 2001
From: Eduard Zingerman <eddyz87@gmail.com>
Date: Mon, 19 Aug 2024 11:30:27 -0700
Subject: [PATCH 402/441] Revert "[BPF] introduce
 `__attribute__((bpf_fastcall))` (#101228)"

This reverts commit e9b2e16dc98345bb1b91b1a6dacb3cec85f49e31.
Reverting because of the test failure:
https://lab.llvm.org/buildbot/#/builders/187/builds/509
---
 clang/include/clang/Basic/Attr.td             |   9 --
 clang/include/clang/Basic/AttrDocs.td         |  19 ---
 clang/lib/CodeGen/CGCall.cpp                  |   2 -
 clang/test/CodeGen/bpf-attr-bpf-fastcall-1.c  |  24 ----
 ...a-attribute-supported-attributes-list.test |   1 -
 clang/test/Sema/bpf-attr-bpf-fastcall.c       |  14 ---
 llvm/lib/Target/BPF/BPFCallingConv.td         |   1 -
 llvm/lib/Target/BPF/BPFISelLowering.cpp       |  31 -----
 llvm/lib/Target/BPF/BPFInstrInfo.td           |   4 +-
 llvm/lib/Target/BPF/BPFMIPeephole.cpp         |  86 --------------
 llvm/lib/Target/BPF/BPFRegisterInfo.cpp       |  11 --
 llvm/lib/Target/BPF/BPFRegisterInfo.h         |   3 -
 llvm/test/CodeGen/BPF/bpf-fastcall-1.ll       |  46 --------
 llvm/test/CodeGen/BPF/bpf-fastcall-2.ll       |  68 -----------
 llvm/test/CodeGen/BPF/bpf-fastcall-3.ll       |  62 ----------
 .../CodeGen/BPF/bpf-fastcall-regmask-1.ll     | 110 ------------------
 16 files changed, 3 insertions(+), 488 deletions(-)
 delete mode 100644 clang/test/CodeGen/bpf-attr-bpf-fastcall-1.c
 delete mode 100644 clang/test/Sema/bpf-attr-bpf-fastcall.c
 delete mode 100644 llvm/test/CodeGen/BPF/bpf-fastcall-1.ll
 delete mode 100644 llvm/test/CodeGen/BPF/bpf-fastcall-2.ll
 delete mode 100644 llvm/test/CodeGen/BPF/bpf-fastcall-3.ll
 delete mode 100644 llvm/test/CodeGen/BPF/bpf-fastcall-regmask-1.ll

diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td
index 98bedfe20f5d980..10a9d9e899e007f 100644
--- a/clang/include/clang/Basic/Attr.td
+++ b/clang/include/clang/Basic/Attr.td
@@ -2200,15 +2200,6 @@ def BTFTypeTag : TypeAttr {
   let LangOpts = [COnly];
 }
 
-def BPFFastCall : InheritableAttr,
-                  TargetSpecificAttr<TargetBPF> {
-  let Spellings = [Clang<"bpf_fastcall">];
-  let Subjects = SubjectList<[FunctionLike]>;
-  let Documentation = [BPFFastCallDocs];
-  let LangOpts = [COnly];
-  let SimpleHandler = 1;
-}
-
 def WebAssemblyExportName : InheritableAttr,
                             TargetSpecificAttr<TargetWebAssembly> {
   let Spellings = [Clang<"export_name">];
diff --git a/clang/include/clang/Basic/AttrDocs.td b/clang/include/clang/Basic/AttrDocs.td
index df36a2163b9f0b3..19cbb9a0111a28a 100644
--- a/clang/include/clang/Basic/AttrDocs.td
+++ b/clang/include/clang/Basic/AttrDocs.td
@@ -2345,25 +2345,6 @@ section.
   }];
 }
 
-def BPFFastCallDocs : Documentation {
-  let Category = DocCatType;
-  let Content = [{
-Functions annotated with this attribute are likely to be inlined by BPF JIT.
-It is assumed that inlined implementation uses less caller saved registers,
-than a regular function.
-Specifically, the following registers are likely to be preserved:
-- ``R0`` if function return value is ``void``;
-- ``R2-R5` if function takes 1 argument;
-- ``R3-R5` if function takes 2 arguments;
-- ``R4-R5` if function takes 3 arguments;
-- ``R5`` if function takes 4 arguments;
-
-For such functions Clang generates code pattern that allows BPF JIT
-to recognize and remove unnecessary spills and fills of the preserved
-registers.
-  }];
-}
-
 def MipsInterruptDocs : Documentation {
   let Category = DocCatFunction;
   let Heading = "interrupt (MIPS)";
diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp
index 09803712569755d..e4f221ae55eefa1 100644
--- a/clang/lib/CodeGen/CGCall.cpp
+++ b/clang/lib/CodeGen/CGCall.cpp
@@ -2422,8 +2422,6 @@ void CodeGenModule::ConstructAttributeList(StringRef Name,
       FuncAttrs.addAttribute(llvm::Attribute::NoCfCheck);
     if (TargetDecl->hasAttr<LeafAttr>())
       FuncAttrs.addAttribute(llvm::Attribute::NoCallback);
-    if (TargetDecl->hasAttr<BPFFastCallAttr>())
-      FuncAttrs.addAttribute("bpf_fastcall");
 
     HasOptnone = TargetDecl->hasAttr<OptimizeNoneAttr>();
     if (auto *AllocSize = TargetDecl->getAttr<AllocSizeAttr>()) {
diff --git a/clang/test/CodeGen/bpf-attr-bpf-fastcall-1.c b/clang/test/CodeGen/bpf-attr-bpf-fastcall-1.c
deleted file mode 100644
index fa740d8e44ff519..000000000000000
--- a/clang/test/CodeGen/bpf-attr-bpf-fastcall-1.c
+++ /dev/null
@@ -1,24 +0,0 @@
-// REQUIRES: bpf-registered-target
-// RUN: %clang_cc1 -triple bpf -emit-llvm -disable-llvm-passes %s -o - | FileCheck %s
-
-#define __bpf_fastcall __attribute__((bpf_fastcall))
-
-void test(void) __bpf_fastcall;
-void (*ptr)(void) __bpf_fastcall;
-
-void foo(void) {
-  test();
-  (*ptr)();
-}
-
-// CHECK: @ptr = global ptr null
-// CHECK: define {{.*}} void @foo()
-// CHECK: entry:
-// CHECK:   call void @test() #[[call_attr:[0-9]+]]
-// CHECK:   %[[ptr:.*]] = load ptr, ptr @ptr, align 8
-// CHECK:   call void %[[ptr]]() #[[call_attr]]
-// CHECK:   ret void
-
-// CHECK: declare void @test() #[[func_attr:[0-9]+]]
-// CHECK: attributes #[[func_attr]] = { {{.*}}"bpf_fastcall"{{.*}} }
-// CHECK: attributes #[[call_attr]] = { "bpf_fastcall" }
diff --git a/clang/test/Misc/pragma-attribute-supported-attributes-list.test b/clang/test/Misc/pragma-attribute-supported-attributes-list.test
index a7e425e3d5f4313..1a71556213bb16d 100644
--- a/clang/test/Misc/pragma-attribute-supported-attributes-list.test
+++ b/clang/test/Misc/pragma-attribute-supported-attributes-list.test
@@ -22,7 +22,6 @@
 // CHECK-NEXT: AssumeAligned (SubjectMatchRule_objc_method, SubjectMatchRule_function)
 // CHECK-NEXT: Availability ((SubjectMatchRule_record, SubjectMatchRule_enum, SubjectMatchRule_enum_constant, SubjectMatchRule_field, SubjectMatchRule_function, SubjectMatchRule_namespace, SubjectMatchRule_objc_category, SubjectMatchRule_objc_implementation, SubjectMatchRule_objc_interface, SubjectMatchRule_objc_method, SubjectMatchRule_objc_property, SubjectMatchRule_objc_protocol, SubjectMatchRule_record, SubjectMatchRule_type_alias, SubjectMatchRule_variable))
 // CHECK-NEXT: AvailableOnlyInDefaultEvalMethod (SubjectMatchRule_type_alias)
-// CHECK-NEXT: BPFFastCall (SubjectMatchRule_hasType_functionType)
 // CHECK-NEXT: BPFPreserveAccessIndex (SubjectMatchRule_record)
 // CHECK-NEXT: BPFPreserveStaticOffset (SubjectMatchRule_record)
 // CHECK-NEXT: BTFDeclTag (SubjectMatchRule_variable, SubjectMatchRule_function, SubjectMatchRule_record, SubjectMatchRule_field, SubjectMatchRule_type_alias)
diff --git a/clang/test/Sema/bpf-attr-bpf-fastcall.c b/clang/test/Sema/bpf-attr-bpf-fastcall.c
deleted file mode 100644
index 178b1f50741e877..000000000000000
--- a/clang/test/Sema/bpf-attr-bpf-fastcall.c
+++ /dev/null
@@ -1,14 +0,0 @@
-// REQUIRES: bpf-registered-target
-// RUN: %clang_cc1 %s -triple bpf -verify
-
-__attribute__((bpf_fastcall)) int var; // expected-warning {{'bpf_fastcall' attribute only applies to functions and function pointers}}
-
-__attribute__((bpf_fastcall)) void func();
-__attribute__((bpf_fastcall(1))) void func_invalid(); // expected-error {{'bpf_fastcall' attribute takes no arguments}}
-
-void (*ptr1)(void) __attribute__((bpf_fastcall));
-void (*ptr2)(void);
-void foo(void) {
-  ptr2 = ptr1; // not an error
-  ptr1 = ptr2; // not an error
-}
diff --git a/llvm/lib/Target/BPF/BPFCallingConv.td b/llvm/lib/Target/BPF/BPFCallingConv.td
index a557211437e95f9..ef4ef1930aa8fb4 100644
--- a/llvm/lib/Target/BPF/BPFCallingConv.td
+++ b/llvm/lib/Target/BPF/BPFCallingConv.td
@@ -46,4 +46,3 @@ def CC_BPF32 : CallingConv<[
 ]>;
 
 def CSR : CalleeSavedRegs<(add R6, R7, R8, R9, R10)>;
-def CSR_PreserveAll : CalleeSavedRegs<(add R0, R1, R2, R3, R4, R5, R6, R7, R8, R9, R10)>;
diff --git a/llvm/lib/Target/BPF/BPFISelLowering.cpp b/llvm/lib/Target/BPF/BPFISelLowering.cpp
index ff23d3b055d0d5c..071fe004806e3e5 100644
--- a/llvm/lib/Target/BPF/BPFISelLowering.cpp
+++ b/llvm/lib/Target/BPF/BPFISelLowering.cpp
@@ -402,21 +402,6 @@ SDValue BPFTargetLowering::LowerFormalArguments(
 
 const size_t BPFTargetLowering::MaxArgs = 5;
 
-static void resetRegMaskBit(const TargetRegisterInfo *TRI, uint32_t *RegMask,
-                            MCRegister Reg) {
-  for (MCPhysReg SubReg : TRI->subregs_inclusive(Reg))
-    RegMask[SubReg / 32] &= ~(1u << (SubReg % 32));
-}
-
-static uint32_t *regMaskFromTemplate(const TargetRegisterInfo *TRI,
-                                     MachineFunction &MF,
-                                     const uint32_t *BaseRegMask) {
-  uint32_t *RegMask = MF.allocateRegMask();
-  unsigned RegMaskSize = MachineOperand::getRegMaskSize(TRI->getNumRegs());
-  memcpy(RegMask, BaseRegMask, sizeof(RegMask[0]) * RegMaskSize);
-  return RegMask;
-}
-
 SDValue BPFTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
                                      SmallVectorImpl<SDValue> &InVals) const {
   SelectionDAG &DAG = CLI.DAG;
@@ -528,22 +513,6 @@ SDValue BPFTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
   for (auto &Reg : RegsToPass)
     Ops.push_back(DAG.getRegister(Reg.first, Reg.second.getValueType()));
 
-  bool HasFastCall =
-      (CLI.CB && isa<CallInst>(CLI.CB) && CLI.CB->hasFnAttr("bpf_fastcall"));
-  const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
-  if (HasFastCall) {
-    uint32_t *RegMask = regMaskFromTemplate(
-        TRI, MF, TRI->getCallPreservedMask(MF, CallingConv::PreserveAll));
-    for (auto const &RegPair : RegsToPass)
-      resetRegMaskBit(TRI, RegMask, RegPair.first);
-    if (!CLI.CB->getType()->isVoidTy())
-      resetRegMaskBit(TRI, RegMask, BPF::R0);
-    Ops.push_back(DAG.getRegisterMask(RegMask));
-  } else {
-    Ops.push_back(
-        DAG.getRegisterMask(TRI->getCallPreservedMask(MF, CLI.CallConv)));
-  }
-
   if (InGlue.getNode())
     Ops.push_back(InGlue);
 
diff --git a/llvm/lib/Target/BPF/BPFInstrInfo.td b/llvm/lib/Target/BPF/BPFInstrInfo.td
index 4baeeb017699d69..2ee630e29790f34 100644
--- a/llvm/lib/Target/BPF/BPFInstrInfo.td
+++ b/llvm/lib/Target/BPF/BPFInstrInfo.td
@@ -677,7 +677,9 @@ let isBranch = 1, isTerminator = 1, hasDelaySlot=0, isBarrier = 1 in {
 }
 
 // Jump and link
-let isCall=1, hasDelaySlot=0, Uses = [R11] in {
+let isCall=1, hasDelaySlot=0, Uses = [R11],
+    // Potentially clobbered registers
+    Defs = [R0, R1, R2, R3, R4, R5] in {
   def JAL  : CALL<"call">;
   def JALX  : CALLX<"callx">;
 }
diff --git a/llvm/lib/Target/BPF/BPFMIPeephole.cpp b/llvm/lib/Target/BPF/BPFMIPeephole.cpp
index 58dad71b9205526..f0edf706bd8fd7a 100644
--- a/llvm/lib/Target/BPF/BPFMIPeephole.cpp
+++ b/llvm/lib/Target/BPF/BPFMIPeephole.cpp
@@ -24,8 +24,6 @@
 #include "BPFInstrInfo.h"
 #include "BPFTargetMachine.h"
 #include "llvm/ADT/Statistic.h"
-#include "llvm/CodeGen/LivePhysRegs.h"
-#include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
@@ -321,7 +319,6 @@ struct BPFMIPreEmitPeephole : public MachineFunctionPass {
   bool in16BitRange(int Num);
   bool eliminateRedundantMov();
   bool adjustBranch();
-  bool insertMissingCallerSavedSpills();
 
 public:
 
@@ -336,7 +333,6 @@ struct BPFMIPreEmitPeephole : public MachineFunctionPass {
     Changed = eliminateRedundantMov();
     if (SupportGotol)
       Changed = adjustBranch() || Changed;
-    Changed |= insertMissingCallerSavedSpills();
     return Changed;
   }
 };
@@ -600,88 +596,6 @@ bool BPFMIPreEmitPeephole::adjustBranch() {
   return Changed;
 }
 
-static const unsigned CallerSavedRegs[] = {BPF::R0, BPF::R1, BPF::R2,
-                                           BPF::R3, BPF::R4, BPF::R5};
-
-struct BPFFastCall {
-  MachineInstr *MI;
-  unsigned LiveCallerSavedRegs;
-};
-
-static void collectBPFFastCalls(const TargetRegisterInfo *TRI,
-                                LivePhysRegs &LiveRegs, MachineBasicBlock &BB,
-                                SmallVectorImpl<BPFFastCall> &Calls) {
-  LiveRegs.init(*TRI);
-  LiveRegs.addLiveOuts(BB);
-  Calls.clear();
-  for (MachineInstr &MI : llvm::reverse(BB)) {
-    if (MI.isCall()) {
-      unsigned LiveCallerSavedRegs = 0;
-      for (MCRegister R : CallerSavedRegs) {
-        bool DoSpillFill = !MI.definesRegister(R, TRI) && LiveRegs.contains(R);
-        if (!DoSpillFill)
-          continue;
-        LiveCallerSavedRegs |= 1 << R;
-      }
-      if (LiveCallerSavedRegs)
-        Calls.push_back({&MI, LiveCallerSavedRegs});
-    }
-    LiveRegs.stepBackward(MI);
-  }
-}
-
-static int64_t computeMinFixedObjOffset(MachineFrameInfo &MFI,
-                                        unsigned SlotSize) {
-  int64_t MinFixedObjOffset = 0;
-  // Same logic as in X86FrameLowering::adjustFrameForMsvcCxxEh()
-  for (int I = MFI.getObjectIndexBegin(); I < MFI.getObjectIndexEnd(); ++I) {
-    if (MFI.isDeadObjectIndex(I))
-      continue;
-    MinFixedObjOffset = std::min(MinFixedObjOffset, MFI.getObjectOffset(I));
-  }
-  MinFixedObjOffset -=
-      (SlotSize + MinFixedObjOffset % SlotSize) & (SlotSize - 1);
-  return MinFixedObjOffset;
-}
-
-bool BPFMIPreEmitPeephole::insertMissingCallerSavedSpills() {
-  MachineFrameInfo &MFI = MF->getFrameInfo();
-  SmallVector<BPFFastCall, 8> Calls;
-  LivePhysRegs LiveRegs;
-  const unsigned SlotSize = 8;
-  int64_t MinFixedObjOffset = computeMinFixedObjOffset(MFI, SlotSize);
-  bool Changed = false;
-  for (MachineBasicBlock &BB : *MF) {
-    collectBPFFastCalls(TRI, LiveRegs, BB, Calls);
-    Changed |= !Calls.empty();
-    for (BPFFastCall &Call : Calls) {
-      int64_t CurOffset = MinFixedObjOffset;
-      for (MCRegister Reg : CallerSavedRegs) {
-        if (((1 << Reg) & Call.LiveCallerSavedRegs) == 0)
-          continue;
-        // Allocate stack object
-        CurOffset -= SlotSize;
-        MFI.CreateFixedSpillStackObject(SlotSize, CurOffset);
-        // Generate spill
-        BuildMI(BB, Call.MI->getIterator(), Call.MI->getDebugLoc(),
-                TII->get(BPF::STD))
-            .addReg(Reg)
-            .addReg(BPF::R10)
-            .addImm(CurOffset)
-            .addImm(0);
-        // Generate fill
-        BuildMI(BB, ++Call.MI->getIterator(), Call.MI->getDebugLoc(),
-                TII->get(BPF::LDD))
-            .addReg(Reg)
-            .addReg(BPF::R10)
-            .addImm(CurOffset)
-            .addImm(0);
-      }
-    }
-  }
-  return Changed;
-}
-
 } // end default namespace
 
 INITIALIZE_PASS(BPFMIPreEmitPeephole, "bpf-mi-pemit-peephole",
diff --git a/llvm/lib/Target/BPF/BPFRegisterInfo.cpp b/llvm/lib/Target/BPF/BPFRegisterInfo.cpp
index 69e1318954a973a..84af6806abb36c0 100644
--- a/llvm/lib/Target/BPF/BPFRegisterInfo.cpp
+++ b/llvm/lib/Target/BPF/BPFRegisterInfo.cpp
@@ -40,17 +40,6 @@ BPFRegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const {
   return CSR_SaveList;
 }
 
-const uint32_t *
-BPFRegisterInfo::getCallPreservedMask(const MachineFunction &MF,
-                                      CallingConv::ID CC) const {
-  switch (CC) {
-  default:
-    return CSR_RegMask;
-  case CallingConv::PreserveAll:
-    return CSR_PreserveAll_RegMask;
-  }
-}
-
 BitVector BPFRegisterInfo::getReservedRegs(const MachineFunction &MF) const {
   BitVector Reserved(getNumRegs());
   markSuperRegs(Reserved, BPF::W10); // [W|R]10 is read only frame pointer
diff --git a/llvm/lib/Target/BPF/BPFRegisterInfo.h b/llvm/lib/Target/BPF/BPFRegisterInfo.h
index db868769a1579a5..f7dea75ebea6f9e 100644
--- a/llvm/lib/Target/BPF/BPFRegisterInfo.h
+++ b/llvm/lib/Target/BPF/BPFRegisterInfo.h
@@ -26,9 +26,6 @@ struct BPFRegisterInfo : public BPFGenRegisterInfo {
 
   const MCPhysReg *getCalleeSavedRegs(const MachineFunction *MF) const override;
 
-  const uint32_t *getCallPreservedMask(const MachineFunction &MF,
-                                       CallingConv::ID) const override;
-
   BitVector getReservedRegs(const MachineFunction &MF) const override;
 
   bool eliminateFrameIndex(MachineBasicBlock::iterator MI, int SPAdj,
diff --git a/llvm/test/CodeGen/BPF/bpf-fastcall-1.ll b/llvm/test/CodeGen/BPF/bpf-fastcall-1.ll
deleted file mode 100644
index fd81314a495ef87..000000000000000
--- a/llvm/test/CodeGen/BPF/bpf-fastcall-1.ll
+++ /dev/null
@@ -1,46 +0,0 @@
-; RUN: llc -O2 --march=bpfel %s -o - | FileCheck %s
-
-; Generated from the following C code:
-;
-;   #define __bpf_fastcall __attribute__((bpf_fastcall))
-;
-;   void bar(void) __bpf_fastcall;
-;   void buz(long i, long j, long k);
-;
-;   void foo(long i, long j, long k) {
-;     bar();
-;     buz(i, j, k);
-;   }
-;
-; Using the following command:
-;
-;   clang --target=bpf -emit-llvm -O2 -S -o - t.c
-;
-; (unnecessary attrs removed maually)
-
-; Check that function marked with bpf_fastcall does not clobber R1-R5.
-
-define dso_local void @foo(i64 noundef %i, i64 noundef %j, i64 noundef %k) {
-entry:
-  tail call void @bar() #1
-  tail call void @buz(i64 noundef %i, i64 noundef %j, i64 noundef %k)
-  ret void
-}
-
-; CHECK:      foo:
-; CHECK:      # %bb.0:
-; CHECK-NEXT:   *(u64 *)(r10 - 8) = r1
-; CHECK-NEXT:   *(u64 *)(r10 - 16) = r2
-; CHECK-NEXT:   *(u64 *)(r10 - 24) = r3
-; CHECK-NEXT:   call bar
-; CHECK-NEXT:   r3 = *(u64 *)(r10 - 24)
-; CHECK-NEXT:   r2 = *(u64 *)(r10 - 16)
-; CHECK-NEXT:   r1 = *(u64 *)(r10 - 8)
-; CHECK-NEXT:   call buz
-; CHECK-NEXT:   exit
-
-declare dso_local void @bar() #0
-declare dso_local void @buz(i64 noundef, i64 noundef, i64 noundef)
-
-attributes #0 = { "bpf_fastcall" }
-attributes #1 = { nounwind "bpf_fastcall" }
diff --git a/llvm/test/CodeGen/BPF/bpf-fastcall-2.ll b/llvm/test/CodeGen/BPF/bpf-fastcall-2.ll
deleted file mode 100644
index e3e29cdddca8ea6..000000000000000
--- a/llvm/test/CodeGen/BPF/bpf-fastcall-2.ll
+++ /dev/null
@@ -1,68 +0,0 @@
-; RUN: llc -O2 --march=bpfel %s -o - | FileCheck %s
-
-; Generated from the following C code:
-;
-;   #define __bpf_fastcall __attribute__((bpf_fastcall))
-;
-;   void bar(void) __bpf_fastcall;
-;   void buz(long i, long j);
-;
-;   void foo(long i, long j, long k, long l) {
-;     bar();
-;     if (k > 42l)
-;       buz(i, 1);
-;     else
-;       buz(1, j);
-;   }
-;
-; Using the following command:
-;
-;   clang --target=bpf -emit-llvm -O2 -S -o - t.c
-;
-; (unnecessary attrs removed maually)
-
-; Check that function marked with bpf_fastcall does not clobber R1-R5.
-; Use R1 in one branch following call and R2 in another branch following call.
-
-define dso_local void @foo(i64 noundef %i, i64 noundef %j, i64 noundef %k, i64 noundef %l) {
-entry:
-  tail call void @bar() #0
-  %cmp = icmp sgt i64 %k, 42
-  br i1 %cmp, label %if.then, label %if.else
-
-if.then:
-  tail call void @buz(i64 noundef %i, i64 noundef 1)
-  br label %if.end
-
-if.else:
-  tail call void @buz(i64 noundef 1, i64 noundef %j)
-  br label %if.end
-
-if.end:
-  ret void
-}
-
-; CHECK:      foo:                                    # @foo
-; CHECK:      # %bb.0:                                # %entry
-; CHECK-NEXT:   *(u64 *)(r10 - 8) = r1
-; CHECK-NEXT:   *(u64 *)(r10 - 16) = r2
-; CHECK-NEXT:   *(u64 *)(r10 - 24) = r3
-; CHECK-NEXT:   call bar
-; CHECK-NEXT:   r3 = *(u64 *)(r10 - 24)
-; CHECK-NEXT:   r2 = *(u64 *)(r10 - 16)
-; CHECK-NEXT:   r1 = *(u64 *)(r10 - 8)
-; CHECK-NEXT:   r4 = 43
-; CHECK-NEXT:   if r4 s> r3 goto [[ELSE:.*]]
-; CHECK-NEXT: # %bb.1:                                # %if.then
-; CHECK-NEXT:   r2 = 1
-; CHECK-NEXT:   goto [[END:.*]]
-; CHECK-NEXT: [[ELSE]]:                               # %if.else
-; CHECK-NEXT:   r1 = 1
-; CHECK-NEXT: [[END]]:                                # %if.end
-; CHECK-NEXT:   call buz
-; CHECK-NEXT:   exit
-
-declare dso_local void @bar() #0
-declare dso_local void @buz(i64 noundef, i64 noundef)
-
-attributes #0 = { "bpf_fastcall" }
diff --git a/llvm/test/CodeGen/BPF/bpf-fastcall-3.ll b/llvm/test/CodeGen/BPF/bpf-fastcall-3.ll
deleted file mode 100644
index 81ca4e1ac57bc7c..000000000000000
--- a/llvm/test/CodeGen/BPF/bpf-fastcall-3.ll
+++ /dev/null
@@ -1,62 +0,0 @@
-; RUN: llc -O2 --march=bpfel %s -o - | FileCheck %s
-
-; Generated from the following C code:
-;
-; #define __bpf_fastcall __attribute__((bpf_fastcall))
-;
-; void quux(void *);
-; void bar(long) __bpf_fastcall;
-; void buz(long i, long j);
-;
-; void foo(long i, long j) {
-;   long k;
-;   bar(i);
-;   bar(i);
-;   buz(i, j);
-;   quux(&k);
-; }
-;
-; Using the following command:
-;
-;   clang --target=bpf -emit-llvm -O2 -S -o - t.c
-;
-; (unnecessary attrs removed maually)
-
-; Check that function marked with bpf_fastcall does not clobber R1-R5.
-; Check that spills/fills wrapping the call use and reuse lowest stack offsets.
-
-define dso_local void @foo(i64 noundef %i, i64 noundef %j) {
-entry:
-  %k = alloca i64, align 8
-  tail call void @bar(i64 noundef %i) #0
-  tail call void @bar(i64 noundef %i) #0
-  tail call void @buz(i64 noundef %i, i64 noundef %j)
-  call void @quux(ptr noundef nonnull %k)
-  ret void
-}
-
-; CHECK:      # %bb.0:
-; CHECK-NEXT:   r3 = r1
-; CHECK-NEXT:   *(u64 *)(r10 - 16) = r2
-; CHECK-NEXT:   *(u64 *)(r10 - 24) = r3
-; CHECK-NEXT:   call bar
-; CHECK-NEXT:   r3 = *(u64 *)(r10 - 24)
-; CHECK-NEXT:   r2 = *(u64 *)(r10 - 16)
-; CHECK-NEXT:   r1 = r3
-; CHECK-NEXT:   *(u64 *)(r10 - 16) = r2
-; CHECK-NEXT:   *(u64 *)(r10 - 24) = r3
-; CHECK-NEXT:   call bar
-; CHECK-NEXT:   r3 = *(u64 *)(r10 - 24)
-; CHECK-NEXT:   r2 = *(u64 *)(r10 - 16)
-; CHECK-NEXT:   r1 = r3
-; CHECK-NEXT:   call buz
-; CHECK-NEXT:   r1 = r10
-; CHECK-NEXT:   r1 += -8
-; CHECK-NEXT:   call quux
-; CHECK-NEXT:   exit
-
-declare dso_local void @bar(i64 noundef) #0
-declare dso_local void @buz(i64 noundef, i64 noundef)
-declare dso_local void @quux(ptr noundef)
-
-attributes #0 = { "bpf_fastcall" }
diff --git a/llvm/test/CodeGen/BPF/bpf-fastcall-regmask-1.ll b/llvm/test/CodeGen/BPF/bpf-fastcall-regmask-1.ll
deleted file mode 100644
index 857d2f000d1d5a3..000000000000000
--- a/llvm/test/CodeGen/BPF/bpf-fastcall-regmask-1.ll
+++ /dev/null
@@ -1,110 +0,0 @@
-; RUN: llc -O2 --march=bpfel \
-; RUN:   -print-after=stack-slot-coloring %s \
-; RUN:   -o /dev/null 2>&1 | FileCheck %s
-
-; Generated from the following C code:
-;
-;   #define __bpf_fastcall __attribute__((bpf_fastcall))
-;
-;   void bar1(void) __bpf_fastcall;
-;   void buz1(long i, long j, long k);
-;   void foo1(long i, long j, long k) {
-;     bar1();
-;     buz1(i, j, k);
-;   }
-;
-;   long bar2(void) __bpf_fastcall;
-;   void buz2(long i, long j, long k);
-;   void foo2(long i, long j, long k) {
-;     bar2();
-;     buz2(i, j, k);
-;   }
-;
-;   void bar3(long) __bpf_fastcall;
-;   void buz3(long i, long j, long k);
-;   void foo3(long i, long j, long k) {
-;     bar3(i);
-;     buz3(i, j, k);
-;   }
-;
-;   long bar4(long, long) __bpf_fastcall;
-;   void buz4(long i, long j, long k);
-;   void foo4(long i, long j, long k) {
-;     bar4(i, j);
-;     buz4(i, j, k);
-;   }
-;
-; Using the following command:
-;
-;   clang --target=bpf -emit-llvm -O2 -S -o - t.c
-;
-; (unnecessary attrs removed maually)
-
-; Check regmask for calls to functions marked with bpf_fastcall:
-; - void function w/o parameters
-; - non-void function w/o parameters
-; - void function with parameters
-; - non-void function with parameters
-
-declare dso_local void @bar1() #0
-declare dso_local void @buz1(i64 noundef, i64 noundef, i64 noundef)
-define dso_local void @foo1(i64 noundef %i, i64 noundef %j, i64 noundef %k) {
-entry:
-  tail call void @bar1() #1
-  tail call void @buz1(i64 noundef %i, i64 noundef %j, i64 noundef %k)
-  ret void
-}
-
-; CHECK:      JAL @bar1, <regmask $r0 $r1 $r2 $r3 $r4 $r5 $r6 $r7 $r8 $r9 $r10
-; CHECK-SAME:                     $w0 $w1 $w2 $w3 $w4 $w5 $w6 $w7 $w8 $w9 $w10>
-; CHECK-SAME:          , implicit $r11, implicit-def $r11
-; CHECK:      JAL @buz1, <regmask $r6 $r7 $r8 $r9 $r10 $w6 $w7 $w8 $w9 $w10>
-; CHECK-SAME:          , implicit $r11, implicit $r1, implicit $r2, implicit $r3, implicit-def $r11
-
-declare dso_local i64 @bar2() #0
-declare dso_local void @buz2(i64 noundef, i64 noundef, i64 noundef)
-define dso_local void @foo2(i64 noundef %i, i64 noundef %j, i64 noundef %k) {
-entry:
-  tail call i64 @bar2() #1
-  tail call void @buz2(i64 noundef %i, i64 noundef %j, i64 noundef %k)
-  ret void
-}
-
-; CHECK:      JAL @bar2, <regmask $r1 $r2 $r3 $r4 $r5 $r6 $r7 $r8 $r9 $r10
-; CHECK-SAME:                     $w1 $w2 $w3 $w4 $w5 $w6 $w7 $w8 $w9 $w10>
-; CHECK-SAME:          , implicit $r11, implicit-def $r11, implicit-def dead $r0
-; CHECK:      JAL @buz2, <regmask $r6 $r7 $r8 $r9 $r10 $w6 $w7 $w8 $w9 $w10>
-; CHECK-SAME:          , implicit $r11, implicit $r1, implicit $r2, implicit $r3, implicit-def $r11
-
-declare dso_local void @bar3(i64) #0
-declare dso_local void @buz3(i64 noundef, i64 noundef, i64 noundef)
-define dso_local void @foo3(i64 noundef %i, i64 noundef %j, i64 noundef %k) {
-entry:
-  tail call void @bar3(i64 noundef %i) #1
-  tail call void @buz3(i64 noundef %i, i64 noundef %j, i64 noundef %k)
-  ret void
-}
-
-; CHECK:      JAL @bar3, <regmask $r0 $r2 $r3 $r4 $r5 $r6 $r7 $r8 $r9 $r10
-; CHECK-SAME:                     $w0 $w2 $w3 $w4 $w5 $w6 $w7 $w8 $w9 $w10>
-; CHECK-SAME:          , implicit $r11, implicit $r1, implicit-def $r11
-; CHECK:      JAL @buz3, <regmask $r6 $r7 $r8 $r9 $r10 $w6 $w7 $w8 $w9 $w10>
-; CHECK-SAME:          , implicit $r11, implicit $r1, implicit $r2, implicit $r3, implicit-def $r11
-
-declare dso_local i64 @bar4(i64 noundef, i64 noundef) #0
-declare dso_local void @buz4(i64 noundef, i64 noundef, i64 noundef)
-define dso_local void @foo4(i64 noundef %i, i64 noundef %j, i64 noundef %k) {
-entry:
-  tail call i64 @bar4(i64 noundef %i, i64 noundef %j) #1
-  tail call void @buz4(i64 noundef %i, i64 noundef %j, i64 noundef %k)
-  ret void
-}
-
-; CHECK:      JAL @bar4, <regmask $r3 $r4 $r5 $r6 $r7 $r8 $r9 $r10
-; CHECK-SAME:                     $w3 $w4 $w5 $w6 $w7 $w8 $w9 $w10>
-; CHECK-SAME:          , implicit $r11, implicit $r1, implicit $r2, implicit-def $r11, implicit-def dead $r0
-; CHECK:      JAL @buz4, <regmask $r6 $r7 $r8 $r9 $r10 $w6 $w7 $w8 $w9 $w10>
-; CHECK-SAME:          , implicit $r11, implicit $r1, implicit $r2, implicit $r3, implicit-def $r11
-
-attributes #0 = { "bpf_fastcall" }
-attributes #1 = { nounwind "bpf_fastcall" }

From 92a8ec7abbd853e89556c7b942f30054273af9c6 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <ramkumar.ramachandra@codasip.com>
Date: Mon, 19 Aug 2024 19:55:58 +0100
Subject: [PATCH 403/441] LSV: fix style after cursory reading (NFC) (#104793)

---
 .../Vectorize/LoadStoreVectorizer.cpp         | 72 +++++++++----------
 1 file changed, 36 insertions(+), 36 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
index c35ea431296b704..02ec1d5c259cd61 100644
--- a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
@@ -157,6 +157,8 @@ using EqClassKey =
 struct ChainElem {
   Instruction *Inst;
   APInt OffsetFromLeader;
+  ChainElem(Instruction *Inst, APInt OffsetFromLeader)
+      : Inst(std::move(Inst)), OffsetFromLeader(std::move(OffsetFromLeader)) {}
 };
 using Chain = SmallVector<ChainElem, 1>;
 
@@ -187,7 +189,7 @@ constexpr unsigned StackAdjustedAlignment = 4;
 Instruction *propagateMetadata(Instruction *I, const Chain &C) {
   SmallVector<Value *, 8> Values;
   for (const ChainElem &E : C)
-    Values.push_back(E.Inst);
+    Values.emplace_back(E.Inst);
   return propagateMetadata(I, Values);
 }
 
@@ -202,12 +204,12 @@ void reorder(Instruction *I) {
   SmallPtrSet<Instruction *, 16> InstructionsToMove;
   SmallVector<Instruction *, 16> Worklist;
 
-  Worklist.push_back(I);
+  Worklist.emplace_back(I);
   while (!Worklist.empty()) {
     Instruction *IW = Worklist.pop_back_val();
     int NumOperands = IW->getNumOperands();
-    for (int i = 0; i < NumOperands; i++) {
-      Instruction *IM = dyn_cast<Instruction>(IW->getOperand(i));
+    for (int Idx = 0; Idx < NumOperands; Idx++) {
+      Instruction *IM = dyn_cast<Instruction>(IW->getOperand(Idx));
       if (!IM || IM->getOpcode() == Instruction::PHI)
         continue;
 
@@ -220,7 +222,7 @@ void reorder(Instruction *I) {
 
       if (!IM->comesBefore(I)) {
         InstructionsToMove.insert(IM);
-        Worklist.push_back(IM);
+        Worklist.emplace_back(IM);
       }
     }
   }
@@ -228,7 +230,7 @@ void reorder(Instruction *I) {
   // All instructions to move should follow I. Start from I, not from begin().
   for (auto BBI = I->getIterator(), E = I->getParent()->end(); BBI != E;) {
     Instruction *IM = &*(BBI++);
-    if (!InstructionsToMove.count(IM))
+    if (!InstructionsToMove.contains(IM))
       continue;
     IM->moveBefore(I);
   }
@@ -438,11 +440,11 @@ bool Vectorizer::run() {
     assert(!BB->empty());
 
     SmallVector<BasicBlock::iterator, 8> Barriers;
-    Barriers.push_back(BB->begin());
+    Barriers.emplace_back(BB->begin());
     for (Instruction &I : *BB)
       if (!isGuaranteedToTransferExecutionToSuccessor(&I))
-        Barriers.push_back(I.getIterator());
-    Barriers.push_back(BB->end());
+        Barriers.emplace_back(I.getIterator());
+    Barriers.emplace_back(BB->end());
 
     for (auto It = Barriers.begin(), End = std::prev(Barriers.end()); It != End;
          ++It)
@@ -559,14 +561,14 @@ std::vector<Chain> Vectorizer::splitChainByMayAliasInstrs(Chain &C) {
 
     std::vector<Chain> Chains;
     SmallVector<ChainElem, 1> NewChain;
-    NewChain.push_back(*ChainBegin);
+    NewChain.emplace_back(*ChainBegin);
     for (auto ChainIt = std::next(ChainBegin); ChainIt != ChainEnd; ++ChainIt) {
       if (isSafeToMove<IsLoad>(ChainIt->Inst, NewChain.front().Inst,
                                ChainOffsets)) {
         LLVM_DEBUG(dbgs() << "LSV: No intervening may-alias instrs; can merge "
                           << *ChainIt->Inst << " into " << *ChainBegin->Inst
                           << "\n");
-        NewChain.push_back(*ChainIt);
+        NewChain.emplace_back(*ChainIt);
       } else {
         LLVM_DEBUG(
             dbgs() << "LSV: Found intervening may-alias instrs; cannot merge "
@@ -576,7 +578,7 @@ std::vector<Chain> Vectorizer::splitChainByMayAliasInstrs(Chain &C) {
             dbgs() << "LSV: got nontrivial chain without aliasing instrs:\n";
             dumpChain(NewChain);
           });
-          Chains.push_back(std::move(NewChain));
+          Chains.emplace_back(std::move(NewChain));
         }
 
         // Start a new chain.
@@ -588,7 +590,7 @@ std::vector<Chain> Vectorizer::splitChainByMayAliasInstrs(Chain &C) {
         dbgs() << "LSV: got nontrivial chain without aliasing instrs:\n";
         dumpChain(NewChain);
       });
-      Chains.push_back(std::move(NewChain));
+      Chains.emplace_back(std::move(NewChain));
     }
     return Chains;
   };
@@ -689,7 +691,7 @@ std::vector<Chain> Vectorizer::splitChainByAlignment(Chain &C) {
   });
 
   bool IsLoadChain = isa<LoadInst>(C[0].Inst);
-  auto getVectorFactor = [&](unsigned VF, unsigned LoadStoreSize,
+  auto GetVectorFactor = [&](unsigned VF, unsigned LoadStoreSize,
                              unsigned ChainSizeBytes, VectorType *VecTy) {
     return IsLoadChain ? TTI.getLoadVectorFactor(VF, LoadStoreSize,
                                                  ChainSizeBytes, VecTy)
@@ -721,8 +723,8 @@ std::vector<Chain> Vectorizer::splitChainByAlignment(Chain &C) {
                  C[CBegin].OffsetFromLeader;
       if (Sz.sgt(VecRegBytes))
         break;
-      CandidateChains.push_back(
-          {CEnd, static_cast<unsigned>(Sz.getLimitedValue())});
+      CandidateChains.emplace_back(CEnd,
+                                   static_cast<unsigned>(Sz.getLimitedValue()));
     }
 
     // Consider the longest chain first.
@@ -746,7 +748,7 @@ std::vector<Chain> Vectorizer::splitChainByAlignment(Chain &C) {
       unsigned VF = 8 * VecRegBytes / VecElemBits;
 
       // Check that TTI is happy with this vectorization factor.
-      unsigned TargetVF = getVectorFactor(VF, VecElemBits,
+      unsigned TargetVF = GetVectorFactor(VF, VecElemBits,
                                           VecElemBits * NumVecElems / 8, VecTy);
       if (TargetVF != VF && TargetVF < NumVecElems) {
         LLVM_DEBUG(
@@ -840,7 +842,7 @@ std::vector<Chain> Vectorizer::splitChainByAlignment(Chain &C) {
       // Hooray, we can vectorize this chain!
       Chain &NewChain = Ret.emplace_back();
       for (unsigned I = CBegin; I <= CEnd; ++I)
-        NewChain.push_back(C[I]);
+        NewChain.emplace_back(C[I]);
       CBegin = CEnd; // Skip over the instructions we've added to the chain.
       break;
     }
@@ -959,7 +961,7 @@ bool Vectorizer::vectorizeChain(Chain &C) {
       Vec = Builder.CreateInsertElement(Vec, V, Builder.getInt32(VecIdx++));
     };
     for (const ChainElem &E : C) {
-      auto I = cast<StoreInst>(E.Inst);
+      auto *I = cast<StoreInst>(E.Inst);
       if (FixedVectorType *VT =
               dyn_cast<FixedVectorType>(getLoadStoreType(I))) {
         for (int J = 0, JE = VT->getNumElements(); J < JE; ++J) {
@@ -982,7 +984,7 @@ bool Vectorizer::vectorizeChain(Chain &C) {
   propagateMetadata(VecInst, C);
 
   for (const ChainElem &E : C)
-    ToErase.push_back(E.Inst);
+    ToErase.emplace_back(E.Inst);
 
   ++NumVectorInstructions;
   NumScalarsVectorized += C.size();
@@ -1291,7 +1293,7 @@ std::optional<APInt> Vectorizer::getConstantOffsetSelects(
                         << *ContextInst << ", Depth=" << Depth << "\n");
       std::optional<APInt> TrueDiff = getConstantOffset(
           SelectA->getTrueValue(), SelectB->getTrueValue(), ContextInst, Depth);
-      if (!TrueDiff.has_value())
+      if (!TrueDiff)
         return std::nullopt;
       std::optional<APInt> FalseDiff =
           getConstantOffset(SelectA->getFalseValue(), SelectB->getFalseValue(),
@@ -1308,7 +1310,7 @@ Vectorizer::collectEquivalenceClasses(BasicBlock::iterator Begin,
                                       BasicBlock::iterator End) {
   EquivalenceClassMap Ret;
 
-  auto getUnderlyingObject = [](const Value *Ptr) -> const Value * {
+  auto GetUnderlyingObject = [](const Value *Ptr) -> const Value * {
     const Value *ObjPtr = llvm::getUnderlyingObject(Ptr);
     if (const auto *Sel = dyn_cast<SelectInst>(ObjPtr)) {
       // The select's themselves are distinct instructions even if they share
@@ -1369,10 +1371,10 @@ Vectorizer::collectEquivalenceClasses(BasicBlock::iterator Begin,
         (VecTy && TTI.getLoadVectorFactor(VF, TySize, TySize / 8, VecTy) == 0))
       continue;
 
-    Ret[{getUnderlyingObject(Ptr), AS,
+    Ret[{GetUnderlyingObject(Ptr), AS,
          DL.getTypeSizeInBits(getLoadStoreType(&I)->getScalarType()),
          /*IsLoad=*/LI != nullptr}]
-        .push_back(&I);
+        .emplace_back(&I);
   }
 
   return Ret;
@@ -1434,15 +1436,14 @@ std::vector<Chain> Vectorizer::gatherChains(ArrayRef<Instruction *> Instrs) {
     auto ChainIter = MRU.begin();
     for (size_t J = 0; J < MaxChainsToTry && ChainIter != MRU.end();
          ++J, ++ChainIter) {
-      std::optional<APInt> Offset = getConstantOffset(
-          getLoadStorePointerOperand(ChainIter->first),
-          getLoadStorePointerOperand(I),
-          /*ContextInst=*/
-          (ChainIter->first->comesBefore(I) ? I : ChainIter->first));
-      if (Offset.has_value()) {
+      if (std::optional<APInt> Offset = getConstantOffset(
+              getLoadStorePointerOperand(ChainIter->first),
+              getLoadStorePointerOperand(I),
+              /*ContextInst=*/
+              (ChainIter->first->comesBefore(I) ? I : ChainIter->first))) {
         // `Offset` might not have the expected number of bits, if e.g. AS has a
         // different number of bits than opaque pointers.
-        ChainIter->second.push_back(ChainElem{I, Offset.value()});
+        ChainIter->second.emplace_back(I, Offset.value());
         // Move ChainIter to the front of the MRU list.
         MRU.remove(*ChainIter);
         MRU.push_front(*ChainIter);
@@ -1454,7 +1455,7 @@ std::vector<Chain> Vectorizer::gatherChains(ArrayRef<Instruction *> Instrs) {
     if (!MatchFound) {
       APInt ZeroOffset(ASPtrBits, 0);
       InstrListElem *E = new (Allocator.Allocate()) InstrListElem(I);
-      E->second.push_back(ChainElem{I, ZeroOffset});
+      E->second.emplace_back(I, ZeroOffset);
       MRU.push_front(*E);
       Chains.insert(E);
     }
@@ -1465,7 +1466,7 @@ std::vector<Chain> Vectorizer::gatherChains(ArrayRef<Instruction *> Instrs) {
   // Iterate over MRU rather than Chains so the order is deterministic.
   for (auto &E : MRU)
     if (E.second.size() > 1)
-      Ret.push_back(std::move(E.second));
+      Ret.emplace_back(std::move(E.second));
   return Ret;
 }
 
@@ -1509,9 +1510,8 @@ std::optional<APInt> Vectorizer::getConstantOffset(Value *PtrA, Value *PtrB,
       return (OffsetB - OffsetA + Dist).sextOrTrunc(OrigBitWidth);
     }
   }
-  std::optional<APInt> Diff =
-      getConstantOffsetComplexAddrs(PtrA, PtrB, ContextInst, Depth);
-  if (Diff.has_value())
+  if (std::optional<APInt> Diff =
+          getConstantOffsetComplexAddrs(PtrA, PtrB, ContextInst, Depth))
     return (OffsetB - OffsetA + Diff->sext(OffsetB.getBitWidth()))
         .sextOrTrunc(OrigBitWidth);
   return std::nullopt;

From 22b4496e86c32127e997f3e7385ef64d2d80cc4b Mon Sep 17 00:00:00 2001
From: Dmitry Yanovsky <kerambyte@gmail.com>
Date: Mon, 19 Aug 2024 19:56:45 +0100
Subject: [PATCH 404/441] [ADT] Fix alignment check in unique_function
 constructor (#99403)

Right now the check fails for any state-capturing lambda since this
expression - `alignof(decltype(StorageUnion.InlineStorage))` - returns 1
for the alignment value and not 4/8 as expected
([MSVC|Clang|GCC](https://godbolt.org/z/eTEdq4xjM)). So this check fails
for pretty much any state-capturing callable we try to store into a
`unique_function` and we take the out-of-line storage path:

\llvm-project\llvm\include\llvm\ADT\FunctionExtras.h,
`UniqueFunctionBase` constructor (line ~266):

```
if (sizeof(CallableT) > InlineStorageSize ||
    alignof(CallableT) > alignof(decltype(StorageUnion.InlineStorage))) {
  // ...
}
```

The fix is simply to use an explicit const variable to store the
alignment value.

There is no easy way to unit-test the fix since inline storage is
considered to be an implementation detail so we shouldn't assume how the
lambda ends up being stored.
---
 llvm/include/llvm/ADT/FunctionExtras.h    |  6 ++++--
 llvm/unittests/ADT/FunctionExtrasTest.cpp | 19 +++++++++++++++++++
 2 files changed, 23 insertions(+), 2 deletions(-)

diff --git a/llvm/include/llvm/ADT/FunctionExtras.h b/llvm/include/llvm/ADT/FunctionExtras.h
index 49e0e8ab0db400d..ad5c89cbc8bda7a 100644
--- a/llvm/include/llvm/ADT/FunctionExtras.h
+++ b/llvm/include/llvm/ADT/FunctionExtras.h
@@ -80,6 +80,7 @@ using EnableIfCallable = std::enable_if_t<std::disjunction<
 template <typename ReturnT, typename... ParamTs> class UniqueFunctionBase {
 protected:
   static constexpr size_t InlineStorageSize = sizeof(void *) * 3;
+  static constexpr size_t InlineStorageAlign = alignof(void *);
 
   template <typename T, class = void>
   struct IsSizeLessThanThresholdT : std::false_type {};
@@ -161,7 +162,8 @@ template <typename ReturnT, typename... ParamTs> class UniqueFunctionBase {
     // provide three pointers worth of storage here.
     // This is mutable as an inlined `const unique_function<void() const>` may
     // still modify its own mutable members.
-    alignas(void *) mutable std::byte InlineStorage[InlineStorageSize];
+    alignas(InlineStorageAlign) mutable std::byte
+        InlineStorage[InlineStorageSize];
   } StorageUnion;
 
   // A compressed pointer to either our dispatching callback or our table of
@@ -262,7 +264,7 @@ template <typename ReturnT, typename... ParamTs> class UniqueFunctionBase {
     bool IsInlineStorage = true;
     void *CallableAddr = getInlineStorage();
     if (sizeof(CallableT) > InlineStorageSize ||
-        alignof(CallableT) > alignof(decltype(StorageUnion.InlineStorage))) {
+        alignof(CallableT) > InlineStorageAlign) {
       IsInlineStorage = false;
       // Allocate out-of-line storage. FIXME: Use an explicit alignment
       // parameter in C++17 mode.
diff --git a/llvm/unittests/ADT/FunctionExtrasTest.cpp b/llvm/unittests/ADT/FunctionExtrasTest.cpp
index fc856a976946bf6..7abdc8b77e05950 100644
--- a/llvm/unittests/ADT/FunctionExtrasTest.cpp
+++ b/llvm/unittests/ADT/FunctionExtrasTest.cpp
@@ -310,4 +310,23 @@ class Incomplete {};
 Incomplete incompleteFunction() { return {}; }
 const Incomplete incompleteFunctionConst() { return {}; }
 
+// Check that we can store a pointer-sized payload inline in the unique_function.
+TEST(UniqueFunctionTest, InlineStorageWorks) {
+  // We do assume a couple of implementation details of the unique_function here:
+  //  - It can store certain small-enough payload inline
+  //  - Inline storage size is at least >= sizeof(void*)
+  void *ptr;
+  unique_function<void(void *)> UniqueFunctionWithInlineStorage{
+      [ptr](void *self) {
+        auto mid = reinterpret_cast<uintptr_t>(&ptr);
+        auto beg = reinterpret_cast<uintptr_t>(self);
+        auto end = reinterpret_cast<uintptr_t>(self) +
+                   sizeof(unique_function<void(void *)>);
+        // Make sure the address of the captured pointer lies somewhere within
+        // the unique_function object.
+        EXPECT_TRUE(mid >= beg && mid < end);
+      }};
+  UniqueFunctionWithInlineStorage(&UniqueFunctionWithInlineStorage);
+}
+
 } // anonymous namespace

From 3b80a28c8f27536f49e5fa30a7e2b7050bad6cd6 Mon Sep 17 00:00:00 2001
From: Amr Hesham <amr96@programmer.net>
Date: Mon, 19 Aug 2024 20:58:06 +0200
Subject: [PATCH 405/441] [mlir][spirv] Fix incorrect metadata in SPIR-V Header
 (#104242)

SRIP-V Generator Magic Number is a 32 bit word: The high order 16 bits
are a tool ID. The low order 16 bits are for verion number, currently
the tool id is not on the high order 16 bit so it must shifed

fixes: #99071
---
 mlir/lib/Target/SPIRV/SPIRVBinaryUtils.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mlir/lib/Target/SPIRV/SPIRVBinaryUtils.cpp b/mlir/lib/Target/SPIRV/SPIRVBinaryUtils.cpp
index 3c8f64419d379ea..18a108be764b379 100644
--- a/mlir/lib/Target/SPIRV/SPIRVBinaryUtils.cpp
+++ b/mlir/lib/Target/SPIRV/SPIRVBinaryUtils.cpp
@@ -52,7 +52,7 @@ void spirv::appendModuleHeader(SmallVectorImpl<uint32_t> &header,
   // +-------------------------------------------------------------------------+
   header.push_back(spirv::kMagicNumber);
   header.push_back((majorVersion << 16) | (minorVersion << 8));
-  header.push_back(kGeneratorNumber);
+  header.push_back((kGeneratorNumber << 16) | LLVM_VERSION_MAJOR);
   header.push_back(idBound); // <id> bound
   header.push_back(0);       // Schema (reserved word)
 }

From 7597e0930638e0a20ca9bfc193a3d89575ce4469 Mon Sep 17 00:00:00 2001
From: Amr Hesham <amr96@programmer.net>
Date: Mon, 19 Aug 2024 21:07:25 +0200
Subject: [PATCH 406/441] [clang] Increase the default expression nesting limit
 (#104717)

Increase the default expression nesting limit from 256 to 1024

Fixes: #94728

Compile time with different Bracket depth

Clang version 20.0.0git
(https://github.com/AmrDeveloper/llvm-project.git
673b9e08de8a661c9deed2ee497889312f059f3d)
Target: arm64-apple-darwin23.5.0

Bracket depth = 256, time = 0.243
Bracket depth = 512, time = 0.329
Bracket depth = 1024, time = 0.489
Bracket depth = 2048, time = 0.851
---
 clang/docs/ReleaseNotes.rst           | 2 ++
 clang/include/clang/Driver/Options.td | 2 +-
 clang/test/Parser/parser_overflow.c   | 4 ++--
 3 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 017df3cae468a1e..249249971dec7ca 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -174,6 +174,8 @@ Deprecated Compiler Flags
 Modified Compiler Flags
 -----------------------
 
+- The compiler flag `-fbracket-depth` default value is increased from 256 to 2048.
+
 Removed Compiler Flags
 -------------------------
 
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index cfd9e595c551784..c66e035a259b3ff 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -7959,7 +7959,7 @@ def fapply_global_visibility_to_externs : Flag<["-"], "fapply-global-visibility-
   MarshallingInfoFlag<LangOpts<"SetVisibilityForExternDecls">>;
 def fbracket_depth : Separate<["-"], "fbracket-depth">,
   HelpText<"Maximum nesting level for parentheses, brackets, and braces">,
-  MarshallingInfoInt<LangOpts<"BracketDepth">, "256">;
+  MarshallingInfoInt<LangOpts<"BracketDepth">, "2048">;
 defm const_strings : BoolOption<"f", "const-strings",
   LangOpts<"ConstStrings">, DefaultFalse,
   PosFlag<SetTrue, [], [ClangOption, CC1Option], "Use">,
diff --git a/clang/test/Parser/parser_overflow.c b/clang/test/Parser/parser_overflow.c
index 9514e808550a4b8..53c79bc06d993d9 100644
--- a/clang/test/Parser/parser_overflow.c
+++ b/clang/test/Parser/parser_overflow.c
@@ -1,5 +1,5 @@
 // RUN: not %clang_cc1 %s -fsyntax-only -DHUGE 2>&1 | FileCheck %s
-// RUN: not %clang_cc1 %s -fsyntax-only 2>&1 | FileCheck %s
+// RUN: %clang_cc1 %s -fsyntax-only
 // RUN: not %clang_cc1 %s -fsyntax-only -fbracket-depth 299 2>&1 | FileCheck %s
 // RUN: %clang_cc1 %s -fsyntax-only -fbracket-depth 300
 // RUN: not %clang %s -fsyntax-only -fbracket-depth=299 2>&1 | FileCheck %s
@@ -15,5 +15,5 @@ void foo(void) {
 #endif
 }
 
-// CHECK: fatal error: bracket nesting level exceeded maximum of {{256|299}}
+// CHECK: fatal error: bracket nesting level exceeded maximum of {{2048|299}}
 // CHECK: note: use -fbracket-depth=N to increase maximum nesting level

From 4c967afcc6f3a109e8a325a0327f869c4bfc57c2 Mon Sep 17 00:00:00 2001
From: Michael Kenzel <michael.kenzel@gmail.com>
Date: Mon, 19 Aug 2024 21:14:32 +0200
Subject: [PATCH 407/441] [libc++abi] Remove unnecessary dependency on
 std::unique_ptr (#73277)

The demangling terminate handler uses a function `demangle()` to perform
the demangling. This function returns an `std::unique_ptr`, relying on
custom deleter and `const_cast` hacks to deal with the facts that only
one branch actually allocates, and that the pointer type needs to be
`const char*`. However, the destructor of the returned `std::unique_ptr`
will never actually run, because the sole place this function is ever
called is right before the terminate handler aborts the program. So all
this is unnecessary, and creates a dependency onto `<memory>`. This
change removes the `demangle()` function and replaces the call with an
immediately invoked lambda expression that simply returns a raw pointer
in both cases, which should be fine because the memory can never be
freed here anyways.
---
 libcxxabi/src/cxa_default_handlers.cpp | 30 ++++++++++++--------------
 1 file changed, 14 insertions(+), 16 deletions(-)

diff --git a/libcxxabi/src/cxa_default_handlers.cpp b/libcxxabi/src/cxa_default_handlers.cpp
index 7bcfca069fb7f71..60e402c55b395b6 100644
--- a/libcxxabi/src/cxa_default_handlers.cpp
+++ b/libcxxabi/src/cxa_default_handlers.cpp
@@ -10,8 +10,6 @@
 //===----------------------------------------------------------------------===//
 
 #include <exception>
-#include <memory>
-#include <stdlib.h>
 #include "abort_message.h"
 #include "cxxabi.h"
 #include "cxa_handlers.h"
@@ -23,17 +21,7 @@
 
 static constinit const char* cause = "uncaught";
 
-#ifndef _LIBCXXABI_NO_EXCEPTIONS
-// Demangle the given string, or return the string as-is in case of an error.
-static std::unique_ptr<char const, void (*)(char const*)> demangle(char const* str)
-{
-#if !defined(LIBCXXABI_NON_DEMANGLING_TERMINATE)
-    if (const char* result = __cxxabiv1::__cxa_demangle(str, nullptr, nullptr, nullptr))
-        return {result, [](char const* p) { std::free(const_cast<char*>(p)); }};
-#endif
-    return {str, [](char const*) { /* nothing to free */ }};
-}
-
+#  ifndef _LIBCXXABI_NO_EXCEPTIONS
 __attribute__((noreturn))
 static void demangling_terminate_handler()
 {
@@ -61,7 +49,17 @@ static void demangling_terminate_handler()
             exception_header + 1;
     const __shim_type_info* thrown_type =
         static_cast<const __shim_type_info*>(exception_header->exceptionType);
-    auto name = demangle(thrown_type->name());
+
+    auto name = [str = thrown_type->name()] {
+#    ifndef LIBCXXABI_NON_DEMANGLING_TERMINATE
+      if (const char* result = __cxxabiv1::__cxa_demangle(str, nullptr, nullptr, nullptr))
+        // We're about to abort(), this memory can never be freed; so it's fine
+        // to just return a raw pointer
+        return result;
+#    endif
+      return str;
+    }();
+
     // If the uncaught exception can be caught with std::exception&
     const __shim_type_info* catch_type =
         static_cast<const __shim_type_info*>(&typeid(std::exception));
@@ -69,12 +67,12 @@ static void demangling_terminate_handler()
     {
         // Include the what() message from the exception
         const std::exception* e = static_cast<const std::exception*>(thrown_object);
-        abort_message("terminating due to %s exception of type %s: %s", cause, name.get(), e->what());
+        abort_message("terminating due to %s exception of type %s: %s", cause, name, e->what());
     }
     else
     {
         // Else just note that we're terminating due to an exception
-        abort_message("terminating due to %s exception of type %s", cause, name.get());
+        abort_message("terminating due to %s exception of type %s", cause, name);
     }
 }
 #else // !_LIBCXXABI_NO_EXCEPTIONS

From 87dd31234a3c46ace097af73ecaedcdb6b306ef3 Mon Sep 17 00:00:00 2001
From: Rahul Joshi <rjoshi@nvidia.com>
Date: Mon, 19 Aug 2024 12:20:47 -0700
Subject: [PATCH 408/441] [ADT] Minor code cleanup in STLExtras.h (#104808)

- Remove unnecessary return from `replace`.
- Add comment for `min_element` and `max_element`.
---
 llvm/include/llvm/ADT/STLExtras.h | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/llvm/include/llvm/ADT/STLExtras.h b/llvm/include/llvm/ADT/STLExtras.h
index a810a1c318c42cb..249ac384bec41e6 100644
--- a/llvm/include/llvm/ADT/STLExtras.h
+++ b/llvm/include/llvm/ADT/STLExtras.h
@@ -1847,7 +1847,7 @@ OutputIt replace_copy(R &&Range, OutputIt Out, const T &OldValue,
 /// begin/end explicitly.
 template <typename R, typename T>
 void replace(R &&Range, const T &OldValue, const T &NewValue) {
-  return std::replace(adl_begin(Range), adl_end(Range), OldValue, NewValue);
+  std::replace(adl_begin(Range), adl_end(Range), OldValue, NewValue);
 }
 
 /// Provide wrappers to std::move which take ranges instead of having to
@@ -1982,6 +1982,8 @@ auto upper_bound(R &&Range, T &&Value, Compare C) {
                           std::forward<T>(Value), C);
 }
 
+/// Provide wrappers to std::min_element which take ranges instead of having to
+/// pass begin/end explicitly.
 template <typename R> auto min_element(R &&Range) {
   return std::min_element(adl_begin(Range), adl_end(Range));
 }
@@ -1990,6 +1992,8 @@ template <typename R, typename Compare> auto min_element(R &&Range, Compare C) {
   return std::min_element(adl_begin(Range), adl_end(Range), C);
 }
 
+/// Provide wrappers to std::max_element which take ranges instead of having to
+/// pass begin/end explicitly.
 template <typename R> auto max_element(R &&Range) {
   return std::max_element(adl_begin(Range), adl_end(Range));
 }

From e8ca306fbefdb8385c53d6cc029251d1d5be7049 Mon Sep 17 00:00:00 2001
From: Dmitry Yanovsky <kerambyte@gmail.com>
Date: Mon, 19 Aug 2024 20:38:06 +0100
Subject: [PATCH 409/441] [ADT] Add a missing call to a unique_function
 destructor after move (#98747)

Right now immediately after moving the captured state, we 'disarm' the
moved-from object's destructor by resetting the
`RHS.CallbackAndInlineFlag`. This means that we never properly destroy
the RHS object's captured state.
---
 llvm/include/llvm/ADT/FunctionExtras.h    |  1 +
 llvm/unittests/ADT/CountCopyAndMove.cpp   |  2 ++
 llvm/unittests/ADT/CountCopyAndMove.h     | 13 +++++++++++--
 llvm/unittests/ADT/FunctionExtrasTest.cpp | 15 +++++++++++++++
 4 files changed, 29 insertions(+), 2 deletions(-)

diff --git a/llvm/include/llvm/ADT/FunctionExtras.h b/llvm/include/llvm/ADT/FunctionExtras.h
index ad5c89cbc8bda7a..d92868e3715f40e 100644
--- a/llvm/include/llvm/ADT/FunctionExtras.h
+++ b/llvm/include/llvm/ADT/FunctionExtras.h
@@ -314,6 +314,7 @@ template <typename ReturnT, typename... ParamTs> class UniqueFunctionBase {
       // Non-trivial move, so dispatch to a type-erased implementation.
       getNonTrivialCallbacks()->MovePtr(getInlineStorage(),
                                         RHS.getInlineStorage());
+      getNonTrivialCallbacks()->DestroyPtr(RHS.getInlineStorage());
     }
 
     // Clear the old callback and inline flag to get back to as-if-null.
diff --git a/llvm/unittests/ADT/CountCopyAndMove.cpp b/llvm/unittests/ADT/CountCopyAndMove.cpp
index fe1e2f4a5b89fd5..022dbee40dc6f1c 100644
--- a/llvm/unittests/ADT/CountCopyAndMove.cpp
+++ b/llvm/unittests/ADT/CountCopyAndMove.cpp
@@ -10,6 +10,8 @@
 
 using namespace llvm;
 
+int CountCopyAndMove::DefaultConstructions = 0;
+int CountCopyAndMove::ValueConstructions = 0;
 int CountCopyAndMove::CopyConstructions = 0;
 int CountCopyAndMove::CopyAssignments = 0;
 int CountCopyAndMove::MoveConstructions = 0;
diff --git a/llvm/unittests/ADT/CountCopyAndMove.h b/llvm/unittests/ADT/CountCopyAndMove.h
index 126054427b81a35..c91d34bd8ebb507 100644
--- a/llvm/unittests/ADT/CountCopyAndMove.h
+++ b/llvm/unittests/ADT/CountCopyAndMove.h
@@ -12,6 +12,8 @@
 namespace llvm {
 
 struct CountCopyAndMove {
+  static int DefaultConstructions;
+  static int ValueConstructions;
   static int CopyConstructions;
   static int CopyAssignments;
   static int MoveConstructions;
@@ -19,8 +21,8 @@ struct CountCopyAndMove {
   static int Destructions;
   int val;
 
-  CountCopyAndMove() = default;
-  explicit CountCopyAndMove(int val) : val(val) {}
+  CountCopyAndMove() { ++DefaultConstructions; }
+  explicit CountCopyAndMove(int val) : val(val) { ++ValueConstructions; }
   CountCopyAndMove(const CountCopyAndMove &other) : val(other.val) {
     ++CopyConstructions;
   }
@@ -40,6 +42,8 @@ struct CountCopyAndMove {
   ~CountCopyAndMove() { ++Destructions; }
 
   static void ResetCounts() {
+    DefaultConstructions = 0;
+    ValueConstructions = 0;
     CopyConstructions = 0;
     CopyAssignments = 0;
     MoveConstructions = 0;
@@ -47,6 +51,11 @@ struct CountCopyAndMove {
     Destructions = 0;
   }
 
+  static int TotalConstructions() {
+    return DefaultConstructions + ValueConstructions + MoveConstructions +
+           CopyConstructions;
+  }
+
   static int TotalCopies() { return CopyConstructions + CopyAssignments; }
 
   static int TotalMoves() { return MoveConstructions + MoveAssignments; }
diff --git a/llvm/unittests/ADT/FunctionExtrasTest.cpp b/llvm/unittests/ADT/FunctionExtrasTest.cpp
index 7abdc8b77e05950..f64b888dbe1591c 100644
--- a/llvm/unittests/ADT/FunctionExtrasTest.cpp
+++ b/llvm/unittests/ADT/FunctionExtrasTest.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/ADT/FunctionExtras.h"
+#include "CountCopyAndMove.h"
 #include "gtest/gtest.h"
 
 #include <memory>
@@ -329,4 +330,18 @@ TEST(UniqueFunctionTest, InlineStorageWorks) {
   UniqueFunctionWithInlineStorage(&UniqueFunctionWithInlineStorage);
 }
 
+// Check that the moved-from captured state is properly destroyed during
+// move construction/assignment.
+TEST(UniqueFunctionTest, MovedFromStateIsDestroyedCorrectly) {
+  CountCopyAndMove::ResetCounts();
+  {
+    unique_function<void()> CapturingFunction{
+        [Counter = CountCopyAndMove{}] {}};
+    unique_function<void()> CapturingFunctionMoved{
+        std::move(CapturingFunction)};
+  }
+  EXPECT_EQ(CountCopyAndMove::TotalConstructions(),
+            CountCopyAndMove::Destructions);
+}
+
 } // anonymous namespace

From 1393aeb91e88381ced7512788761e42bde8fd1cc Mon Sep 17 00:00:00 2001
From: Krzysztof Drewniak <Krzysztof.Drewniak@amd.com>
Date: Mon, 19 Aug 2024 14:52:21 -0500
Subject: [PATCH 410/441] Pre-commit AMDGPU tests for masked
 load/store/scatter/gather (#104645)

I'm planning to fix the masked operation scalarazer to not generate
suboptimal code on AMD GPUs and other SIMT machines, and so am adding
tests now.
---
 .../AMDGPU/expamd-masked-load.ll              | 170 ++++++++++++++++++
 .../AMDGPU/expand-masked-gather.ll            |  38 ++++
 .../AMDGPU/expand-masked-scatter.ll           |  36 ++++
 .../AMDGPU/expand-masked-store.ll             | 161 +++++++++++++++++
 .../AMDGPU/lit.local.cfg                      |   2 +
 5 files changed, 407 insertions(+)
 create mode 100644 llvm/test/Transforms/ScalarizeMaskedMemIntrin/AMDGPU/expamd-masked-load.ll
 create mode 100644 llvm/test/Transforms/ScalarizeMaskedMemIntrin/AMDGPU/expand-masked-gather.ll
 create mode 100644 llvm/test/Transforms/ScalarizeMaskedMemIntrin/AMDGPU/expand-masked-scatter.ll
 create mode 100644 llvm/test/Transforms/ScalarizeMaskedMemIntrin/AMDGPU/expand-masked-store.ll
 create mode 100644 llvm/test/Transforms/ScalarizeMaskedMemIntrin/AMDGPU/lit.local.cfg

diff --git a/llvm/test/Transforms/ScalarizeMaskedMemIntrin/AMDGPU/expamd-masked-load.ll b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/AMDGPU/expamd-masked-load.ll
new file mode 100644
index 000000000000000..35e5bcde4c0dbd2
--- /dev/null
+++ b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/AMDGPU/expamd-masked-load.ll
@@ -0,0 +1,170 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S %s -passes=scalarize-masked-mem-intrin -mtriple=amdgcn-amd-amdhsa | FileCheck %s
+
+; COM: Test that, unlike on CPU targets, the mask doesn't get bitcast to a scalar,
+; COM: since, on GPUs, each i1 takes up at least one register and so they should
+; COM: be treated separately.
+
+define <2 x i32> @scalarize_v2i32(ptr %p, <2 x i1> %mask, <2 x i32> %passthru) {
+; CHECK-LABEL: define <2 x i32> @scalarize_v2i32(
+; CHECK-SAME: ptr [[P:%.*]], <2 x i1> [[MASK:%.*]], <2 x i32> [[PASSTHRU:%.*]]) {
+; CHECK-NEXT:    [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK]] to i2
+; CHECK-NEXT:    [[TMP1:%.*]] = and i2 [[SCALAR_MASK]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i2 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[TMP2]], label %[[COND_LOAD:.*]], label %[[ELSE:.*]]
+; CHECK:       [[COND_LOAD]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x i32> [[PASSTHRU]], i32 [[TMP4]], i64 0
+; CHECK-NEXT:    br label %[[ELSE]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[RES_PHI_ELSE:%.*]] = phi <2 x i32> [ [[TMP5]], %[[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]
+; CHECK-NEXT:    [[TMP6:%.*]] = and i2 [[SCALAR_MASK]], -2
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ne i2 [[TMP6]], 0
+; CHECK-NEXT:    br i1 [[TMP7]], label %[[COND_LOAD1:.*]], label %[[ELSE2:.*]]
+; CHECK:       [[COND_LOAD1]]:
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 1
+; CHECK-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP8]], align 4
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <2 x i32> [[RES_PHI_ELSE]], i32 [[TMP9]], i64 1
+; CHECK-NEXT:    br label %[[ELSE2]]
+; CHECK:       [[ELSE2]]:
+; CHECK-NEXT:    [[RES_PHI_ELSE3:%.*]] = phi <2 x i32> [ [[TMP10]], %[[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <2 x i32> [[RES_PHI_ELSE3]]
+;
+  %ret = call <2 x i32> @llvm.masked.load.v2i32.p0(ptr %p, i32 128, <2 x i1> %mask, <2 x i32> %passthru)
+  ret <2 x i32> %ret
+}
+
+define <2 x i32> @scalarize_v2i32_splat_mask(ptr %p, i1 %mask, <2 x i32> %passthrough) {
+; CHECK-LABEL: define <2 x i32> @scalarize_v2i32_splat_mask(
+; CHECK-SAME: ptr [[P:%.*]], i1 [[MASK:%.*]], <2 x i32> [[PASSTHROUGH:%.*]]) {
+; CHECK-NEXT:    [[MASK_VEC:%.*]] = insertelement <2 x i1> poison, i1 [[MASK]], i32 0
+; CHECK-NEXT:    [[MASK_SPLAT:%.*]] = shufflevector <2 x i1> [[MASK_VEC]], <2 x i1> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[MASK_SPLAT_FIRST:%.*]] = extractelement <2 x i1> [[MASK_SPLAT]], i64 0
+; CHECK-NEXT:    br i1 [[MASK_SPLAT_FIRST]], label %[[COND_LOAD:.*]], label %[[BB1:.*]]
+; CHECK:       [[COND_LOAD]]:
+; CHECK-NEXT:    [[RET_COND_LOAD:%.*]] = load <2 x i32>, ptr [[P]], align 8
+; CHECK-NEXT:    br label %[[BB1]]
+; CHECK:       [[BB1]]:
+; CHECK-NEXT:    [[RET:%.*]] = phi <2 x i32> [ [[RET_COND_LOAD]], %[[COND_LOAD]] ], [ [[PASSTHROUGH]], [[TMP0:%.*]] ]
+; CHECK-NEXT:    ret <2 x i32> [[RET]]
+;
+  %mask.vec = insertelement <2 x i1> poison, i1 %mask, i32 0
+  %mask.splat = shufflevector <2 x i1> %mask.vec, <2 x i1> poison, <2 x i32> zeroinitializer
+  %ret = call <2 x i32> @llvm.masked.load.v2i32.p0(ptr %p, i32 8, <2 x i1> %mask.splat, <2 x i32> %passthrough)
+  ret <2 x i32> %ret
+}
+
+define <2 x half> @scalarize_v2f16(ptr %p, <2 x i1> %mask, <2 x half> %passthru) {
+; CHECK-LABEL: define <2 x half> @scalarize_v2f16(
+; CHECK-SAME: ptr [[P:%.*]], <2 x i1> [[MASK:%.*]], <2 x half> [[PASSTHRU:%.*]]) {
+; CHECK-NEXT:    [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK]] to i2
+; CHECK-NEXT:    [[TMP1:%.*]] = and i2 [[SCALAR_MASK]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i2 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[TMP2]], label %[[COND_LOAD:.*]], label %[[ELSE:.*]]
+; CHECK:       [[COND_LOAD]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds half, ptr [[P]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = load half, ptr [[TMP3]], align 2
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x half> [[PASSTHRU]], half [[TMP4]], i64 0
+; CHECK-NEXT:    br label %[[ELSE]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[RES_PHI_ELSE:%.*]] = phi <2 x half> [ [[TMP5]], %[[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]
+; CHECK-NEXT:    [[TMP6:%.*]] = and i2 [[SCALAR_MASK]], -2
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ne i2 [[TMP6]], 0
+; CHECK-NEXT:    br i1 [[TMP7]], label %[[COND_LOAD1:.*]], label %[[ELSE2:.*]]
+; CHECK:       [[COND_LOAD1]]:
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds half, ptr [[P]], i32 1
+; CHECK-NEXT:    [[TMP9:%.*]] = load half, ptr [[TMP8]], align 2
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <2 x half> [[RES_PHI_ELSE]], half [[TMP9]], i64 1
+; CHECK-NEXT:    br label %[[ELSE2]]
+; CHECK:       [[ELSE2]]:
+; CHECK-NEXT:    [[RES_PHI_ELSE3:%.*]] = phi <2 x half> [ [[TMP10]], %[[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <2 x half> [[RES_PHI_ELSE3]]
+;
+  %ret = call <2 x half> @llvm.masked.load.v2f16.p0(ptr %p, i32 128, <2 x i1> %mask, <2 x half> %passthru)
+  ret <2 x half> %ret
+}
+
+define <2 x i32> @scalarize_v2i32_p3(ptr addrspace(3) %p, <2 x i1> %mask, <2 x i32> %passthru) {
+; CHECK-LABEL: define <2 x i32> @scalarize_v2i32_p3(
+; CHECK-SAME: ptr addrspace(3) [[P:%.*]], <2 x i1> [[MASK:%.*]], <2 x i32> [[PASSTHRU:%.*]]) {
+; CHECK-NEXT:    [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK]] to i2
+; CHECK-NEXT:    [[TMP1:%.*]] = and i2 [[SCALAR_MASK]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i2 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[TMP2]], label %[[COND_LOAD:.*]], label %[[ELSE:.*]]
+; CHECK:       [[COND_LOAD]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, ptr addrspace(3) [[P]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr addrspace(3) [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x i32> [[PASSTHRU]], i32 [[TMP4]], i64 0
+; CHECK-NEXT:    br label %[[ELSE]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[RES_PHI_ELSE:%.*]] = phi <2 x i32> [ [[TMP5]], %[[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]
+; CHECK-NEXT:    [[TMP6:%.*]] = and i2 [[SCALAR_MASK]], -2
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ne i2 [[TMP6]], 0
+; CHECK-NEXT:    br i1 [[TMP7]], label %[[COND_LOAD1:.*]], label %[[ELSE2:.*]]
+; CHECK:       [[COND_LOAD1]]:
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr addrspace(3) [[P]], i32 1
+; CHECK-NEXT:    [[TMP9:%.*]] = load i32, ptr addrspace(3) [[TMP8]], align 4
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <2 x i32> [[RES_PHI_ELSE]], i32 [[TMP9]], i64 1
+; CHECK-NEXT:    br label %[[ELSE2]]
+; CHECK:       [[ELSE2]]:
+; CHECK-NEXT:    [[RES_PHI_ELSE3:%.*]] = phi <2 x i32> [ [[TMP10]], %[[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <2 x i32> [[RES_PHI_ELSE3]]
+;
+  %ret = call <2 x i32> @llvm.masked.load.v2i32.p3(ptr addrspace(3) %p, i32 128, <2 x i1> %mask, <2 x i32> %passthru)
+  ret <2 x i32> %ret
+}
+
+define <2 x i32> @scalarize_v2i32_lane_mask(ptr %p, <2 x i32> %passthrough) {
+; CHECK-LABEL: define <2 x i32> @scalarize_v2i32_lane_mask(
+; CHECK-SAME: ptr [[P:%.*]], <2 x i32> [[PASSTHROUGH:%.*]]) {
+; CHECK-NEXT:    [[ITEM_ID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT:    [[MASK:%.*]] = icmp ult i32 [[ITEM_ID]], 16
+; CHECK-NEXT:    [[MASK_VEC:%.*]] = insertelement <2 x i1> poison, i1 [[MASK]], i32 0
+; CHECK-NEXT:    [[MASK_SPLAT:%.*]] = shufflevector <2 x i1> [[MASK_VEC]], <2 x i1> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[MASK_SPLAT_FIRST:%.*]] = extractelement <2 x i1> [[MASK_SPLAT]], i64 0
+; CHECK-NEXT:    br i1 [[MASK_SPLAT_FIRST]], label %[[COND_LOAD:.*]], label %[[BB1:.*]]
+; CHECK:       [[COND_LOAD]]:
+; CHECK-NEXT:    [[RET_COND_LOAD:%.*]] = load <2 x i32>, ptr [[P]], align 8
+; CHECK-NEXT:    br label %[[BB1]]
+; CHECK:       [[BB1]]:
+; CHECK-NEXT:    [[RET:%.*]] = phi <2 x i32> [ [[RET_COND_LOAD]], %[[COND_LOAD]] ], [ [[PASSTHROUGH]], [[TMP0:%.*]] ]
+; CHECK-NEXT:    ret <2 x i32> [[RET]]
+;
+  %item.id = call i32 @llvm.amdgcn.workitem.id.x()
+  %mask = icmp ult i32 %item.id, 16
+  %mask.vec = insertelement <2 x i1> poison, i1 %mask, i32 0
+  %mask.splat = shufflevector <2 x i1> %mask.vec, <2 x i1> poison, <2 x i32> zeroinitializer
+  %ret = call <2 x i32> @llvm.masked.load.v2i32.p0(ptr %p, i32 8, <2 x i1> %mask.splat, <2 x i32> %passthrough)
+  ret <2 x i32> %ret
+}
+
+define <2 x i32> @scalarize_v2i32_group_mask(ptr %p, <2 x i32> %passthrough) {
+; CHECK-LABEL: define <2 x i32> @scalarize_v2i32_group_mask(
+; CHECK-SAME: ptr [[P:%.*]], <2 x i32> [[PASSTHROUGH:%.*]]) {
+; CHECK-NEXT:    [[GROUP_ID:%.*]] = call i32 @llvm.amdgcn.workgroup.id.x()
+; CHECK-NEXT:    [[MASK:%.*]] = icmp ult i32 [[GROUP_ID]], 4
+; CHECK-NEXT:    [[MASK_VEC:%.*]] = insertelement <2 x i1> poison, i1 [[MASK]], i32 0
+; CHECK-NEXT:    [[MASK_SPLAT:%.*]] = shufflevector <2 x i1> [[MASK_VEC]], <2 x i1> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[MASK_SPLAT_FIRST:%.*]] = extractelement <2 x i1> [[MASK_SPLAT]], i64 0
+; CHECK-NEXT:    br i1 [[MASK_SPLAT_FIRST]], label %[[COND_LOAD:.*]], label %[[BB1:.*]]
+; CHECK:       [[COND_LOAD]]:
+; CHECK-NEXT:    [[RET_COND_LOAD:%.*]] = load <2 x i32>, ptr [[P]], align 8
+; CHECK-NEXT:    br label %[[BB1]]
+; CHECK:       [[BB1]]:
+; CHECK-NEXT:    [[RET:%.*]] = phi <2 x i32> [ [[RET_COND_LOAD]], %[[COND_LOAD]] ], [ [[PASSTHROUGH]], [[TMP0:%.*]] ]
+; CHECK-NEXT:    ret <2 x i32> [[RET]]
+;
+  %group.id = call i32 @llvm.amdgcn.workgroup.id.x()
+  %mask = icmp ult i32 %group.id, 4
+  %mask.vec = insertelement <2 x i1> poison, i1 %mask, i32 0
+  %mask.splat = shufflevector <2 x i1> %mask.vec, <2 x i1> poison, <2 x i32> zeroinitializer
+  %ret = call <2 x i32> @llvm.masked.load.v2i32.p0(ptr %p, i32 8, <2 x i1> %mask.splat, <2 x i32> %passthrough)
+  ret <2 x i32> %ret
+}
+
+declare <2 x i32> @llvm.masked.load.v2i32.p0(ptr, i32, <2 x i1>, <2 x i32>)
+declare <2 x half> @llvm.masked.load.v2f16.p0(ptr, i32, <2 x i1>, <2 x half>)
+declare <2 x i32> @llvm.masked.load.v2i32.p3(ptr addrspace(3), i32, <2 x i1>, <2 x i32>)
+declare noundef i32 @llvm.amdgcn.workitem.id.x()
+declare noundef i32 @llvm.amdgcn.workgroup.id.x()
diff --git a/llvm/test/Transforms/ScalarizeMaskedMemIntrin/AMDGPU/expand-masked-gather.ll b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/AMDGPU/expand-masked-gather.ll
new file mode 100644
index 000000000000000..94d0e2943d93664
--- /dev/null
+++ b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/AMDGPU/expand-masked-gather.ll
@@ -0,0 +1,38 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S %s -passes=scalarize-masked-mem-intrin -mtriple=amdgcn-amd-amdhsa | FileCheck %s
+
+; COM: Test that, unlike on CPU targets, the mask doesn't get bitcast to a scalar,
+; COM: since, on GPUs, each i1 takes up at least one register and so they should
+; COM: be treated separately.
+
+define <2 x i32> @scalarize_v2i32(<2 x ptr> %p, <2 x i1> %mask, <2 x i32> %passthru) {
+; CHECK-LABEL: define <2 x i32> @scalarize_v2i32(
+; CHECK-SAME: <2 x ptr> [[P:%.*]], <2 x i1> [[MASK:%.*]], <2 x i32> [[PASSTHRU:%.*]]) {
+; CHECK-NEXT:    [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK]] to i2
+; CHECK-NEXT:    [[TMP1:%.*]] = and i2 [[SCALAR_MASK]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i2 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[TMP2]], label %[[COND_LOAD:.*]], label %[[ELSE:.*]]
+; CHECK:       [[COND_LOAD]]:
+; CHECK-NEXT:    [[PTR0:%.*]] = extractelement <2 x ptr> [[P]], i64 0
+; CHECK-NEXT:    [[LOAD0:%.*]] = load i32, ptr [[PTR0]], align 8
+; CHECK-NEXT:    [[RES0:%.*]] = insertelement <2 x i32> [[PASSTHRU]], i32 [[LOAD0]], i64 0
+; CHECK-NEXT:    br label %[[ELSE]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[RES_PHI_ELSE:%.*]] = phi <2 x i32> [ [[RES0]], %[[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = and i2 [[SCALAR_MASK]], -2
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne i2 [[TMP3]], 0
+; CHECK-NEXT:    br i1 [[TMP4]], label %[[COND_LOAD1:.*]], label %[[ELSE2:.*]]
+; CHECK:       [[COND_LOAD1]]:
+; CHECK-NEXT:    [[PTR1:%.*]] = extractelement <2 x ptr> [[P]], i64 1
+; CHECK-NEXT:    [[LOAD1:%.*]] = load i32, ptr [[PTR1]], align 8
+; CHECK-NEXT:    [[RES1:%.*]] = insertelement <2 x i32> [[RES_PHI_ELSE]], i32 [[LOAD1]], i64 1
+; CHECK-NEXT:    br label %[[ELSE2]]
+; CHECK:       [[ELSE2]]:
+; CHECK-NEXT:    [[RES_PHI_ELSE3:%.*]] = phi <2 x i32> [ [[RES1]], %[[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <2 x i32> [[RES_PHI_ELSE3]]
+;
+  %ret = call <2 x i32> @llvm.masked.gather.v2i32.v2p0(<2 x ptr> %p, i32 8, <2 x i1> %mask, <2 x i32> %passthru)
+  ret <2 x i32> %ret
+}
+
+declare <2 x i32> @llvm.masked.gather.v2i32.v2p0(<2 x ptr>, i32, <2 x i1>, <2 x i32>)
diff --git a/llvm/test/Transforms/ScalarizeMaskedMemIntrin/AMDGPU/expand-masked-scatter.ll b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/AMDGPU/expand-masked-scatter.ll
new file mode 100644
index 000000000000000..45debf35d06e4f3
--- /dev/null
+++ b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/AMDGPU/expand-masked-scatter.ll
@@ -0,0 +1,36 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S %s -passes=scalarize-masked-mem-intrin -mtriple=amdgcn-amd-amdhsa | FileCheck %s
+
+; COM: Test that, unlike on CPU targets, the mask doesn't get bitcast to a scalar,
+; COM: since, on GPUs, each i1 takes up at least one register and so they should
+; COM: be treated separately.
+
+define void @scalarize_v2i32(<2 x ptr> %p, <2 x i1> %mask, <2 x i32> %value) {
+; CHECK-LABEL: define void @scalarize_v2i32(
+; CHECK-SAME: <2 x ptr> [[P:%.*]], <2 x i1> [[MASK:%.*]], <2 x i32> [[VALUE:%.*]]) {
+; CHECK-NEXT:    [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK]] to i2
+; CHECK-NEXT:    [[TMP1:%.*]] = and i2 [[SCALAR_MASK]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i2 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[TMP2]], label %[[COND_STORE:.*]], label %[[ELSE:.*]]
+; CHECK:       [[COND_STORE]]:
+; CHECK-NEXT:    [[ELT0:%.*]] = extractelement <2 x i32> [[VALUE]], i64 0
+; CHECK-NEXT:    [[PTR0:%.*]] = extractelement <2 x ptr> [[P]], i64 0
+; CHECK-NEXT:    store i32 [[ELT0]], ptr [[PTR0]], align 8
+; CHECK-NEXT:    br label %[[ELSE]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = and i2 [[SCALAR_MASK]], -2
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne i2 [[TMP3]], 0
+; CHECK-NEXT:    br i1 [[TMP4]], label %[[COND_STORE1:.*]], label %[[ELSE2:.*]]
+; CHECK:       [[COND_STORE1]]:
+; CHECK-NEXT:    [[ELT1:%.*]] = extractelement <2 x i32> [[VALUE]], i64 1
+; CHECK-NEXT:    [[PTR1:%.*]] = extractelement <2 x ptr> [[P]], i64 1
+; CHECK-NEXT:    store i32 [[ELT1]], ptr [[PTR1]], align 8
+; CHECK-NEXT:    br label %[[ELSE2]]
+; CHECK:       [[ELSE2]]:
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> %value, <2 x ptr> %p, i32 8, <2 x i1> %mask)
+  ret void
+}
+
+declare void @llvm.masked.scatter.v2i32.v2p0(<2 x i32>, <2 x ptr>, i32, <2 x i1>)
diff --git a/llvm/test/Transforms/ScalarizeMaskedMemIntrin/AMDGPU/expand-masked-store.ll b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/AMDGPU/expand-masked-store.ll
new file mode 100644
index 000000000000000..1efd008b77e1c06
--- /dev/null
+++ b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/AMDGPU/expand-masked-store.ll
@@ -0,0 +1,161 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S %s -passes=scalarize-masked-mem-intrin -mtriple=amdgcn-amd-amdhsa | FileCheck %s
+
+; COM: Test that, unlike on CPU targets, the mask doesn't get bitcast to a scalar,
+; COM: since, on GPUs, each i1 takes up at least one register and so they should
+; COM: be treated separately.
+
+define void @scalarize_v2i32(ptr %p, <2 x i1> %mask, <2 x i32> %data) {
+; CHECK-LABEL: define void @scalarize_v2i32(
+; CHECK-SAME: ptr [[P:%.*]], <2 x i1> [[MASK:%.*]], <2 x i32> [[DATA:%.*]]) {
+; CHECK-NEXT:    [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK]] to i2
+; CHECK-NEXT:    [[TMP1:%.*]] = and i2 [[SCALAR_MASK]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i2 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[TMP2]], label %[[COND_STORE:.*]], label %[[ELSE:.*]]
+; CHECK:       [[COND_STORE]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x i32> [[DATA]], i64 0
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 0
+; CHECK-NEXT:    store i32 [[TMP3]], ptr [[TMP4]], align 4
+; CHECK-NEXT:    br label %[[ELSE]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[TMP5:%.*]] = and i2 [[SCALAR_MASK]], -2
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne i2 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[COND_STORE1:.*]], label %[[ELSE2:.*]]
+; CHECK:       [[COND_STORE1]]:
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i32> [[DATA]], i64 1
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 1
+; CHECK-NEXT:    store i32 [[TMP7]], ptr [[TMP8]], align 4
+; CHECK-NEXT:    br label %[[ELSE2]]
+; CHECK:       [[ELSE2]]:
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.masked.store.v2i32.p0(<2 x i32> %data, ptr %p, i32 128, <2 x i1> %mask)
+  ret void
+}
+
+define void @scalarize_v2i32_splat_mask(ptr %p, <2 x i32> %data, i1 %mask) {
+; CHECK-LABEL: define void @scalarize_v2i32_splat_mask(
+; CHECK-SAME: ptr [[P:%.*]], <2 x i32> [[DATA:%.*]], i1 [[MASK:%.*]]) {
+; CHECK-NEXT:    [[MASK_VEC:%.*]] = insertelement <2 x i1> poison, i1 [[MASK]], i32 0
+; CHECK-NEXT:    [[MASK_SPLAT:%.*]] = shufflevector <2 x i1> [[MASK_VEC]], <2 x i1> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[MASK_SPLAT_FIRST:%.*]] = extractelement <2 x i1> [[MASK_SPLAT]], i64 0
+; CHECK-NEXT:    br i1 [[MASK_SPLAT_FIRST]], label %[[COND_STORE:.*]], label %[[BB1:.*]]
+; CHECK:       [[COND_STORE]]:
+; CHECK-NEXT:    store <2 x i32> [[DATA]], ptr [[P]], align 8
+; CHECK-NEXT:    br label %[[BB1]]
+; CHECK:       [[BB1]]:
+; CHECK-NEXT:    ret void
+;
+  %mask.vec = insertelement <2 x i1> poison, i1 %mask, i32 0
+  %mask.splat = shufflevector <2 x i1> %mask.vec, <2 x i1> poison, <2 x i32> zeroinitializer
+  call void @llvm.masked.store.v2i32.p0(<2 x i32> %data, ptr %p, i32 8, <2 x i1> %mask.splat)
+  ret void
+}
+
+define void @scalarize_v2f16(ptr %p, <2 x i1> %mask, <2 x half> %data) {
+; CHECK-LABEL: define void @scalarize_v2f16(
+; CHECK-SAME: ptr [[P:%.*]], <2 x i1> [[MASK:%.*]], <2 x half> [[DATA:%.*]]) {
+; CHECK-NEXT:    [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK]] to i2
+; CHECK-NEXT:    [[TMP1:%.*]] = and i2 [[SCALAR_MASK]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i2 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[TMP2]], label %[[COND_STORE:.*]], label %[[ELSE:.*]]
+; CHECK:       [[COND_STORE]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x half> [[DATA]], i64 0
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds half, ptr [[P]], i32 0
+; CHECK-NEXT:    store half [[TMP3]], ptr [[TMP4]], align 2
+; CHECK-NEXT:    br label %[[ELSE]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[TMP5:%.*]] = and i2 [[SCALAR_MASK]], -2
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne i2 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[COND_STORE1:.*]], label %[[ELSE2:.*]]
+; CHECK:       [[COND_STORE1]]:
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x half> [[DATA]], i64 1
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds half, ptr [[P]], i32 1
+; CHECK-NEXT:    store half [[TMP7]], ptr [[TMP8]], align 2
+; CHECK-NEXT:    br label %[[ELSE2]]
+; CHECK:       [[ELSE2]]:
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.masked.store.v2f16.p0(<2 x half> %data, ptr %p, i32 128, <2 x i1> %mask)
+  ret void
+}
+
+define void @scalarize_v2i32_p3(ptr addrspace(3) %p, <2 x i1> %mask, <2 x i32> %data) {
+; CHECK-LABEL: define void @scalarize_v2i32_p3(
+; CHECK-SAME: ptr addrspace(3) [[P:%.*]], <2 x i1> [[MASK:%.*]], <2 x i32> [[DATA:%.*]]) {
+; CHECK-NEXT:    [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK]] to i2
+; CHECK-NEXT:    [[TMP1:%.*]] = and i2 [[SCALAR_MASK]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i2 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[TMP2]], label %[[COND_STORE:.*]], label %[[ELSE:.*]]
+; CHECK:       [[COND_STORE]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x i32> [[DATA]], i64 0
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, ptr addrspace(3) [[P]], i32 0
+; CHECK-NEXT:    store i32 [[TMP3]], ptr addrspace(3) [[TMP4]], align 4
+; CHECK-NEXT:    br label %[[ELSE]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[TMP5:%.*]] = and i2 [[SCALAR_MASK]], -2
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne i2 [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[COND_STORE1:.*]], label %[[ELSE2:.*]]
+; CHECK:       [[COND_STORE1]]:
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i32> [[DATA]], i64 1
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr addrspace(3) [[P]], i32 1
+; CHECK-NEXT:    store i32 [[TMP7]], ptr addrspace(3) [[TMP8]], align 4
+; CHECK-NEXT:    br label %[[ELSE2]]
+; CHECK:       [[ELSE2]]:
+; CHECK-NEXT:    ret void
+;
+  call void @llvm.masked.store.v2i32.p3(<2 x i32> %data, ptr addrspace(3) %p, i32 128, <2 x i1> %mask)
+  ret void
+}
+
+define void @scalarize_v2i32_lane_mask(ptr %p, <2 x i32> %data) {
+; CHECK-LABEL: define void @scalarize_v2i32_lane_mask(
+; CHECK-SAME: ptr [[P:%.*]], <2 x i32> [[DATA:%.*]]) {
+; CHECK-NEXT:    [[ITEM_ID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT:    [[MASK:%.*]] = icmp ult i32 [[ITEM_ID]], 16
+; CHECK-NEXT:    [[MASK_VEC:%.*]] = insertelement <2 x i1> poison, i1 [[MASK]], i32 0
+; CHECK-NEXT:    [[MASK_SPLAT:%.*]] = shufflevector <2 x i1> [[MASK_VEC]], <2 x i1> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[MASK_SPLAT_FIRST:%.*]] = extractelement <2 x i1> [[MASK_SPLAT]], i64 0
+; CHECK-NEXT:    br i1 [[MASK_SPLAT_FIRST]], label %[[COND_STORE:.*]], label %[[BB1:.*]]
+; CHECK:       [[COND_STORE]]:
+; CHECK-NEXT:    store <2 x i32> [[DATA]], ptr [[P]], align 8
+; CHECK-NEXT:    br label %[[BB1]]
+; CHECK:       [[BB1]]:
+; CHECK-NEXT:    ret void
+;
+  %item.id = call i32 @llvm.amdgcn.workitem.id.x()
+  %mask = icmp ult i32 %item.id, 16
+  %mask.vec = insertelement <2 x i1> poison, i1 %mask, i32 0
+  %mask.splat = shufflevector <2 x i1> %mask.vec, <2 x i1> poison, <2 x i32> zeroinitializer
+  call void @llvm.masked.store.v2i32.p0(<2 x i32> %data, ptr %p, i32 8, <2 x i1> %mask.splat)
+  ret void
+}
+
+define void @scalarize_v2i32_group_mask(ptr %p, <2 x i32> %data) {
+; CHECK-LABEL: define void @scalarize_v2i32_group_mask(
+; CHECK-SAME: ptr [[P:%.*]], <2 x i32> [[DATA:%.*]]) {
+; CHECK-NEXT:    [[ITEM_ID:%.*]] = call i32 @llvm.amdgcn.workgroup.id.x()
+; CHECK-NEXT:    [[MASK:%.*]] = icmp ult i32 [[ITEM_ID]], 4
+; CHECK-NEXT:    [[MASK_VEC:%.*]] = insertelement <2 x i1> poison, i1 [[MASK]], i32 0
+; CHECK-NEXT:    [[MASK_SPLAT:%.*]] = shufflevector <2 x i1> [[MASK_VEC]], <2 x i1> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[MASK_SPLAT_FIRST:%.*]] = extractelement <2 x i1> [[MASK_SPLAT]], i64 0
+; CHECK-NEXT:    br i1 [[MASK_SPLAT_FIRST]], label %[[COND_STORE:.*]], label %[[BB1:.*]]
+; CHECK:       [[COND_STORE]]:
+; CHECK-NEXT:    store <2 x i32> [[DATA]], ptr [[P]], align 8
+; CHECK-NEXT:    br label %[[BB1]]
+; CHECK:       [[BB1]]:
+; CHECK-NEXT:    ret void
+;
+  %group.id = call i32 @llvm.amdgcn.workgroup.id.x()
+  %mask = icmp ult i32 %group.id, 4
+  %mask.vec = insertelement <2 x i1> poison, i1 %mask, i32 0
+  %mask.splat = shufflevector <2 x i1> %mask.vec, <2 x i1> poison, <2 x i32> zeroinitializer
+  call void @llvm.masked.store.v2i32.p0(<2 x i32> %data, ptr %p, i32 8, <2 x i1> %mask.splat)
+  ret void
+}
+
+declare void @llvm.masked.store.v2i32.p0(<2 x i32>, ptr, i32, <2 x i1>)
+declare void @llvm.masked.store.v2f16.p0(<2 x half>, ptr, i32, <2 x i1>)
+declare void @llvm.masked.store.v2i32.p3(<2 x i32>, ptr addrspace(3), i32, <2 x i1>)
+declare noundef i32 @llvm.amdgcn.workitem.id.x()
+declare noundef i32 @llvm.amdgcn.workgroup.id.x()
diff --git a/llvm/test/Transforms/ScalarizeMaskedMemIntrin/AMDGPU/lit.local.cfg b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/AMDGPU/lit.local.cfg
new file mode 100644
index 000000000000000..7c492428aec7614
--- /dev/null
+++ b/llvm/test/Transforms/ScalarizeMaskedMemIntrin/AMDGPU/lit.local.cfg
@@ -0,0 +1,2 @@
+if not "AMDGPU" in config.root.targets:
+    config.unsupported = True

From 22825ddd88aa29994a8a260a31c959aed0b35b5b Mon Sep 17 00:00:00 2001
From: Scott Linder <Scott.Linder@amd.com>
Date: Tue, 13 Aug 2024 22:53:19 +0000
Subject: [PATCH 411/441] [AMDGPU][Docs] DWARF aspace-aware base types

Propose an extension to base type DIEs such that DW_ATE_address-encoded
base types can include an architecture specific address space. Use this
to implement DW_OP_convert conversions between AMDGPU address space
addresses where meaningful.
---
 ...arfExtensionsForHeterogeneousDebugging.rst |  6 ++++
 llvm/docs/AMDGPUUsage.rst                     | 28 +++++++++++++++++++
 2 files changed, 34 insertions(+)

diff --git a/llvm/docs/AMDGPUDwarfExtensionsForHeterogeneousDebugging.rst b/llvm/docs/AMDGPUDwarfExtensionsForHeterogeneousDebugging.rst
index 7d4e06ebf0493c4..b71e80f922250b9 100644
--- a/llvm/docs/AMDGPUDwarfExtensionsForHeterogeneousDebugging.rst
+++ b/llvm/docs/AMDGPUDwarfExtensionsForHeterogeneousDebugging.rst
@@ -3804,6 +3804,12 @@ A.5.1 Base Type Entries
       entry, or allow some forms of ``DW_TAG_array_type`` (those that have the
       ``DW_AT_GNU_vector`` attribute) to be used as stack entry value types?
 
+2. A ``DW_TAG_base_type`` debugger information entry with the encoding
+   ``DW_ATE_address`` may have a ``DW_AT_LLVM_address_space`` attribute whose
+   value is an architecture specific address space (see
+   :ref:`amdgpu-dwarf-address-spaces`). If ommitted it defaults to
+   ``DW_ASPACE_LLVM_none``.
+
 .. _amdgpu-dwarf-type-modifier-entries:
 
 A.5.3 Type Modifier Entries
diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst
index 97fca32d4ece66b..80140734cbefd64 100644
--- a/llvm/docs/AMDGPUUsage.rst
+++ b/llvm/docs/AMDGPUUsage.rst
@@ -2918,6 +2918,34 @@ controlled by the execution mask. An undefined location description together
 with ``DW_OP_LLVM_extend`` is used to indicate the lane was not active on entry
 to the subprogram. See :ref:`amdgpu-dwarf-dw-at-llvm-lane-pc` for an example.
 
+.. _amdgpu-dwarf-base-type-conversions:
+
+Base Type Conversions
+---------------------
+
+For AMDGPU expressions, ``DW_OP_convert`` may be used to convert between
+``DW_ATE_address``-encoded base types in different address spaces.
+
+Conversions are defined as in :ref:`amdgpu-address-spaces` when all relevant
+conditions described there are met, and otherwise result in an evaluation
+error.
+
+.. note::
+
+  For a target which does not support a particular address space, converting to
+  or from that address space is always an evaluation error.
+
+  For targets which support the generic address space, converting from
+  ``DW_ASPACE_AMDGPU_generic`` to ``DW_ASPACE_LLVM_none`` is always defined and
+  requires no change to the literal value of the address.
+
+  Converting from ``DW_ASPACE_AMDGPU_generic`` to any of
+  ``DW_ASPACE_AMDGPU_local``, ``DW_ASPACE_AMDGPU_private_wave`` or
+  ``DW_ASPACE_AMDGPU_private_lane`` is defined when the relevant hardware
+  support is present and setup has been completed. Conversion to
+  ``DW_ASPACE_AMDGPU_private_lane`` additionally requires the context to
+  include the active lane.
+
 Debugger Information Entry Attributes
 -------------------------------------
 

From 7022498ac2f236e411e8a0f9a48669e754000a4b Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Tue, 20 Aug 2024 00:10:45 +0400
Subject: [PATCH 412/441] AMDGPU/NewPM: Start implementing addCodeGenPrepare
 (#102816)

---
 llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 11 +++++++++++
 llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h   |  3 +++
 2 files changed, 14 insertions(+)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index f15d02ac9301ef1..04b96a32a8970ae 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -73,6 +73,7 @@
 #include "llvm/Transforms/Utils.h"
 #include "llvm/Transforms/Utils/FixIrreducible.h"
 #include "llvm/Transforms/Utils/LCSSA.h"
+#include "llvm/Transforms/Utils/LowerSwitch.h"
 #include "llvm/Transforms/Utils/SimplifyLibCalls.h"
 #include "llvm/Transforms/Utils/UnifyLoopExits.h"
 #include "llvm/Transforms/Vectorize/LoadStoreVectorizer.h"
@@ -1768,6 +1769,16 @@ AMDGPUCodeGenPassBuilder::AMDGPUCodeGenPassBuilder(
               ShadowStackGCLoweringPass>();
 }
 
+void AMDGPUCodeGenPassBuilder::addCodeGenPrepare(AddIRPass &addPass) const {
+  Base::addCodeGenPrepare(addPass);
+
+  // LowerSwitch pass may introduce unreachable blocks that can cause unexpected
+  // behavior for subsequent passes. Placing it here seems better that these
+  // blocks would get cleaned up by UnreachableBlockElim inserted next in the
+  // pass flow.
+  addPass(LowerSwitchPass());
+}
+
 void AMDGPUCodeGenPassBuilder::addPreISel(AddIRPass &addPass) const {
   const bool LateCFGStructurize = AMDGPUTargetMachine::EnableLateStructurizeCFG;
   const bool DisableStructurizer = AMDGPUTargetMachine::DisableStructurizer;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h
index 4dffddd2d44ccf2..576bf40e3328d57 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h
@@ -165,11 +165,14 @@ class AMDGPUPassConfig : public TargetPassConfig {
 
 class AMDGPUCodeGenPassBuilder
     : public CodeGenPassBuilder<AMDGPUCodeGenPassBuilder, GCNTargetMachine> {
+  using Base = CodeGenPassBuilder<AMDGPUCodeGenPassBuilder, GCNTargetMachine>;
+
 public:
   AMDGPUCodeGenPassBuilder(GCNTargetMachine &TM,
                            const CGPassBuilderOption &Opts,
                            PassInstrumentationCallbacks *PIC);
 
+  void addCodeGenPrepare(AddIRPass &) const;
   void addPreISel(AddIRPass &addPass) const;
   void addAsmPrinter(AddMachinePass &, CreateMCStreamer) const;
   Error addInstSelector(AddMachinePass &) const;

From 435cb0dc5eca08cdd8d9ed0d887fa1693cc2bf33 Mon Sep 17 00:00:00 2001
From: Aaron Ballman <aaron@aaronballman.com>
Date: Mon, 19 Aug 2024 16:54:12 -0400
Subject: [PATCH 413/441] [C++23] Fix infinite recursion (Clang 19.x
 regression) (#104829)

d469794d0cdfd2fea50a6ce0c0e33abb242d744c was fixing an issue with
triggering vtable instantiations, but it accidentally introduced
infinite recursion when the type to be checked is the same as the type
used in a base specifier or field declaration.

Fixes #104802
---
 clang/lib/Sema/SemaDeclCXX.cpp  |  9 +++++++--
 clang/test/SemaCXX/gh102293.cpp | 27 ++++++++++++++++++++++++++-
 2 files changed, 33 insertions(+), 3 deletions(-)

diff --git a/clang/lib/Sema/SemaDeclCXX.cpp b/clang/lib/Sema/SemaDeclCXX.cpp
index e05595e565d54a4..d89a47f3e6226a8 100644
--- a/clang/lib/Sema/SemaDeclCXX.cpp
+++ b/clang/lib/Sema/SemaDeclCXX.cpp
@@ -7056,11 +7056,16 @@ void Sema::CheckCompletedCXXClass(Scope *S, CXXRecordDecl *Record) {
         if (!RD->hasConstexprDestructor())
           return false;
 
+        QualType CanUnqualT = T.getCanonicalType().getUnqualifiedType();
         for (const CXXBaseSpecifier &B : RD->bases())
-          if (!Check(B.getType(), Check))
+          if (B.getType().getCanonicalType().getUnqualifiedType() !=
+                  CanUnqualT &&
+              !Check(B.getType(), Check))
             return false;
         for (const FieldDecl *FD : RD->fields())
-          if (!Check(FD->getType(), Check))
+          if (FD->getType().getCanonicalType().getUnqualifiedType() !=
+                  CanUnqualT &&
+              !Check(FD->getType(), Check))
             return false;
         return true;
       };
diff --git a/clang/test/SemaCXX/gh102293.cpp b/clang/test/SemaCXX/gh102293.cpp
index 30629fc03bf6a9a..d4218cc13dcecdc 100644
--- a/clang/test/SemaCXX/gh102293.cpp
+++ b/clang/test/SemaCXX/gh102293.cpp
@@ -1,5 +1,4 @@
 // RUN: %clang_cc1 -std=c++23 -fsyntax-only -verify %s
-// expected-no-diagnostics
 
 template <typename T> static void destroy() {
     T t;
@@ -20,3 +19,29 @@ struct S : HasVT {
   HasD<> v;
 };
 
+// Ensure we don't get infinite recursion from the check, however. See GH104802
+namespace GH104802 {
+class foo {       // expected-note {{definition of 'GH104802::foo' is not complete until the closing '}'}}
+  foo a;          // expected-error {{field has incomplete type 'foo'}}
+
+  virtual int c();
+};
+
+class bar {       // expected-note {{definition of 'GH104802::bar' is not complete until the closing '}'}}
+  const bar a;    // expected-error {{field has incomplete type 'const bar'}}
+
+  virtual int c();
+};
+
+class baz {       // expected-note {{definition of 'GH104802::baz' is not complete until the closing '}'}}
+  typedef class baz blech;
+  blech a;        // expected-error {{field has incomplete type 'blech' (aka 'GH104802::baz')}}
+
+  virtual int c();
+};
+
+class quux : quux { // expected-error {{base class has incomplete type}} \
+                     expected-note {{definition of 'GH104802::quux' is not complete until the closing '}'}}
+  virtual int c();
+};
+}

From ff2e619dfcd77328812a42d2ba2b11c3ff96f410 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Mon, 19 Aug 2024 13:57:34 -0700
Subject: [PATCH 414/441] [MC] Remove duplicate getFixupKindInfo calls. NFC

Due to 8728e097dfbec3630a1dd907431c0f14274a1ae8
(`std::unique_ptr<MCAsmBackend> Backend`), the compiler doesn't know
that `Backend` will not be modified across function calls.
---
 llvm/lib/MC/MCAssembler.cpp | 15 ++++-----------
 1 file changed, 4 insertions(+), 11 deletions(-)

diff --git a/llvm/lib/MC/MCAssembler.cpp b/llvm/lib/MC/MCAssembler.cpp
index cbeb41f58c99be7..55139dae7b1d71f 100644
--- a/llvm/lib/MC/MCAssembler.cpp
+++ b/llvm/lib/MC/MCAssembler.cpp
@@ -167,18 +167,12 @@ bool MCAssembler::evaluateFixup(const MCFixup &Fixup, const MCFragment *DF,
     }
   }
 
-  assert(getBackendPtr() && "Expected assembler backend");
-  bool IsTarget = getBackendPtr()->getFixupKindInfo(Fixup.getKind()).Flags &
-                  MCFixupKindInfo::FKF_IsTarget;
-
-  if (IsTarget)
+  unsigned FixupFlags = getBackend().getFixupKindInfo(Fixup.getKind()).Flags;
+  if (FixupFlags & MCFixupKindInfo::FKF_IsTarget)
     return getBackend().evaluateTargetFixup(*this, Fixup, DF, Target, STI,
                                             Value, WasForced);
 
-  unsigned FixupFlags = getBackendPtr()->getFixupKindInfo(Fixup.getKind()).Flags;
-  bool IsPCRel = getBackendPtr()->getFixupKindInfo(Fixup.getKind()).Flags &
-                 MCFixupKindInfo::FKF_IsPCRel;
-
+  bool IsPCRel = FixupFlags & MCFixupKindInfo::FKF_IsPCRel;
   bool IsResolved = false;
   if (IsPCRel) {
     if (Target.getSymB()) {
@@ -213,8 +207,7 @@ bool MCAssembler::evaluateFixup(const MCFixup &Fixup, const MCFragment *DF,
       Value -= getSymbolOffset(Sym);
   }
 
-  bool ShouldAlignPC = getBackend().getFixupKindInfo(Fixup.getKind()).Flags &
-                       MCFixupKindInfo::FKF_IsAlignedDownTo32Bits;
+  bool ShouldAlignPC = FixupFlags & MCFixupKindInfo::FKF_IsAlignedDownTo32Bits;
   assert((ShouldAlignPC ? IsPCRel : true) &&
     "FKF_IsAlignedDownTo32Bits is only allowed on PC-relative fixups!");
 

From e732d1ce86783b1d7fe30645fcb30434109505b9 Mon Sep 17 00:00:00 2001
From: Yuxuan Chen <ych@fb.com>
Date: Mon, 19 Aug 2024 14:06:10 -0700
Subject: [PATCH 415/441] [Clang] Fix ICE in SemaOpenMP with structured binding
 (#104822)

Fixes https://github.com/llvm/llvm-project/issues/104810.

Clang currently crashes on the following program:
```
struct S {
  int i;
};

auto [a] = S{1};

void foo() {
    a;
}
```
when `-fopenmp` is enabled.

Because `a` is neither `VarDecl` nor `FieldDecl`. It's a `BindingDecl`
that's not handled in `SemaOpenMP.cpp`'s `getCanonicalDecl`. It appears
to me that this pattern matching is merely just for a refined return
type of the overrides. It can also be achieved with just using the
virtual `Decl::getCanonicalDecl()` instead. Do the final casting should
be safe for `ValueDecl`s.
---
 clang/lib/Sema/SemaOpenMP.cpp      | 12 ++----------
 clang/test/SemaOpenMP/gh104810.cpp | 12 ++++++++++++
 2 files changed, 14 insertions(+), 10 deletions(-)
 create mode 100644 clang/test/SemaOpenMP/gh104810.cpp

diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index c1442b8a58eda4f..74c646f64b42f20 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -1256,16 +1256,8 @@ static const ValueDecl *getCanonicalDecl(const ValueDecl *D) {
   if (const auto *CED = dyn_cast<OMPCapturedExprDecl>(D))
     if (const auto *ME = dyn_cast<MemberExpr>(getExprAsWritten(CED->getInit())))
       D = ME->getMemberDecl();
-  const auto *VD = dyn_cast<VarDecl>(D);
-  const auto *FD = dyn_cast<FieldDecl>(D);
-  if (VD != nullptr) {
-    VD = VD->getCanonicalDecl();
-    D = VD;
-  } else {
-    assert(FD);
-    FD = FD->getCanonicalDecl();
-    D = FD;
-  }
+
+  D = cast<ValueDecl>(D->getCanonicalDecl());
   return D;
 }
 
diff --git a/clang/test/SemaOpenMP/gh104810.cpp b/clang/test/SemaOpenMP/gh104810.cpp
new file mode 100644
index 000000000000000..926408931977604
--- /dev/null
+++ b/clang/test/SemaOpenMP/gh104810.cpp
@@ -0,0 +1,12 @@
+// RUN: %clang_cc1 -fopenmp -fsyntax-only %s
+
+// expected-no-diagnostics
+struct S {
+  int i;
+};
+
+auto [a] = S{1};
+
+void foo() {
+    a;
+}

From 8a677c194072dc19ef680bebba9aa7eafb05b373 Mon Sep 17 00:00:00 2001
From: Rahul Joshi <rjoshi@nvidia.com>
Date: Mon, 19 Aug 2024 14:09:38 -0700
Subject: [PATCH 416/441] [NFC][ADT] Add range wrapper for std::mismatch
 (#104838)

---
 llvm/include/llvm/ADT/STLExtras.h | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/llvm/include/llvm/ADT/STLExtras.h b/llvm/include/llvm/ADT/STLExtras.h
index 249ac384bec41e6..b2df0cd650ad7b3 100644
--- a/llvm/include/llvm/ADT/STLExtras.h
+++ b/llvm/include/llvm/ADT/STLExtras.h
@@ -2002,6 +2002,20 @@ template <typename R, typename Compare> auto max_element(R &&Range, Compare C) {
   return std::max_element(adl_begin(Range), adl_end(Range), C);
 }
 
+/// Provide wrappers to std::mismatch which take ranges instead of having to
+/// pass begin/end explicitly.
+/// This function returns a pair of iterators for the first mismatching elements
+/// from `R1` and `R2`. As an example, if:
+///
+/// R1 = [0, 1, 4, 6], R2 = [0, 1, 5, 6]
+///
+/// this function will return a pair of iterators, first pointing to R1[2] and
+/// second pointing to R2[2].
+template <typename R1, typename R2> auto mismatch(R1 &&Range1, R2 &&Range2) {
+  return std::mismatch(adl_begin(Range1), adl_end(Range1), adl_begin(Range2),
+                       adl_end(Range2));
+}
+
 template <typename R>
 void stable_sort(R &&Range) {
   std::stable_sort(adl_begin(Range), adl_end(Range));

From 6a125c7e77cd0e73abc9e60ad1c8eb06e31b11d7 Mon Sep 17 00:00:00 2001
From: Vitaly Buka <vitalybuka@google.com>
Date: Mon, 19 Aug 2024 14:41:15 -0700
Subject: [PATCH 417/441] [asan] Better `___asan_gen_` names (#104728)

Use different suffixes for each of 3 types
of objects using `___asan_gen_`.
---
 .../Instrumentation/AddressSanitizer.cpp       | 18 +++++++++++-------
 .../AddressSanitizer/asan-funclet.ll           | 18 +++++++++---------
 .../AddressSanitizer/fake-stack.ll             |  8 ++++----
 .../AddressSanitizer/global_metadata.ll        |  2 +-
 .../AddressSanitizer/global_metadata_array.ll  |  2 +-
 .../AddressSanitizer/global_with_comdat.ll     |  4 ++--
 .../AddressSanitizer/instrument_global.ll      |  2 +-
 .../instrument_late_initializer.ll             |  2 +-
 .../AddressSanitizer/local_alias.ll            |  4 ++--
 .../AddressSanitizer/odr-check-ignore.ll       | 16 ++++++++--------
 .../AddressSanitizer/win-string-literal.ll     |  2 +-
 11 files changed, 41 insertions(+), 37 deletions(-)

diff --git a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
index cda35c5af84fff4..2655f49c48e3110 100644
--- a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
@@ -642,6 +642,10 @@ static uint64_t GetCtorAndDtorPriority(Triple &TargetTriple) {
   }
 }
 
+static Twine genName(StringRef suffix) {
+  return Twine(kAsanGenPrefix) + suffix;
+}
+
 namespace {
 /// Helper RAII class to post-process inserted asan runtime calls during a
 /// pass on a single Function. Upon end of scope, detects and applies the
@@ -2243,7 +2247,7 @@ void ModuleAddressSanitizer::SetComdatForGlobalMetadata(
       // If G is unnamed, it must be internal. Give it an artificial name
       // so we can put it in a comdat.
       assert(G->hasLocalLinkage());
-      G->setName(Twine(kAsanGenPrefix) + "_anon_global");
+      G->setName(genName("anon_global"));
     }
 
     if (!InternalSuffix.empty() && G->hasLocalLinkage()) {
@@ -2538,10 +2542,10 @@ void ModuleAddressSanitizer::instrumentGlobals(IRBuilder<> &IRB,
   // We shouldn't merge same module names, as this string serves as unique
   // module ID in runtime.
   GlobalVariable *ModuleName =
-      n != 0
-          ? createPrivateGlobalForString(M, M.getModuleIdentifier(),
-                                         /*AllowMerging*/ false, kAsanGenPrefix)
-          : nullptr;
+      n != 0 ? createPrivateGlobalForString(M, M.getModuleIdentifier(),
+                                            /*AllowMerging*/ false,
+                                            genName("module"))
+             : nullptr;
 
   for (size_t i = 0; i < n; i++) {
     GlobalVariable *G = GlobalsToChange[i];
@@ -2556,7 +2560,7 @@ void ModuleAddressSanitizer::instrumentGlobals(IRBuilder<> &IRB,
     std::string NameForGlobal = G->getName().str();
     GlobalVariable *Name =
         createPrivateGlobalForString(M, llvm::demangle(NameForGlobal),
-                                     /*AllowMerging*/ true, kAsanGenPrefix);
+                                     /*AllowMerging*/ true, genName("global"));
 
     Type *Ty = G->getValueType();
     const uint64_t SizeInBytes = DL.getTypeAllocSize(Ty);
@@ -3583,7 +3587,7 @@ void FunctionStackPoisoner::processStaticAllocas() {
       IntptrPtrTy);
   GlobalVariable *StackDescriptionGlobal =
       createPrivateGlobalForString(*F.getParent(), DescriptionString,
-                                   /*AllowMerging*/ true, kAsanGenPrefix);
+                                   /*AllowMerging*/ true, genName("stack"));
   Value *Description = IRB.CreatePointerCast(StackDescriptionGlobal, IntptrTy);
   IRB.CreateStore(Description, BasePlus1);
   // Write the PC to redzone[2].
diff --git a/llvm/test/Instrumentation/AddressSanitizer/asan-funclet.ll b/llvm/test/Instrumentation/AddressSanitizer/asan-funclet.ll
index 62c0a24c60d77f1..e986f1085426887 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/asan-funclet.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/asan-funclet.ll
@@ -62,7 +62,7 @@ define void @FuncletPersonality(ptr %ptrParam) sanitize_address personality ptr
 ; CHECK-INLINE-NEXT:    store i64 1102416563, ptr [[TMP23]], align 8
 ; CHECK-INLINE-NEXT:    [[TMP24:%.*]] = add i64 [[TMP12]], 8
 ; CHECK-INLINE-NEXT:    [[TMP25:%.*]] = inttoptr i64 [[TMP24]] to ptr
-; CHECK-INLINE-NEXT:    store i64 ptrtoint (ptr @___asan_gen_ to i64), ptr [[TMP25]], align 8
+; CHECK-INLINE-NEXT:    store i64 ptrtoint (ptr @___asan_gen_stack to i64), ptr [[TMP25]], align 8
 ; CHECK-INLINE-NEXT:    [[TMP26:%.*]] = add i64 [[TMP12]], 16
 ; CHECK-INLINE-NEXT:    [[TMP27:%.*]] = inttoptr i64 [[TMP26]] to ptr
 ; CHECK-INLINE-NEXT:    store i64 ptrtoint (ptr @FuncletPersonality to i64), ptr [[TMP27]], align 8
@@ -95,7 +95,7 @@ define void @FuncletPersonality(ptr %ptrParam) sanitize_address personality ptr
 ; CHECK-INLINE-NEXT:    [[TMP42:%.*]] = inttoptr i64 [[TMP41]] to ptr
 ; CHECK-INLINE-NEXT:    [[TMP43:%.*]] = load i8, ptr [[TMP42]], align 1
 ; CHECK-INLINE-NEXT:    [[TMP44:%.*]] = icmp ne i8 [[TMP43]], 0
-; CHECK-INLINE-NEXT:    br i1 [[TMP44]], label [[TMP45:%.*]], label [[TMP50:%.*]], !prof [[PROF0:![0-9]+]]
+; CHECK-INLINE-NEXT:    br i1 [[TMP44]], label [[TMP45:%.*]], label [[TMP50:%.*]], !prof [[PROF1:![0-9]+]]
 ; CHECK-INLINE:       45:
 ; CHECK-INLINE-NEXT:    [[TMP46:%.*]] = and i64 [[TMP21]], 7
 ; CHECK-INLINE-NEXT:    [[TMP47:%.*]] = trunc i64 [[TMP46]] to i8
@@ -153,7 +153,7 @@ define void @FuncletPersonality(ptr %ptrParam) sanitize_address personality ptr
 ; CHECK-INLINE-NEXT:    [[TMP82:%.*]] = inttoptr i64 [[TMP81]] to ptr
 ; CHECK-INLINE-NEXT:    [[TMP83:%.*]] = load i8, ptr [[TMP82]], align 1
 ; CHECK-INLINE-NEXT:    [[TMP84:%.*]] = icmp ne i8 [[TMP83]], 0
-; CHECK-INLINE-NEXT:    br i1 [[TMP84]], label [[TMP85:%.*]], label [[TMP90:%.*]], !prof [[PROF0]]
+; CHECK-INLINE-NEXT:    br i1 [[TMP84]], label [[TMP85:%.*]], label [[TMP90:%.*]], !prof [[PROF1]]
 ; CHECK-INLINE:       85:
 ; CHECK-INLINE-NEXT:    [[TMP86:%.*]] = and i64 [[TMP77]], 7
 ; CHECK-INLINE-NEXT:    [[TMP87:%.*]] = trunc i64 [[TMP86]] to i8
@@ -193,7 +193,7 @@ define void @FuncletPersonality(ptr %ptrParam) sanitize_address personality ptr
 ; CHECK-INLINE-NEXT:    [[TMP101:%.*]] = inttoptr i64 [[TMP100]] to ptr
 ; CHECK-INLINE-NEXT:    [[TMP102:%.*]] = load i8, ptr [[TMP101]], align 1
 ; CHECK-INLINE-NEXT:    [[TMP103:%.*]] = icmp ne i8 [[TMP102]], 0
-; CHECK-INLINE-NEXT:    br i1 [[TMP103]], label [[TMP104:%.*]], label [[TMP109:%.*]], !prof [[PROF0]]
+; CHECK-INLINE-NEXT:    br i1 [[TMP103]], label [[TMP104:%.*]], label [[TMP109:%.*]], !prof [[PROF1]]
 ; CHECK-INLINE:       104:
 ; CHECK-INLINE-NEXT:    [[TMP105:%.*]] = and i64 [[TMP54]], 7
 ; CHECK-INLINE-NEXT:    [[TMP106:%.*]] = trunc i64 [[TMP105]] to i8
@@ -221,7 +221,7 @@ define void @FuncletPersonality(ptr %ptrParam) sanitize_address personality ptr
 ; CHECK-INLINE-NEXT:    [[TMP119:%.*]] = inttoptr i64 [[TMP118]] to ptr
 ; CHECK-INLINE-NEXT:    [[TMP120:%.*]] = load i8, ptr [[TMP119]], align 1
 ; CHECK-INLINE-NEXT:    [[TMP121:%.*]] = icmp ne i8 [[TMP120]], 0
-; CHECK-INLINE-NEXT:    br i1 [[TMP121]], label [[TMP122:%.*]], label [[TMP127:%.*]], !prof [[PROF0]]
+; CHECK-INLINE-NEXT:    br i1 [[TMP121]], label [[TMP122:%.*]], label [[TMP127:%.*]], !prof [[PROF1]]
 ; CHECK-INLINE:       122:
 ; CHECK-INLINE-NEXT:    [[TMP123:%.*]] = and i64 [[TMP116]], 7
 ; CHECK-INLINE-NEXT:    [[TMP124:%.*]] = trunc i64 [[TMP123]] to i8
@@ -236,7 +236,7 @@ define void @FuncletPersonality(ptr %ptrParam) sanitize_address personality ptr
 ; CHECK-INLINE-NEXT:    [[TMP130:%.*]] = inttoptr i64 [[TMP129]] to ptr
 ; CHECK-INLINE-NEXT:    [[TMP131:%.*]] = load i8, ptr [[TMP130]], align 1
 ; CHECK-INLINE-NEXT:    [[TMP132:%.*]] = icmp ne i8 [[TMP131]], 0
-; CHECK-INLINE-NEXT:    br i1 [[TMP132]], label [[TMP133:%.*]], label [[EHEXIT:%.*]], !prof [[PROF0]]
+; CHECK-INLINE-NEXT:    br i1 [[TMP132]], label [[TMP133:%.*]], label [[EHEXIT:%.*]], !prof [[PROF1]]
 ; CHECK-INLINE:       133:
 ; CHECK-INLINE-NEXT:    [[TMP134:%.*]] = and i64 [[TMP114]], 7
 ; CHECK-INLINE-NEXT:    [[TMP135:%.*]] = trunc i64 [[TMP134]] to i8
@@ -308,7 +308,7 @@ define void @FuncletPersonality(ptr %ptrParam) sanitize_address personality ptr
 ; CHECK-OUTLINE-NEXT:    store i64 1102416563, ptr [[TMP27]], align 8
 ; CHECK-OUTLINE-NEXT:    [[TMP28:%.*]] = add i64 [[TMP12]], 8
 ; CHECK-OUTLINE-NEXT:    [[TMP29:%.*]] = inttoptr i64 [[TMP28]] to ptr
-; CHECK-OUTLINE-NEXT:    store i64 ptrtoint (ptr @___asan_gen_ to i64), ptr [[TMP29]], align 8
+; CHECK-OUTLINE-NEXT:    store i64 ptrtoint (ptr @___asan_gen_stack to i64), ptr [[TMP29]], align 8
 ; CHECK-OUTLINE-NEXT:    [[TMP30:%.*]] = add i64 [[TMP12]], 16
 ; CHECK-OUTLINE-NEXT:    [[TMP31:%.*]] = inttoptr i64 [[TMP30]] to ptr
 ; CHECK-OUTLINE-NEXT:    store i64 ptrtoint (ptr @FuncletPersonality to i64), ptr [[TMP31]], align 8
@@ -524,7 +524,7 @@ define void @OtherPersonality(ptr %ptrParam) sanitize_address personality ptr @d
 ; CHECK-NEXT:    store i64 1102416563, ptr [[TMP14]], align 8
 ; CHECK-NEXT:    [[TMP15:%.*]] = add i64 [[TMP11]], 8
 ; CHECK-NEXT:    [[TMP16:%.*]] = inttoptr i64 [[TMP15]] to ptr
-; CHECK-NEXT:    store i64 ptrtoint (ptr @___asan_gen_.1 to i64), ptr [[TMP16]], align 8
+; CHECK-NEXT:    store i64 ptrtoint (ptr @___asan_gen_stack.1 to i64), ptr [[TMP16]], align 8
 ; CHECK-NEXT:    [[TMP17:%.*]] = add i64 [[TMP11]], 16
 ; CHECK-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
 ; CHECK-NEXT:    store i64 ptrtoint (ptr @OtherPersonality to i64), ptr [[TMP18]], align 8
@@ -580,5 +580,5 @@ ehcleanup:                                        ; preds = %entry
   cleanupret from %0 unwind to caller
 }
 ;.
-; CHECK-INLINE: [[PROF0]] = !{!"branch_weights", i32 1, i32 1048575}
+; CHECK-INLINE: [[PROF1]] = !{!"branch_weights", i32 1, i32 1048575}
 ;.
diff --git a/llvm/test/Instrumentation/AddressSanitizer/fake-stack.ll b/llvm/test/Instrumentation/AddressSanitizer/fake-stack.ll
index ea2a21ec3b8db0c..3cccabb13cbc6f6 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/fake-stack.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/fake-stack.ll
@@ -27,7 +27,7 @@ define void @Simple() uwtable sanitize_address {
 ; NEVER-NEXT:    store i64 1102416563, ptr [[TMP3]], align 8
 ; NEVER-NEXT:    [[TMP4:%.*]] = add i64 [[TMP0]], 8
 ; NEVER-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
-; NEVER-NEXT:    store i64 ptrtoint (ptr @___asan_gen_ to i64), ptr [[TMP5]], align 8
+; NEVER-NEXT:    store i64 ptrtoint (ptr @___asan_gen_stack to i64), ptr [[TMP5]], align 8
 ; NEVER-NEXT:    [[TMP6:%.*]] = add i64 [[TMP0]], 16
 ; NEVER-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
 ; NEVER-NEXT:    store i64 ptrtoint (ptr @Simple to i64), ptr [[TMP7]], align 8
@@ -69,7 +69,7 @@ define void @Simple() uwtable sanitize_address {
 ; RUNTIME-NEXT:    store i64 1102416563, ptr [[TMP13]], align 8
 ; RUNTIME-NEXT:    [[TMP14:%.*]] = add i64 [[TMP10]], 8
 ; RUNTIME-NEXT:    [[TMP15:%.*]] = inttoptr i64 [[TMP14]] to ptr
-; RUNTIME-NEXT:    store i64 ptrtoint (ptr @___asan_gen_ to i64), ptr [[TMP15]], align 8
+; RUNTIME-NEXT:    store i64 ptrtoint (ptr @___asan_gen_stack to i64), ptr [[TMP15]], align 8
 ; RUNTIME-NEXT:    [[TMP16:%.*]] = add i64 [[TMP10]], 16
 ; RUNTIME-NEXT:    [[TMP17:%.*]] = inttoptr i64 [[TMP16]] to ptr
 ; RUNTIME-NEXT:    store i64 ptrtoint (ptr @Simple to i64), ptr [[TMP17]], align 8
@@ -119,7 +119,7 @@ define void @Simple() uwtable sanitize_address {
 ; ALWAYS-NEXT:    store i64 1102416563, ptr [[TMP8]], align 8
 ; ALWAYS-NEXT:    [[TMP9:%.*]] = add i64 [[TMP5]], 8
 ; ALWAYS-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP9]] to ptr
-; ALWAYS-NEXT:    store i64 ptrtoint (ptr @___asan_gen_ to i64), ptr [[TMP10]], align 8
+; ALWAYS-NEXT:    store i64 ptrtoint (ptr @___asan_gen_stack to i64), ptr [[TMP10]], align 8
 ; ALWAYS-NEXT:    [[TMP11:%.*]] = add i64 [[TMP5]], 16
 ; ALWAYS-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
 ; ALWAYS-NEXT:    store i64 ptrtoint (ptr @Simple to i64), ptr [[TMP12]], align 8
@@ -167,7 +167,7 @@ define void @Huge() uwtable sanitize_address {
 ; CHECK-NEXT:    store i64 1102416563, ptr [[TMP3]], align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[TMP0]], 8
 ; CHECK-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
-; CHECK-NEXT:    store i64 ptrtoint (ptr @___asan_gen_.1 to i64), ptr [[TMP5]], align 8
+; CHECK-NEXT:    store i64 ptrtoint (ptr @___asan_gen_stack.1 to i64), ptr [[TMP5]], align 8
 ; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[TMP0]], 16
 ; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
 ; CHECK-NEXT:    store i64 ptrtoint (ptr @Huge to i64), ptr [[TMP7]], align 8
diff --git a/llvm/test/Instrumentation/AddressSanitizer/global_metadata.ll b/llvm/test/Instrumentation/AddressSanitizer/global_metadata.ll
index 47df1d43549cec4..6883a99be174952 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/global_metadata.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/global_metadata.ll
@@ -20,7 +20,7 @@ target triple = "x86_64-unknown-linux-gnu"
 ; CHECK: @.str = internal constant { [14 x i8], [18 x i8] } { [14 x i8] c"Hello, world!\00", [18 x i8] zeroinitializer }, comdat({{.*}}), align 32
 
 ; Check emitted location descriptions:
-; CHECK: [[VARNAME:@___asan_gen_.[0-9]+]] = private unnamed_addr constant [7 x i8] c"global\00", align 1
+; CHECK: [[VARNAME:@___asan_gen_global[.0-9]*]] = private unnamed_addr constant [7 x i8] c"global\00", align 1
 ; NOALIAS: @__asan_global_global = {{.*}}i64 ptrtoint (ptr @global to i64){{.*}} section "asan_globals"{{.*}}, !associated
 ; NOALIAS: @__asan_global_.str = {{.*}}i64 ptrtoint (ptr @{{.str|1}} to i64){{.*}} section "asan_globals"{{.*}}, !associated
 ; ALIAS: @__asan_global_global = {{.*}}i64 ptrtoint (ptr @0 to i64){{.*}} section "asan_globals"{{.*}}, !associated
diff --git a/llvm/test/Instrumentation/AddressSanitizer/global_metadata_array.ll b/llvm/test/Instrumentation/AddressSanitizer/global_metadata_array.ll
index 17b14164c3dc07a..ab1d5ffc2ef3434 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/global_metadata_array.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/global_metadata_array.ll
@@ -22,7 +22,7 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 ; CHECK: @.str = internal constant { [14 x i8], [18 x i8] } { [14 x i8] c"Hello, world!\00", [18 x i8] zeroinitializer }{{.*}}, align 32
 
 ; Check emitted location descriptions:
-; CHECK: [[VARNAME:@___asan_gen_.[0-9]+]] = private unnamed_addr constant [7 x i8] c"global\00", align 1
+; CHECK: [[VARNAME:@___asan_gen_global[.0-9]*]] = private unnamed_addr constant [7 x i8] c"global\00", align 1
 
 ; Check that location descriptors and global names were passed into __asan_register_globals:
 ; CHECK: i64 ptrtoint (ptr [[VARNAME]] to i64)
diff --git a/llvm/test/Instrumentation/AddressSanitizer/global_with_comdat.ll b/llvm/test/Instrumentation/AddressSanitizer/global_with_comdat.ll
index e61c0d8ce1f1b92..b2578292406bc58 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/global_with_comdat.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/global_with_comdat.ll
@@ -46,9 +46,9 @@ target triple = "x86_64-unknown-linux-gnu"
 ; NOCOMDAT: @.str = internal constant { [14 x i8], [18 x i8] } { [14 x i8] c"Hello, world!\00", [18 x i8] zeroinitializer }, align 32
 
 ; Check emitted location descriptions:
-; CHECK: [[VARNAME:@___asan_gen_.[0-9]+]] = private unnamed_addr constant [7 x i8] c"global\00", align 1
+; CHECK: [[VARNAME:@___asan_gen_global[.0-9]*]] = private unnamed_addr constant [7 x i8] c"global\00", align 1
 ; COMDAT: @__asan_global_global = {{.*}}i64 ptrtoint (ptr @__odr_asan_gen_global to i64){{.*}} section "asan_globals"{{.*}}, comdat($global), !associated
-; COMDAT: @__asan_global_.str = {{.*}}i64 ptrtoint (ptr @___asan_gen_ to i64){{.*}} section "asan_globals"{{.*}}, comdat($.str.{{.*}}), !associated
+; COMDAT: @__asan_global_.str = {{.*}}i64 ptrtoint (ptr @___asan_gen_global{{[.0-9]*}} to i64){{.*}} section "asan_globals"{{.*}}, comdat($.str.{{.*}}), !associated
 
 ; The metadata has to be inserted to llvm.compiler.used to avoid being stripped
 ; during LTO.
diff --git a/llvm/test/Instrumentation/AddressSanitizer/instrument_global.ll b/llvm/test/Instrumentation/AddressSanitizer/instrument_global.ll
index aa943cff6c4936e..2f26a7d9f44535e 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/instrument_global.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/instrument_global.ll
@@ -7,7 +7,7 @@ target triple = "x86_64-unknown-linux-gnu"
 ; If a global is present, __asan_[un]register_globals should be called from
 ; module ctor/dtor
 
-; CHECK: @___asan_gen_ = private constant [8 x i8] c"<stdin>\00", align 1
+; CHECK: @___asan_gen_module = private constant [8 x i8] c"<stdin>\00", align 1
 ; CHECK: @llvm.used = appending global [2 x ptr] [ptr @asan.module_ctor, ptr @asan.module_dtor], section "llvm.metadata"
 ; CHECK: @llvm.global_ctors = appending global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 1, ptr @asan.module_ctor, ptr @asan.module_ctor }]
 ; CHECK: @llvm.global_dtors = appending global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 1, ptr @asan.module_dtor, ptr @asan.module_dtor }]
diff --git a/llvm/test/Instrumentation/AddressSanitizer/instrument_late_initializer.ll b/llvm/test/Instrumentation/AddressSanitizer/instrument_late_initializer.ll
index 45d526a42c9f7b9..0c946f5934661f4 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/instrument_late_initializer.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/instrument_late_initializer.ll
@@ -51,7 +51,7 @@ define internal void @__late_ctor() sanitize_address section ".text.startup" {
 ; CHECK-LABEL: define internal void @__late_ctor(
 ; CHECK-SAME: ) #[[ATTR1:[0-9]+]] section ".text.startup" {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    call void @__asan_before_dynamic_init(i64 ptrtoint (ptr @___asan_gen_ to i64))
+; CHECK-NEXT:    call void @__asan_before_dynamic_init(i64 ptrtoint (ptr @___asan_gen_module to i64))
 ; CHECK-NEXT:    call void @__asan_after_dynamic_init()
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Instrumentation/AddressSanitizer/local_alias.ll b/llvm/test/Instrumentation/AddressSanitizer/local_alias.ll
index ada8120155e5efc..15496274445036f 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/local_alias.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/local_alias.ll
@@ -18,9 +18,9 @@ target triple = "x86_64-unknown-linux-gnu"
 ; Check that we generate internal alias and odr indicator symbols for global to be protected.
 ; CHECK-NOINDICATOR-NOT: __odr_asan_gen_a
 ; CHECK-NOALIAS-NOT: private alias
-; CHECK-INDICATOR: @___asan_gen_.1 = private unnamed_addr constant [2 x i8] c"a\00", align 1
+; CHECK-INDICATOR: @___asan_gen_global{{[.0-9]*}} = private unnamed_addr constant [2 x i8] c"a\00", align 1
 ; CHECK-INDICATOR: @__odr_asan_gen_a = global i8 0, align 1
-; CHECK-INDICATOR: @___asan_gen_.4 = private unnamed_addr constant [2 x i8] c"d\00", align 1
+; CHECK-INDICATOR: @___asan_gen_global{{[.0-9]*}} = private unnamed_addr constant [2 x i8] c"d\00", align 1
 ; CHECK-INDICATOR: @__odr_asan_gen__ZL1d = global i8 0, align 1
 ; CHECK-ALIAS: @0 = private alias { [2 x i32], [24 x i8] }, ptr @a
 
diff --git a/llvm/test/Instrumentation/AddressSanitizer/odr-check-ignore.ll b/llvm/test/Instrumentation/AddressSanitizer/odr-check-ignore.ll
index 3160802b1e09e33..28e58a52ddb982c 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/odr-check-ignore.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/odr-check-ignore.ll
@@ -9,15 +9,15 @@ target triple = "x86_64-unknown-linux-gnu"
 @c = internal global [2 x i32] zeroinitializer, align 4
 @d = unnamed_addr global [2 x i32] zeroinitializer, align 4
 
-; NOALIAS:      @__asan_global_a = private global { i64, i64, i64, i64, i64, i64, i64, i64 } { i64 ptrtoint (ptr @a to i64), i64 8, i64 32, i64 ptrtoint (ptr @___asan_gen_.1 to i64), i64 ptrtoint (ptr @___asan_gen_ to i64), i64 0, i64 0, i64 0 }
-; NOALIAS-NEXT: @__asan_global_b = private global { i64, i64, i64, i64, i64, i64, i64, i64 } { i64 ptrtoint (ptr @b to i64), i64 8, i64 32, i64 ptrtoint (ptr @___asan_gen_.2 to i64), i64 ptrtoint (ptr @___asan_gen_ to i64), i64 0, i64 0, i64 -1 }
-; NOALIAS-NEXT: @__asan_global_c = private global { i64, i64, i64, i64, i64, i64, i64, i64 } { i64 ptrtoint (ptr @c to i64), i64 8, i64 32, i64 ptrtoint (ptr @___asan_gen_.3 to i64), i64 ptrtoint (ptr @___asan_gen_ to i64), i64 0, i64 0, i64 -1 }
-; NOALIAS-NEXT: @__asan_global_d = private global { i64, i64, i64, i64, i64, i64, i64, i64 } { i64 ptrtoint (ptr @d to i64), i64 8, i64 32, i64 ptrtoint (ptr @___asan_gen_.4 to i64), i64 ptrtoint (ptr @___asan_gen_ to i64), i64 0, i64 0, i64 0 }
+; NOALIAS:      @__asan_global_a = private global { i64, i64, i64, i64, i64, i64, i64, i64 } { i64 ptrtoint (ptr @a to i64), i64 8, i64 32, i64 ptrtoint (ptr @___asan_gen_global{{.*}} to i64), i64 ptrtoint (ptr @___asan_gen_module to i64), i64 0, i64 0, i64 0 }
+; NOALIAS-NEXT: @__asan_global_b = private global { i64, i64, i64, i64, i64, i64, i64, i64 } { i64 ptrtoint (ptr @b to i64), i64 8, i64 32, i64 ptrtoint (ptr @___asan_gen_global{{.*}} to i64), i64 ptrtoint (ptr @___asan_gen_module to i64), i64 0, i64 0, i64 -1 }
+; NOALIAS-NEXT: @__asan_global_c = private global { i64, i64, i64, i64, i64, i64, i64, i64 } { i64 ptrtoint (ptr @c to i64), i64 8, i64 32, i64 ptrtoint (ptr @___asan_gen_global{{.*}} to i64), i64 ptrtoint (ptr @___asan_gen_module to i64), i64 0, i64 0, i64 -1 }
+; NOALIAS-NEXT: @__asan_global_d = private global { i64, i64, i64, i64, i64, i64, i64, i64 } { i64 ptrtoint (ptr @d to i64), i64 8, i64 32, i64 ptrtoint (ptr @___asan_gen_global{{.*}} to i64), i64 ptrtoint (ptr @___asan_gen_module to i64), i64 0, i64 0, i64 0 }
 
-; ALIAS:      @__asan_global_a = private global { i64, i64, i64, i64, i64, i64, i64, i64 } { i64 ptrtoint (ptr @0 to i64), i64 8, i64 32, i64 ptrtoint (ptr @___asan_gen_.1 to i64), i64 ptrtoint (ptr @___asan_gen_ to i64), i64 0, i64 0, i64 0 }
-; ALIAS-NEXT: @__asan_global_b = private global { i64, i64, i64, i64, i64, i64, i64, i64 } { i64 ptrtoint (ptr @1 to i64), i64 8, i64 32, i64 ptrtoint (ptr @___asan_gen_.2 to i64), i64 ptrtoint (ptr @___asan_gen_ to i64), i64 0, i64 0, i64 -1 }
-; ALIAS-NEXT: @__asan_global_c = private global { i64, i64, i64, i64, i64, i64, i64, i64 } { i64 ptrtoint (ptr @2 to i64), i64 8, i64 32, i64 ptrtoint (ptr @___asan_gen_.3 to i64), i64 ptrtoint (ptr @___asan_gen_ to i64), i64 0, i64 0, i64 -1 }
-; ALIAS-NEXT: @__asan_global_d = private global { i64, i64, i64, i64, i64, i64, i64, i64 } { i64 ptrtoint (ptr @3 to i64), i64 8, i64 32, i64 ptrtoint (ptr @___asan_gen_.4 to i64), i64 ptrtoint (ptr @___asan_gen_ to i64), i64 0, i64 0, i64 0 }
+; ALIAS:      @__asan_global_a = private global { i64, i64, i64, i64, i64, i64, i64, i64 } { i64 ptrtoint (ptr @0 to i64), i64 8, i64 32, i64 ptrtoint (ptr @___asan_gen_global{{.*}} to i64), i64 ptrtoint (ptr @___asan_gen_module to i64), i64 0, i64 0, i64 0 }
+; ALIAS-NEXT: @__asan_global_b = private global { i64, i64, i64, i64, i64, i64, i64, i64 } { i64 ptrtoint (ptr @1 to i64), i64 8, i64 32, i64 ptrtoint (ptr @___asan_gen_global{{.*}} to i64), i64 ptrtoint (ptr @___asan_gen_module to i64), i64 0, i64 0, i64 -1 }
+; ALIAS-NEXT: @__asan_global_c = private global { i64, i64, i64, i64, i64, i64, i64, i64 } { i64 ptrtoint (ptr @2 to i64), i64 8, i64 32, i64 ptrtoint (ptr @___asan_gen_global{{.*}} to i64), i64 ptrtoint (ptr @___asan_gen_module to i64), i64 0, i64 0, i64 -1 }
+; ALIAS-NEXT: @__asan_global_d = private global { i64, i64, i64, i64, i64, i64, i64, i64 } { i64 ptrtoint (ptr @3 to i64), i64 8, i64 32, i64 ptrtoint (ptr @___asan_gen_global{{.*}} to i64), i64 ptrtoint (ptr @___asan_gen_module to i64), i64 0, i64 0, i64 0 }
 ; ALIAS:      @0 = private alias {{.*}} @a
 ; ALIAS-NEXT: @1 = private alias {{.*}} @b
 ; ALIAS-NEXT: @2 = private alias {{.*}} @c
diff --git a/llvm/test/Instrumentation/AddressSanitizer/win-string-literal.ll b/llvm/test/Instrumentation/AddressSanitizer/win-string-literal.ll
index ac4bc4623e1ebf2..41691f0fc8dd4e5 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/win-string-literal.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/win-string-literal.ll
@@ -13,7 +13,7 @@
 ; CHECK:      @"__asan_global_??_C@_04JIHMPGLA@asdf?$AA@" =
 ; CHECK-SAME: private global { i64, i64, i64, i64, i64, i64, i64, i64 }
 ; CHECK-SAME: { i64 ptrtoint (ptr @"??_C@_04JIHMPGLA@asdf?$AA@" to i64),
-; CHECK-SAME:   i64 5, i64 32, i64 ptrtoint (ptr @___asan_gen_.1 to i64), i64 ptrtoint (ptr @___asan_gen_ to i64), i64 0, i64 0, i64 0 }, section ".ASAN$GL",
+; CHECK-SAME:   i64 5, i64 32, i64 ptrtoint (ptr @___asan_gen_global to i64), i64 ptrtoint (ptr @___asan_gen_module to i64), i64 0, i64 0, i64 0 }, section ".ASAN$GL",
 ; CHECK-SAME:   comdat($"??_C@_04JIHMPGLA@asdf?$AA@"), align 64
 
 ; ModuleID = 't.cpp'

From 5af3dfb1d9caee206d080db2afab01006210739f Mon Sep 17 00:00:00 2001
From: Vitaly Buka <vitalybuka@google.com>
Date: Mon, 19 Aug 2024 14:44:32 -0700
Subject: [PATCH 418/441] [NFC][asan] Create `ModuleName` lazily (#104729)

Avoids tracking conditions when it's needed.
---
 .../Instrumentation/AddressSanitizer.cpp      | 40 ++++++++++---------
 1 file changed, 22 insertions(+), 18 deletions(-)

diff --git a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
index 2655f49c48e3110..01dbdc156f698ed 100644
--- a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
@@ -944,13 +944,14 @@ class ModuleAddressSanitizer {
   bool shouldInstrumentGlobal(GlobalVariable *G) const;
   bool ShouldUseMachOGlobalsSection() const;
   StringRef getGlobalMetadataSection() const;
-  void poisonOneInitializer(Function &GlobalInit, GlobalValue *ModuleName);
-  void createInitializerPoisonCalls(GlobalValue *ModuleName);
+  void poisonOneInitializer(Function &GlobalInit);
+  void createInitializerPoisonCalls();
   uint64_t getMinRedzoneSizeForGlobal() const {
     return getRedzoneSizeForScale(Mapping.Scale);
   }
   uint64_t getRedzoneSizeForGlobal(uint64_t SizeInBytes) const;
   int GetAsanVersion() const;
+  GlobalVariable *getOrCreateModuleName();
 
   Module &M;
   bool CompileKernel;
@@ -978,6 +979,7 @@ class ModuleAddressSanitizer {
 
   Function *AsanCtorFunction = nullptr;
   Function *AsanDtorFunction = nullptr;
+  GlobalVariable *ModuleName = nullptr;
 };
 
 // Stack poisoning does not play well with exception handling.
@@ -1965,14 +1967,14 @@ void AddressSanitizer::instrumentUnusualSizeOrAlignment(
   }
 }
 
-void ModuleAddressSanitizer::poisonOneInitializer(Function &GlobalInit,
-                                                  GlobalValue *ModuleName) {
+void ModuleAddressSanitizer::poisonOneInitializer(Function &GlobalInit) {
   // Set up the arguments to our poison/unpoison functions.
   IRBuilder<> IRB(&GlobalInit.front(),
                   GlobalInit.front().getFirstInsertionPt());
 
   // Add a call to poison all external globals before the given function starts.
-  Value *ModuleNameAddr = ConstantExpr::getPointerCast(ModuleName, IntptrTy);
+  Value *ModuleNameAddr =
+      ConstantExpr::getPointerCast(getOrCreateModuleName(), IntptrTy);
   IRB.CreateCall(AsanPoisonGlobals, ModuleNameAddr);
 
   // Add calls to unpoison all globals before each return instruction.
@@ -1981,8 +1983,7 @@ void ModuleAddressSanitizer::poisonOneInitializer(Function &GlobalInit,
       CallInst::Create(AsanUnpoisonGlobals, "", RI->getIterator());
 }
 
-void ModuleAddressSanitizer::createInitializerPoisonCalls(
-    GlobalValue *ModuleName) {
+void ModuleAddressSanitizer::createInitializerPoisonCalls() {
   GlobalVariable *GV = M.getGlobalVariable("llvm.global_ctors");
   if (!GV)
     return;
@@ -2002,7 +2003,7 @@ void ModuleAddressSanitizer::createInitializerPoisonCalls(
       // Don't instrument CTORs that will run before asan.module_ctor.
       if (Priority->getLimitedValue() <= GetCtorAndDtorPriority(TargetTriple))
         continue;
-      poisonOneInitializer(*F, ModuleName);
+      poisonOneInitializer(*F);
     }
   }
 }
@@ -2539,14 +2540,6 @@ void ModuleAddressSanitizer::instrumentGlobals(IRBuilder<> &IRB,
 
   bool HasDynamicallyInitializedGlobals = false;
 
-  // We shouldn't merge same module names, as this string serves as unique
-  // module ID in runtime.
-  GlobalVariable *ModuleName =
-      n != 0 ? createPrivateGlobalForString(M, M.getModuleIdentifier(),
-                                            /*AllowMerging*/ false,
-                                            genName("module"))
-             : nullptr;
-
   for (size_t i = 0; i < n; i++) {
     GlobalVariable *G = GlobalsToChange[i];
 
@@ -2647,7 +2640,7 @@ void ModuleAddressSanitizer::instrumentGlobals(IRBuilder<> &IRB,
         ConstantInt::get(IntptrTy, SizeInBytes),
         ConstantInt::get(IntptrTy, SizeInBytes + RightRedzoneSize),
         ConstantExpr::getPointerCast(Name, IntptrTy),
-        ConstantExpr::getPointerCast(ModuleName, IntptrTy),
+        ConstantExpr::getPointerCast(getOrCreateModuleName(), IntptrTy),
         ConstantInt::get(IntptrTy, MD.IsDynInit),
         Constant::getNullValue(IntptrTy),
         ConstantExpr::getPointerCast(ODRIndicator, IntptrTy));
@@ -2694,7 +2687,7 @@ void ModuleAddressSanitizer::instrumentGlobals(IRBuilder<> &IRB,
 
   // Create calls for poisoning before initializers run and unpoisoning after.
   if (HasDynamicallyInitializedGlobals)
-    createInitializerPoisonCalls(ModuleName);
+    createInitializerPoisonCalls();
 
   LLVM_DEBUG(dbgs() << M);
 }
@@ -2734,6 +2727,17 @@ int ModuleAddressSanitizer::GetAsanVersion() const {
   return Version;
 }
 
+GlobalVariable *ModuleAddressSanitizer::getOrCreateModuleName() {
+  if (!ModuleName) {
+    // We shouldn't merge same module names, as this string serves as unique
+    // module ID in runtime.
+    ModuleName =
+        createPrivateGlobalForString(M, M.getModuleIdentifier(),
+                                     /*AllowMerging*/ false, genName("module"));
+  }
+  return ModuleName;
+}
+
 bool ModuleAddressSanitizer::instrumentModule() {
   initializeCallbacks();
 

From fa87eac3bea5b9704522c6b553d0f4a421e8abc9 Mon Sep 17 00:00:00 2001
From: Vitaly Buka <vitalybuka@google.com>
Date: Mon, 19 Aug 2024 14:45:55 -0700
Subject: [PATCH 419/441] =?UTF-8?q?Reland=20"[asan]=20Catch=20`initializat?=
 =?UTF-8?q?ion-order-fiasco`=20in=20modules=20without=E2=80=A6"=20(#104730?=
 =?UTF-8?q?)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Re-land https://github.com/llvm/llvm-project/pull/104621

After https://github.com/llvm/llvm-project/pull/104729 this patch will
not create unused module names, failing some test checks.

This reverts commit 34f941a2f96b804dd24c2a25770d899b018339ff.
---
 .../test/asan/TestCases/initialization-bug-no-global.cpp   | 3 ---
 llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp   | 7 +------
 .../instrument_initializer_without_global.ll               | 2 ++
 3 files changed, 3 insertions(+), 9 deletions(-)

diff --git a/compiler-rt/test/asan/TestCases/initialization-bug-no-global.cpp b/compiler-rt/test/asan/TestCases/initialization-bug-no-global.cpp
index 50698b2a7d06fc4..369449303f3e082 100644
--- a/compiler-rt/test/asan/TestCases/initialization-bug-no-global.cpp
+++ b/compiler-rt/test/asan/TestCases/initialization-bug-no-global.cpp
@@ -1,9 +1,6 @@
 // RUN: %clangxx_asan %min_macos_deployment_target=10.11 -O0 %s %p/Helpers/initialization-bug-extra.cpp -o %t
 // RUN: %env_asan_opts=check_initialization_order=true:strict_init_order=true not %run %t 2>&1 | FileCheck %s
 
-// Not implemented.
-// XFAIL: *
-
 // Do not test with optimization -- the error may be optimized away.
 
 // FIXME: https://code.google.com/p/address-sanitizer/issues/detail?id=186
diff --git a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
index 01dbdc156f698ed..5de0a78d087968b 100644
--- a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
@@ -2538,8 +2538,6 @@ void ModuleAddressSanitizer::instrumentGlobals(IRBuilder<> &IRB,
   SmallVector<GlobalVariable *, 16> NewGlobals(n);
   SmallVector<Constant *, 16> Initializers(n);
 
-  bool HasDynamicallyInitializedGlobals = false;
-
   for (size_t i = 0; i < n; i++) {
     GlobalVariable *G = GlobalsToChange[i];
 
@@ -2645,9 +2643,6 @@ void ModuleAddressSanitizer::instrumentGlobals(IRBuilder<> &IRB,
         Constant::getNullValue(IntptrTy),
         ConstantExpr::getPointerCast(ODRIndicator, IntptrTy));
 
-    if (ClInitializers && MD.IsDynInit)
-      HasDynamicallyInitializedGlobals = true;
-
     LLVM_DEBUG(dbgs() << "NEW GLOBAL: " << *NewGlobal << "\n");
 
     Initializers[i] = Initializer;
@@ -2686,7 +2681,7 @@ void ModuleAddressSanitizer::instrumentGlobals(IRBuilder<> &IRB,
   }
 
   // Create calls for poisoning before initializers run and unpoisoning after.
-  if (HasDynamicallyInitializedGlobals)
+  if (ClInitializers)
     createInitializerPoisonCalls();
 
   LLVM_DEBUG(dbgs() << M);
diff --git a/llvm/test/Instrumentation/AddressSanitizer/instrument_initializer_without_global.ll b/llvm/test/Instrumentation/AddressSanitizer/instrument_initializer_without_global.ll
index c8a6541bacfdfa1..b20de3eb80b786c 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/instrument_initializer_without_global.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/instrument_initializer_without_global.ll
@@ -18,7 +18,9 @@ define internal void @__late_ctor() sanitize_address section ".text.startup" {
 ; CHECK-LABEL: define internal void @__late_ctor(
 ; CHECK-SAME: ) #[[ATTR1:[0-9]+]] section ".text.startup" {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    call void @__asan_before_dynamic_init(i64 ptrtoint (ptr @___asan_gen_module to i64))
 ; CHECK-NEXT:    call void @initializer()
+; CHECK-NEXT:    call void @__asan_after_dynamic_init()
 ; CHECK-NEXT:    ret void
 ;
 ; NOINIT-LABEL: define internal void @__late_ctor(

From 54c6b93bcb4e751ec67ebd69e24c11811b970f1e Mon Sep 17 00:00:00 2001
From: lntue <35648136+lntue@users.noreply.github.com>
Date: Mon, 19 Aug 2024 17:58:46 -0400
Subject: [PATCH 420/441] [libc][NFC] Add sollya script to compute worst case
 range reduction. (#104803)

---
 libc/utils/mathtools/worst_case.sollya | 80 ++++++++++++++++++++++++++
 1 file changed, 80 insertions(+)
 create mode 100644 libc/utils/mathtools/worst_case.sollya

diff --git a/libc/utils/mathtools/worst_case.sollya b/libc/utils/mathtools/worst_case.sollya
new file mode 100644
index 000000000000000..3a8d11b3da44d51
--- /dev/null
+++ b/libc/utils/mathtools/worst_case.sollya
@@ -0,0 +1,80 @@
+// Implement WorstCase functions to compute the worst case for x mod C, with
+// the exponent of x ranges from emin to emax, and precision of x is p.
+// Adapted to Sollya from the Maple function in 
+//   J-M. Muller, "Elementary Functions", 3rd ed, Section 11.3.2.
+//
+// Some examples:
+//
+// 1) Worst case for trig range reduction fast passes:
+//
+// Single precision
+// > WorstCase(24, -6, 32, pi/32, 128); 
+// numbermin :  10741887
+// expmin    :  7
+// Worst case:  0x1.47d0fep30
+// numberofdigits :  -32.888
+//
+// Double precision
+// > WorstCase(53, -8, 32, pi/128, 256);
+// numbermin :  6411027962775774
+// expmin    :  -53
+// Worst case:  0x1.6c6cbc45dc8dep-1
+// numberofdigits :  -66.4867
+//
+// 2) Worst case for exponential function range reduction:
+//
+// Single precision
+// > WorstCase(24, 1, 8, log(2), 128);
+// numbermin :  2907270
+// expmin    :  -22
+// Worst case:  0x1.62e43p-1
+// numberofdigits :  -28.9678
+//
+// Double precision
+// > WorstCase(53, 0, 11, log(2), 128);
+// numbermin :  7804143460206699
+// expmin    :  -51
+// Worst case:  0x1.bb9d3beb8c86bp1
+// numberofdigits :  -57.4931
+//
+verbosity=0;
+procedure WorstCase(p, emin, emax, C, ndigits) {
+    epsilonmin = 12345.0;
+    Digits = ndigits;
+
+    powerofBoverC = 2^(emin - p) / C;
+    for e from emin - p + 1 to emax - p + 1 do {
+        powerofBoverC = 2 * powerofBoverC;
+        a = floor(powerofBoverC);
+        Plast = a;
+        r = round(1/(powerofBoverC - a), ndigits, RN);
+        a = floor(r);
+        Qlast = 1;
+        Q = a;
+        P = Plast * a + 1;
+        while (Q < 2^p - 1) do {
+            r = round(1/(r - a), ndigits, RN);
+            a = floor(r);
+            NewQ = Q * a + Qlast;
+            NewP = P * a + Plast;
+            Qlast = Q;
+            Plast = P;
+            Q = NewQ;
+            P = NewP;
+        };
+        epsilon = C * abs(Plast - Qlast * powerofBoverC);
+        if (epsilon < epsilonmin) then {
+            epsilonmin = epsilon;
+            numbermin = Qlast;
+            expmin = e;
+        };
+    };
+    display=decimal!;
+    print("numbermin : ", numbermin);
+    print("expmin    : ", expmin);
+    display=hexadecimal!;
+    print("Worst case: ", numbermin * 2^expmin);
+    display=decimal!;
+    ell = round(log2(epsilonmin), ndigits, RN);
+    print("numberofdigits : ", ell);
+};

From 24052538a0bb0144e6997e6b61687e3b8716b6ac Mon Sep 17 00:00:00 2001
From: Michael Park <mcypark@gmail.com>
Date: Mon, 19 Aug 2024 15:06:50 -0700
Subject: [PATCH 421/441] Emit `BeginSourceFile` failure with `elog`. (#104845)

There are 3 ways in which `ParseAST::build` can fail and return
`std::nullopt`. 2 of the ways we emit the error message using `elog`,
but for the 3rd way, `log` is used. We should emit all 3 of these
reasons with `elog`.
---
 clang-tools-extra/clangd/ParsedAST.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/clang-tools-extra/clangd/ParsedAST.cpp b/clang-tools-extra/clangd/ParsedAST.cpp
index a2f1504db7e880e..14440acd08b3533 100644
--- a/clang-tools-extra/clangd/ParsedAST.cpp
+++ b/clang-tools-extra/clangd/ParsedAST.cpp
@@ -512,8 +512,8 @@ ParsedAST::build(llvm::StringRef Filename, const ParseInputs &Inputs,
   auto Action = std::make_unique<ClangdFrontendAction>();
   const FrontendInputFile &MainInput = Clang->getFrontendOpts().Inputs[0];
   if (!Action->BeginSourceFile(*Clang, MainInput)) {
-    log("BeginSourceFile() failed when building AST for {0}",
-        MainInput.getFile());
+    elog("BeginSourceFile() failed when building AST for {0}",
+         MainInput.getFile());
     return std::nullopt;
   }
   // If we saw an include guard in the preamble section of the main file,

From 98f2eb7ddab0cadd1eb6e631458ae8406f7d39a1 Mon Sep 17 00:00:00 2001
From: jeffreytan81 <jeffreytan@meta.com>
Date: Mon, 19 Aug 2024 15:11:51 -0700
Subject: [PATCH 422/441] Fix
 StartDebuggingRequestHandler/ReplModeRequestHandler in lldb-dap (#104824)

`SBCommand::AddCommand()` requires `SBCommandPluginInterface` to be heap
based because it will be stored inside
`std::shared_ptr<lldb::SBCommandPluginInterface>` later for reference
counting. But lldb-dap passes
`StartDebuggingRequestHandler/ReplModeRequestHandler` static function
pointer to it which will cause corruption later during destruction.

This PR fixes this issue by making these two handler heap based.

Co-authored-by: jeffreytan81 <jeffreytan@fb.com>
---
 lldb/tools/lldb-dap/DAP.h        | 2 --
 lldb/tools/lldb-dap/lldb-dap.cpp | 4 ++--
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/lldb/tools/lldb-dap/DAP.h b/lldb/tools/lldb-dap/DAP.h
index 57562a149835197..7828272aa15a7d6 100644
--- a/lldb/tools/lldb-dap/DAP.h
+++ b/lldb/tools/lldb-dap/DAP.h
@@ -192,8 +192,6 @@ struct DAP {
   std::mutex call_mutex;
   std::map<int /* request_seq */, ResponseCallback /* reply handler */>
       inflight_reverse_requests;
-  StartDebuggingRequestHandler start_debugging_request_handler;
-  ReplModeRequestHandler repl_mode_request_handler;
   ReplMode repl_mode;
   std::string command_escape_prefix = "`";
   lldb::SBFormat frame_format;
diff --git a/lldb/tools/lldb-dap/lldb-dap.cpp b/lldb/tools/lldb-dap/lldb-dap.cpp
index ea84f31aec3a6c7..f50a6c173107394 100644
--- a/lldb/tools/lldb-dap/lldb-dap.cpp
+++ b/lldb/tools/lldb-dap/lldb-dap.cpp
@@ -1627,12 +1627,12 @@ void request_initialize(const llvm::json::Object &request) {
       "lldb-dap", "Commands for managing lldb-dap.");
   if (GetBoolean(arguments, "supportsStartDebuggingRequest", false)) {
     cmd.AddCommand(
-        "startDebugging", &g_dap.start_debugging_request_handler,
+        "startDebugging", new StartDebuggingRequestHandler(),
         "Sends a startDebugging request from the debug adapter to the client "
         "to start a child debug session of the same type as the caller.");
   }
   cmd.AddCommand(
-      "repl-mode", &g_dap.repl_mode_request_handler,
+      "repl-mode", new ReplModeRequestHandler(),
       "Get or set the repl behavior of lldb-dap evaluation requests.");
 
   g_dap.progress_event_thread = std::thread(ProgressEventThreadFunction);

From c4781396136849e79fe7b0c9ffe1ccb708bb1b67 Mon Sep 17 00:00:00 2001
From: Florian Mayer <fmayer@google.com>
Date: Mon, 19 Aug 2024 15:28:24 -0700
Subject: [PATCH 423/441] Reapply "[HWASan] symbolize stack overflows"
 (#102951) (#104036)

This reverts commit d4f6fcf5aaa0911a91317c0b06779f13077d6b58.

Relanding with fixed obj_offset calculation (precedence of operations
was wrong),
and the suggestion in
https://github.com/llvm/llvm-project/pull/95308#discussion_r1714317539
---
 .../lib/hwasan/scripts/hwasan_symbolize       | 49 +++++++++++++++++--
 .../hwasan_symbolize_stack_overflow.cpp       | 30 ++++++++++++
 2 files changed, 76 insertions(+), 3 deletions(-)
 create mode 100644 compiler-rt/test/hwasan/TestCases/hwasan_symbolize_stack_overflow.cpp

diff --git a/compiler-rt/lib/hwasan/scripts/hwasan_symbolize b/compiler-rt/lib/hwasan/scripts/hwasan_symbolize
index 0408e0eb9993d06..d1b2857ccd8156f 100755
--- a/compiler-rt/lib/hwasan/scripts/hwasan_symbolize
+++ b/compiler-rt/lib/hwasan/scripts/hwasan_symbolize
@@ -109,6 +109,10 @@ class Symbolizer:
     self.__html = False
     self.__last_access_address = None
     self.__last_access_tag = None
+    self.__tag_dump = []
+    self.__tag_dump_match_idx = None
+    self.__matched_stack_uas = False
+    self.__offsets = []
 
   def enable_html(self, enable):
     self.__html = enable
@@ -311,6 +315,34 @@ class Symbolizer:
     if match:
       self.__last_access_tag = int(match.group(2), 16)
 
+  def process_tag_dump_line(self, line, ignore_tags=False):
+    m = re.match(r'.*?(0x[0-9a-f]+):' + '([ ]*[\[ ][0-9a-f][0-9a-f]\]?)' * 16, line)
+    if m is None:
+      return False
+    addr = m.group(1)
+    tags = m.group(*range(2, 18))
+    fault = [i for i, x in enumerate(tags) if '[' in x]
+    if fault:
+      self.__tag_dump_match_idx = len(self.__tag_dump) + fault[0]
+    self.__tag_dump.extend(int(x.strip(' [').rstrip('] '), 16) for x in tags)
+    return True
+
+  def finish_tag_dump(self):
+    if self.__matched_stack_uas or self.__tag_dump_match_idx is None:
+      return
+    for offset, size, local in sorted(self.__offsets, key=lambda x: abs(x[0])):
+      idx = self.__tag_dump_match_idx - offset // 16
+      if idx < 0 or idx > len(self.__tag_dump):
+        continue
+      if self.__tag_dump[idx] == self.__last_access_tag:
+        self.print('')
+        self.print('Potentially referenced stack object:')
+        if offset > 0:
+          self.print('  %d bytes after a variable "%s" in stack frame of function "%s"' % (offset - size, local[2], local[0]))
+        if offset < 0:
+          self.print('  %d bytes before a variable "%s" in stack frame of function "%s"' % (-offset, local[2], local[0]))
+        self.print('  at %s' % (local[1],))
+
   def process_stack_history(self, line, ignore_tags=False):
     if self.__last_access_address is None or self.__last_access_tag is None:
       return
@@ -336,16 +368,18 @@ class Symbolizer:
         size = local[4]
         if frame_offset is None or size is None:
           continue
-        obj_offset = (self.__last_access_address - fp - frame_offset) & fp_mask
-        if obj_offset >= size:
-          continue
+        obj_offset = (self.__last_access_address & fp_mask) - ((fp & fp_mask) + frame_offset)
         tag_offset = local[5]
         if not ignore_tags and (tag_offset is None or base_tag ^ tag_offset != self.__last_access_tag):
           continue
+        if obj_offset < 0 or obj_offset >= size:
+          self.__offsets.append((obj_offset, size, local))
+          continue
         self.print('')
         self.print('Potentially referenced stack object:')
         self.print('  %d bytes inside a variable "%s" in stack frame of function "%s"' % (obj_offset, local[2], local[0]))
         self.print('  at %s' % (local[1],))
+        self.__matched_stack_uas = True
       return True
     return False
 
@@ -456,9 +490,18 @@ def main():
       sys.exit(1)
     symbolizer.read_linkify(args.linkify)
 
+  tag_dump = False
   for line in sys.stdin:
     if sys.version_info.major < 3:
       line = line.decode('utf-8')
+    if tag_dump:
+      tag_dump = symbolizer.process_tag_dump_line(line)
+      if tag_dump:
+        continue
+      symbolizer.finish_tag_dump()
+    if 'Memory tags around the buggy address' in line:
+      tag_dump = True
+
     symbolizer.save_access_address(line)
     if symbolizer.process_stack_history(line, ignore_tags=args.ignore_tags):
       continue
diff --git a/compiler-rt/test/hwasan/TestCases/hwasan_symbolize_stack_overflow.cpp b/compiler-rt/test/hwasan/TestCases/hwasan_symbolize_stack_overflow.cpp
new file mode 100644
index 000000000000000..356f1da0818a3f4
--- /dev/null
+++ b/compiler-rt/test/hwasan/TestCases/hwasan_symbolize_stack_overflow.cpp
@@ -0,0 +1,30 @@
+// RUN: rm -rf %t; mkdir %t
+// RUN: %clang_hwasan -Wl,--build-id -g %s -o %t/hwasan_overflow
+// RUN: %env_hwasan_opts=symbolize=0 not %run %t/hwasan_overflow 16 2>&1 | hwasan_symbolize --symbols %t --index | FileCheck %s --check-prefixes=CHECK,AFTER0
+// RUN: %env_hwasan_opts=symbolize=0 not %run %t/hwasan_overflow 17 2>&1 | hwasan_symbolize --symbols %t --index | FileCheck %s --check-prefixes=CHECK,AFTER1
+// RUN: %env_hwasan_opts=symbolize=0 not %run %t/hwasan_overflow -1 2>&1 | hwasan_symbolize --symbols %t --index | FileCheck %s --check-prefixes=CHECK,BEFORE1
+// RUN: %env_hwasan_opts=symbolize=0 not %run %t/hwasan_overflow -17 2>&1 | hwasan_symbolize --symbols %t --index | FileCheck %s --check-prefixes=CHECK,BEFORE17
+// RUN: %env_hwasan_opts=symbolize=0 not %run %t/hwasan_overflow 1016 2>&1 | hwasan_symbolize --symbols %t --index | FileCheck %s --check-prefixes=CHECK,AFTER1000
+// RUN: %env_hwasan_opts=symbolize=0 not %run %t/hwasan_overflow -1000 2>&1 | hwasan_symbolize --symbols %t --index | FileCheck %s --check-prefixes=CHECK,BEFORE1000
+// RUN: %env_hwasan_opts=symbolize=0 not %run %t/hwasan_overflow 1000000 2>&1 | hwasan_symbolize --symbols %t --index | FileCheck %s --check-prefixes=TOOFAR
+
+// REQUIRES: aarch64-target-arch
+
+#include <stdlib.h>
+
+static volatile char sink;
+
+int main(int argc, char **argv) {
+  volatile char x[16];
+  sink = x[atoi(argv[1])];
+  // TOOFAR-NOT: Potentially referenced stack object:
+  // CHECK: Potentially referenced stack object:
+  // AFTER0:   0 bytes after a variable "x" in stack frame of function "main"
+  // AFTER1:   1 bytes after a variable "x" in stack frame of function "main"
+  // BEFORE1:   1 bytes before a variable "x" in stack frame of function "main"
+  // BEFORE17:   17 bytes before a variable "x" in stack frame of function "main"
+  // BEFORE1000:   1000 bytes before a variable "x" in stack frame of function "main"
+  // AFTER1000:   1000 bytes after a variable "x" in stack frame of function "main"
+  // CHECK:   at {{.*}}hwasan_symbolize_stack_overflow.cpp:[[@LINE-10]]
+  return 0;
+}

From f5664f585bdc3187461f1dfd78d8f16ed2d990a0 Mon Sep 17 00:00:00 2001
From: Rahul Joshi <rjoshi@nvidia.com>
Date: Mon, 19 Aug 2024 15:31:24 -0700
Subject: [PATCH 424/441] [Support] Do not ignore unterminated open { in
 formatv (#104688)

- When an unterminated open { is detected in the format string, instead
of asserting and ignoring the error, replace that string with another to
indicate the error, and remove the assert as well.
- This will make the error evident in both assert and release builds and
make observing the error more convenient (as several uses of this
function are in TableGen and it is often built in release mode even in
debug builds)
---
 llvm/lib/Support/FormatVariadic.cpp | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/llvm/lib/Support/FormatVariadic.cpp b/llvm/lib/Support/FormatVariadic.cpp
index 3c07a80a00ae6d7..e25d036cdf1e8c3 100644
--- a/llvm/lib/Support/FormatVariadic.cpp
+++ b/llvm/lib/Support/FormatVariadic.cpp
@@ -107,15 +107,18 @@ formatv_object_base::splitLiteralAndReplacement(StringRef Fmt) {
       StringRef Right = Fmt.drop_front(NumEscapedBraces * 2);
       return std::make_pair(ReplacementItem{Middle}, Right);
     }
-    // An unterminated open brace is undefined.  We treat the rest of the string
-    // as a literal replacement, but we assert to indicate that this is
-    // undefined and that we consider it an error.
+    // An unterminated open brace is undefined. Assert to indicate that this is
+    // undefined and that we consider it an error. When asserts are disabled,
+    // build a replacement item with an error message.
     std::size_t BC = Fmt.find_first_of('}');
     if (BC == StringRef::npos) {
       assert(
           false &&
-          "Unterminated brace sequence.  Escape with {{ for a literal brace.");
-      return std::make_pair(ReplacementItem{Fmt}, StringRef());
+          "Unterminated brace sequence. Escape with {{ for a literal brace.");
+      return std::make_pair(
+          ReplacementItem{"Unterminated brace sequence. Escape with {{ for a "
+                          "literal brace."},
+          StringRef());
     }
 
     // Even if there is a closing brace, if there is another open brace before

From 31cc4ccdea92a4fee6a327a07251bb0ed1e6a933 Mon Sep 17 00:00:00 2001
From: David CARLIER <devnexen@gmail.com>
Date: Tue, 20 Aug 2024 00:40:54 +0200
Subject: [PATCH 425/441] [compiler-rt][fuzzer] implements SetThreadName for
 fuchsia. (#99953)

---
 compiler-rt/lib/fuzzer/FuzzerUtilFuchsia.cpp | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/compiler-rt/lib/fuzzer/FuzzerUtilFuchsia.cpp b/compiler-rt/lib/fuzzer/FuzzerUtilFuchsia.cpp
index fe79e1908d60299..eefa0b326e51b54 100644
--- a/compiler-rt/lib/fuzzer/FuzzerUtilFuchsia.cpp
+++ b/compiler-rt/lib/fuzzer/FuzzerUtilFuchsia.cpp
@@ -607,7 +607,13 @@ size_t PageSize() {
 }
 
 void SetThreadName(std::thread &thread, const std::string &name) {
-  // TODO ?
+  if (name.size() > 31)
+    name.resize(31);
+  zx_status_t s;
+  if ((s = zx_object_set_property(thread.native_handle(), ZX_PROP_NAME,
+                                  name.c_str(), name.size())) != ZX_OK)
+    Printf("SetThreadName for name %s failed: %s", name.c_str(),
+           zx_status_get_string(s));
 }
 
 } // namespace fuzzer

From 5fcd05967aaa0447d8da01b3f19139f2136863b4 Mon Sep 17 00:00:00 2001
From: Stanislav Mekhanoshin <rampitec@users.noreply.github.com>
Date: Mon, 19 Aug 2024 15:43:27 -0700
Subject: [PATCH 426/441] [AMDGPU] Add VOPD combine dependency tests. NFC.
 (#104841)

---
 llvm/test/CodeGen/AMDGPU/vopd-combine.mir | 49 +++++++++++++++++++++++
 1 file changed, 49 insertions(+)

diff --git a/llvm/test/CodeGen/AMDGPU/vopd-combine.mir b/llvm/test/CodeGen/AMDGPU/vopd-combine.mir
index b8ac50c3aeb5e8f..8d5060177c63d6f 100644
--- a/llvm/test/CodeGen/AMDGPU/vopd-combine.mir
+++ b/llvm/test/CodeGen/AMDGPU/vopd-combine.mir
@@ -23,6 +23,8 @@
   define void @vopd_mov_fixup_fail() { ret void }
   define void @vopd_no_combine_dependent_subreg() { ret void }
   define void @vopd_mov_mov_same_src_bank() { ret void }
+  define void @vopd_combine_opy_overwrites_opx() { ret void }
+  define void @vopd_no_combine_opx_overwrites_opy() { ret void }
 ...
 
 ---
@@ -737,3 +739,50 @@ body:             |
     $vgpr2 = V_MOV_B32_e32 $vgpr1, implicit $exec
     $vgpr3 = V_MOV_B32_e32 $vgpr5, implicit $exec
 ...
+
+---
+# There is no race between mov and add, they are executed at the same time
+name:            vopd_combine_opy_overwrites_opx
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    ; SCHED-LABEL: name: vopd_combine_opy_overwrites_opx
+    ; SCHED: $vgpr1 = IMPLICIT_DEF
+    ; SCHED-NEXT: $vgpr3 = IMPLICIT_DEF
+    ; SCHED-NEXT: $vgpr0 = V_MOV_B32_e32 killed $vgpr1, implicit $exec
+    ; SCHED-NEXT: $vgpr1 = V_ADD_F32_e32 killed $vgpr3, $vgpr3, implicit $mode, implicit $exec
+    ;
+    ; PAIR-GFX11-LABEL: name: vopd_combine_opy_overwrites_opx
+    ; PAIR-GFX11: $vgpr1 = IMPLICIT_DEF
+    ; PAIR-GFX11-NEXT: $vgpr3 = IMPLICIT_DEF
+    ; PAIR-GFX11-NEXT: $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_ADD_F32_e32_gfx11 killed $vgpr1, killed $vgpr3, $vgpr3, implicit $exec, implicit $mode, implicit $exec, implicit $mode, implicit $exec
+    ;
+    ; PAIR-GFX12-LABEL: name: vopd_combine_opy_overwrites_opx
+    ; PAIR-GFX12: $vgpr1 = IMPLICIT_DEF
+    ; PAIR-GFX12-NEXT: $vgpr3 = IMPLICIT_DEF
+    ; PAIR-GFX12-NEXT: $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_ADD_F32_e32_gfx12 killed $vgpr1, killed $vgpr3, $vgpr3, implicit $exec, implicit $mode, implicit $exec, implicit $mode, implicit $exec
+    $vgpr1 = IMPLICIT_DEF
+    $vgpr3 = IMPLICIT_DEF
+    $vgpr0 = V_MOV_B32_e32 $vgpr1, implicit $exec
+    $vgpr1 = V_ADD_F32_e32 $vgpr3, $vgpr3, implicit $mode, implicit $exec
+...
+
+---
+# OPY cannot cosume value produced by OPX
+name:            vopd_no_combine_opx_overwrites_opy
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    ; SCHED-LABEL: name: vopd_no_combine_opx_overwrites_opy
+    ; SCHED: $vgpr1 = IMPLICIT_DEF
+    ; SCHED-NEXT: $vgpr0 = V_MOV_B32_e32 $vgpr1, implicit $exec
+    ; SCHED-NEXT: $vgpr3 = V_ADD_F32_e32 killed $vgpr0, killed $vgpr1, implicit $mode, implicit $exec
+    ;
+    ; PAIR-LABEL: name: vopd_no_combine_opx_overwrites_opy
+    ; PAIR: $vgpr1 = IMPLICIT_DEF
+    ; PAIR-NEXT: $vgpr0 = V_MOV_B32_e32 $vgpr1, implicit $exec
+    ; PAIR-NEXT: $vgpr3 = V_ADD_F32_e32 killed $vgpr0, killed $vgpr1, implicit $mode, implicit $exec
+    $vgpr1 = IMPLICIT_DEF
+    $vgpr0 = V_MOV_B32_e32 $vgpr1, implicit $exec
+    $vgpr3 = V_ADD_F32_e32 $vgpr0, $vgpr1, implicit $mode, implicit $exec
+...

From 42ce62800dd24760ab6e400a3d5f898f11fe818a Mon Sep 17 00:00:00 2001
From: Noah Goldstein <goldstein.w.n@gmail.com>
Date: Mon, 19 Aug 2024 15:48:45 -0700
Subject: [PATCH 427/441] [ValueTracking] Handle incompatible types instead of
 asserting in `isKnownNonEqual`; NFC

Downstream hit this assert, since it doesn't really make any
difference, just change code to return false.
---
 llvm/lib/Analysis/ValueTracking.cpp | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index 9bd61f2996434af..173faa32a3878d8 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -310,8 +310,9 @@ bool llvm::isKnownNonEqual(const Value *V1, const Value *V2,
                            const DataLayout &DL, AssumptionCache *AC,
                            const Instruction *CxtI, const DominatorTree *DT,
                            bool UseInstrInfo) {
-  assert(V1->getType() == V2->getType() &&
-         "Testing equality of non-equal types!");
+  // We don't support looking through casts.
+  if (V1 == V2 || V1->getType() != V2->getType())
+    return false;
   auto *FVTy = dyn_cast<FixedVectorType>(V1->getType());
   APInt DemandedElts =
       FVTy ? APInt::getAllOnes(FVTy->getNumElements()) : APInt(1, 1);

From ea8bb4d633683f5cbfd82491620be3056f347a02 Mon Sep 17 00:00:00 2001
From: estewart08 <ethan.stewart@amd.com>
Date: Mon, 19 Aug 2024 17:59:21 -0500
Subject: [PATCH 428/441] [offload] - Fix issue with standalone debug offload
 build (#104647)

Error: CommandLine Error: Option 'attributor-manifest-internal'
registered more than once

During the standalone debug build of offload the above error is seen at
app runtime when using a prebuilt llvm with LLVM_LINK_LLVM_DYLIB=ON.
This is caused by linking both libLLVM.so and various archives that are
found via llvm_map_components_to_libnames for jit support.
---
 offload/plugins-nextgen/common/CMakeLists.txt | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/offload/plugins-nextgen/common/CMakeLists.txt b/offload/plugins-nextgen/common/CMakeLists.txt
index 284f98875170cdb..aea20c6ec314350 100644
--- a/offload/plugins-nextgen/common/CMakeLists.txt
+++ b/offload/plugins-nextgen/common/CMakeLists.txt
@@ -11,13 +11,15 @@ add_dependencies(PluginCommon intrinsics_gen)
 
 # Only enable JIT for those targets that LLVM can support.
 set(supported_jit_targets AMDGPU NVPTX)
-foreach(target IN LISTS supported_jit_targets)
-  if("${target}" IN_LIST LLVM_TARGETS_TO_BUILD)
-	  target_compile_definitions(PluginCommon PRIVATE "LIBOMPTARGET_JIT_${target}")
-    llvm_map_components_to_libnames(llvm_libs ${target})
-    target_link_libraries(PluginCommon PRIVATE ${llvm_libs})
-  endif()
-endforeach()
+if (NOT LLVM_LINK_LLVM_DYLIB)
+  foreach(target IN LISTS supported_jit_targets)
+    if("${target}" IN_LIST LLVM_TARGETS_TO_BUILD)
+      target_compile_definitions(PluginCommon PRIVATE "LIBOMPTARGET_JIT_${target}")
+      llvm_map_components_to_libnames(llvm_libs ${target})
+      target_link_libraries(PluginCommon PRIVATE ${llvm_libs})
+    endif()
+  endforeach()
+endif()
 
 # Include the RPC server from the `libc` project if availible.
 if(TARGET llvmlibc_rpc_server AND ${LIBOMPTARGET_GPU_LIBC_SUPPORT})

From 43b508566799751aa180f1eaaafc5be693f2f1ae Mon Sep 17 00:00:00 2001
From: Christopher Bate <cbate@nvidia.com>
Date: Mon, 19 Aug 2024 18:49:22 -0600
Subject: [PATCH 429/441] [mlir] NFC: fix dependence of
 (Tensor|Linalg|MemRef|Complex) dialects on LLVM Dialect and LLVM Core in
 CMake build (#104832)

This change removes dependencies declared as either 'LINK_LIBS' or
'LINK_COMPONENTS' across several MLIR libraries. The removed
dependencies appear
to be incorrect and may have been required in older versions of the
project.
These dependencies cause many high level dialects to have transitive
dependence on the LLVM dialect and the LLVM 'Core' library
('llvm/lib/IR').

Note that if using the 'Ninja' CMake generator, one can inspect the
dependencies
(including all transitive libraries) of any given MLIR target but using
the command `ninja -C <build dir> -t browse` and navigating to the
library
of interest in a web browser.
---
 mlir/lib/Conversion/AffineToStandard/CMakeLists.txt       | 3 ---
 mlir/lib/Conversion/ControlFlowToSCF/CMakeLists.txt       | 6 +-----
 mlir/lib/Conversion/ControlFlowToSCF/ControlFlowToSCF.cpp | 1 -
 mlir/lib/Conversion/SCFToControlFlow/CMakeLists.txt       | 5 +----
 mlir/lib/Conversion/SCFToControlFlow/SCFToControlFlow.cpp | 5 ++---
 mlir/lib/Conversion/VectorToSCF/CMakeLists.txt            | 4 ----
 mlir/lib/Dialect/Affine/Transforms/CMakeLists.txt         | 1 -
 mlir/lib/Dialect/Complex/IR/CMakeLists.txt                | 1 -
 mlir/lib/Dialect/Linalg/Transforms/CMakeLists.txt         | 2 --
 mlir/lib/Dialect/MemRef/IR/CMakeLists.txt                 | 3 ---
 mlir/lib/Dialect/MemRef/TransformOps/CMakeLists.txt       | 1 +
 mlir/lib/Dialect/Tensor/IR/CMakeLists.txt                 | 3 ---
 12 files changed, 5 insertions(+), 30 deletions(-)

diff --git a/mlir/lib/Conversion/AffineToStandard/CMakeLists.txt b/mlir/lib/Conversion/AffineToStandard/CMakeLists.txt
index f41e3ca27ee4dde..cd8193aee8da326 100644
--- a/mlir/lib/Conversion/AffineToStandard/CMakeLists.txt
+++ b/mlir/lib/Conversion/AffineToStandard/CMakeLists.txt
@@ -7,9 +7,6 @@ add_mlir_conversion_library(MLIRAffineToStandard
   DEPENDS
   MLIRConversionPassIncGen
 
-  LINK_COMPONENTS
-  Core
-
   LINK_LIBS PUBLIC
   MLIRAffineDialect
   MLIRAffineTransforms
diff --git a/mlir/lib/Conversion/ControlFlowToSCF/CMakeLists.txt b/mlir/lib/Conversion/ControlFlowToSCF/CMakeLists.txt
index 679eca3dfa988c5..e7024dfbd5ee325 100644
--- a/mlir/lib/Conversion/ControlFlowToSCF/CMakeLists.txt
+++ b/mlir/lib/Conversion/ControlFlowToSCF/CMakeLists.txt
@@ -5,11 +5,7 @@ add_mlir_conversion_library(MLIRControlFlowToSCF
   ${MLIR_MAIN_INCLUDE_DIR}/mlir/Conversion/ControlFlowToSCF
 
   DEPENDS
-  MLIRConversionPassIncGen
-  intrinsics_gen
-
-  LINK_COMPONENTS
-  Core
+  MLIRConversionPassIncGen  
 
   LINK_LIBS PUBLIC
   MLIRAnalysis
diff --git a/mlir/lib/Conversion/ControlFlowToSCF/ControlFlowToSCF.cpp b/mlir/lib/Conversion/ControlFlowToSCF/ControlFlowToSCF.cpp
index d3ee89743da9db5..1c592d665f3e4c5 100644
--- a/mlir/lib/Conversion/ControlFlowToSCF/ControlFlowToSCF.cpp
+++ b/mlir/lib/Conversion/ControlFlowToSCF/ControlFlowToSCF.cpp
@@ -16,7 +16,6 @@
 #include "mlir/Dialect/ControlFlow/IR/ControlFlow.h"
 #include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
-#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/Dialect/SCF/IR/SCF.h"
 #include "mlir/Dialect/UB/IR/UBOps.h"
 #include "mlir/Pass/Pass.h"
diff --git a/mlir/lib/Conversion/SCFToControlFlow/CMakeLists.txt b/mlir/lib/Conversion/SCFToControlFlow/CMakeLists.txt
index 63c5199af9290c9..ff64b8fcca0830e 100644
--- a/mlir/lib/Conversion/SCFToControlFlow/CMakeLists.txt
+++ b/mlir/lib/Conversion/SCFToControlFlow/CMakeLists.txt
@@ -5,10 +5,7 @@ add_mlir_conversion_library(MLIRSCFToControlFlow
   ${MLIR_MAIN_INCLUDE_DIR}/mlir/Conversion/SCFToControlFlow
 
   DEPENDS
-  MLIRConversionPassIncGen
-
-  LINK_COMPONENTS
-  Core
+  MLIRConversionPassIncGen  
 
   LINK_LIBS PUBLIC
   MLIRArithDialect
diff --git a/mlir/lib/Conversion/SCFToControlFlow/SCFToControlFlow.cpp b/mlir/lib/Conversion/SCFToControlFlow/SCFToControlFlow.cpp
index 2372ab5b82a772e..cef96c206dda5c8 100644
--- a/mlir/lib/Conversion/SCFToControlFlow/SCFToControlFlow.cpp
+++ b/mlir/lib/Conversion/SCFToControlFlow/SCFToControlFlow.cpp
@@ -15,7 +15,6 @@
 
 #include "mlir/Dialect/Arith/IR/Arith.h"
 #include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h"
-#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/Dialect/SCF/IR/SCF.h"
 #include "mlir/Dialect/SCF/Transforms/Transforms.h"
 #include "mlir/IR/Builders.h"
@@ -379,8 +378,8 @@ LogicalResult ForLowering::matchAndRewrite(ForOp forOp,
   // llvm.loop_annotation attribute.
   SmallVector<NamedAttribute> llvmAttrs;
   llvm::copy_if(forOp->getAttrs(), std::back_inserter(llvmAttrs),
-                [](auto attr) {
-                  return isa<LLVM::LLVMDialect>(attr.getValue().getDialect());
+                [](NamedAttribute attr) {
+                  return attr.getValue().getDialect().getNamespace() == "llvm";
                 });
   condBranchOp->setDiscardableAttrs(llvmAttrs);
   // The result of the loop operation is the values of the condition block
diff --git a/mlir/lib/Conversion/VectorToSCF/CMakeLists.txt b/mlir/lib/Conversion/VectorToSCF/CMakeLists.txt
index 83bbc4a4bc395a6..0915270e453dfdb 100644
--- a/mlir/lib/Conversion/VectorToSCF/CMakeLists.txt
+++ b/mlir/lib/Conversion/VectorToSCF/CMakeLists.txt
@@ -4,12 +4,8 @@ add_mlir_conversion_library(MLIRVectorToSCF
   ADDITIONAL_HEADER_DIRS
   ${MLIR_MAIN_INCLUDE_DIR}/mlir/Conversion/VectorToSCF
 
-  LINK_COMPONENTS
-  Core
-
   LINK_LIBS PUBLIC
   MLIRArithDialect
-  MLIRLLVMDialect
   MLIRMemRefDialect
   MLIRTransforms
   MLIRVectorDialect
diff --git a/mlir/lib/Dialect/Affine/Transforms/CMakeLists.txt b/mlir/lib/Dialect/Affine/Transforms/CMakeLists.txt
index 60d14a373348d38..772f15335d907f2 100644
--- a/mlir/lib/Dialect/Affine/Transforms/CMakeLists.txt
+++ b/mlir/lib/Dialect/Affine/Transforms/CMakeLists.txt
@@ -39,6 +39,5 @@ add_mlir_dialect_library(MLIRAffineTransforms
   MLIRValueBoundsOpInterface
   MLIRVectorDialect
   MLIRVectorUtils
-  MLIRVectorToLLVM
   )
 
diff --git a/mlir/lib/Dialect/Complex/IR/CMakeLists.txt b/mlir/lib/Dialect/Complex/IR/CMakeLists.txt
index 3ee0d26f3225f72..a90f34ec1684d90 100644
--- a/mlir/lib/Dialect/Complex/IR/CMakeLists.txt
+++ b/mlir/lib/Dialect/Complex/IR/CMakeLists.txt
@@ -10,7 +10,6 @@ add_mlir_dialect_library(MLIRComplexDialect
   MLIRComplexAttributesIncGen
 
   LINK_LIBS PUBLIC
-  MLIRArithAttrToLLVMConversion
   MLIRArithDialect
   MLIRDialect
   MLIRInferTypeOpInterface
diff --git a/mlir/lib/Dialect/Linalg/Transforms/CMakeLists.txt b/mlir/lib/Dialect/Linalg/Transforms/CMakeLists.txt
index a7dcc29b5b9be18..47af392def94ac0 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/CMakeLists.txt
+++ b/mlir/lib/Dialect/Linalg/Transforms/CMakeLists.txt
@@ -60,7 +60,6 @@ add_mlir_dialect_library(MLIRLinalgTransforms
   MLIRDestinationStyleOpInterface
   MLIRDialectUtils
   MLIRFuncDialect
-  MLIRFuncToLLVM
   MLIRFuncTransforms
   MLIRIndexDialect
   MLIRInferTypeOpInterface
@@ -87,6 +86,5 @@ add_mlir_dialect_library(MLIRLinalgTransforms
   MLIRVectorDialect
   MLIRVectorTransforms
   MLIRVectorUtils
-  MLIRX86VectorTransforms
   MLIRVectorToSCF
 )
diff --git a/mlir/lib/Dialect/MemRef/IR/CMakeLists.txt b/mlir/lib/Dialect/MemRef/IR/CMakeLists.txt
index fd2fed28badd63f..845914ebd107a26 100644
--- a/mlir/lib/Dialect/MemRef/IR/CMakeLists.txt
+++ b/mlir/lib/Dialect/MemRef/IR/CMakeLists.txt
@@ -10,9 +10,6 @@ add_mlir_dialect_library(MLIRMemRefDialect
   DEPENDS
   MLIRMemRefOpsIncGen
 
-  LINK_COMPONENTS
-  Core
-
   LINK_LIBS PUBLIC
   MLIRArithDialect
   MLIRArithUtils
diff --git a/mlir/lib/Dialect/MemRef/TransformOps/CMakeLists.txt b/mlir/lib/Dialect/MemRef/TransformOps/CMakeLists.txt
index e379663f3b0a01f..4c9e3d3c813f7a6 100644
--- a/mlir/lib/Dialect/MemRef/TransformOps/CMakeLists.txt
+++ b/mlir/lib/Dialect/MemRef/TransformOps/CMakeLists.txt
@@ -19,4 +19,5 @@ add_mlir_dialect_library(MLIRMemRefTransformOps
   MLIRNVGPUDialect
   MLIRTransformDialect
   MLIRVectorDialect
+  MLIRVectorTransforms
 )
diff --git a/mlir/lib/Dialect/Tensor/IR/CMakeLists.txt b/mlir/lib/Dialect/Tensor/IR/CMakeLists.txt
index 549b9f10388bd47..cfdd3847761a49a 100644
--- a/mlir/lib/Dialect/Tensor/IR/CMakeLists.txt
+++ b/mlir/lib/Dialect/Tensor/IR/CMakeLists.txt
@@ -17,9 +17,6 @@ add_mlir_dialect_library(MLIRTensorDialect
   DEPENDS
   MLIRTensorOpsIncGen
 
-  LINK_COMPONENTS
-  Core
-
   LINK_LIBS PUBLIC
   MLIRAffineDialect
   MLIRArithDialect

From 27dc247964472aaf92ea55cc37310cda25a002ab Mon Sep 17 00:00:00 2001
From: Joshua Baehring <98630690+JoshuaMBa@users.noreply.github.com>
Date: Mon, 19 Aug 2024 18:03:35 -0700
Subject: [PATCH 430/441] [scudo] Add partial chunk heuristic to retrieval
 algorithm. (#104807)

Previously the secondary cache retrieval algorithm would not allow
retrievals of memory chunks where the number of unused bytes would be
greater than than `MaxUnusedCachePages * PageSize` bytes. This meant
that even if a memory chunk satisfied the requirements of the optimal
fit algorithm, it may not be returned. This remains true if memory
tagging is enabled. However, if memory tagging is disabled, a new
heuristic has been put in place. Specifically, If a memory chunk is a
non-optimal fit, the cache retrieval algorithm will attempt to release
the excess memory to force a cache hit while keeping RSS down.

In the event that a memory chunk is a non-optimal fit, the retrieval
algorithm will release excess memory as long as the amount of memory to
be released is less than or equal to 16 KB. If the amount of memory to
be released exceeds 16 KB, the retrieval algorithm will not consider
that cached memory chunk valid for retrieval.
---
 compiler-rt/lib/scudo/standalone/secondary.h  | 121 +++++++++++++-----
 .../scudo/standalone/tests/secondary_test.cpp |  36 +++++-
 2 files changed, 123 insertions(+), 34 deletions(-)

diff --git a/compiler-rt/lib/scudo/standalone/secondary.h b/compiler-rt/lib/scudo/standalone/secondary.h
index 27f8697db7838f5..7a563fa64f7192f 100644
--- a/compiler-rt/lib/scudo/standalone/secondary.h
+++ b/compiler-rt/lib/scudo/standalone/secondary.h
@@ -72,6 +72,14 @@ namespace {
 struct CachedBlock {
   static constexpr u16 CacheIndexMax = UINT16_MAX;
   static constexpr u16 InvalidEntry = CacheIndexMax;
+  //   * ReleaseMemoryUpperBound default is currently 16 KB
+  //        - We arrived at this value after noticing that mapping
+  //        in larger memory regions performs better than releasing
+  //        memory and forcing a cache hit. According to the data,
+  //        it suggests that beyond 16KB, the release execution time is
+  //        longer than the map execution time. In this way, the default
+  //        is dependent on the platform.
+  static constexpr uptr ReleaseMemoryUpperBound = 1 << 14;
 
   uptr CommitBase = 0;
   uptr CommitSize = 0;
@@ -90,8 +98,9 @@ struct CachedBlock {
 template <typename Config> class MapAllocatorNoCache {
 public:
   void init(UNUSED s32 ReleaseToOsInterval) {}
-  CachedBlock retrieve(UNUSED uptr Size, UNUSED uptr Alignment,
-                       UNUSED uptr HeadersSize, UNUSED uptr &EntryHeaderPos) {
+  CachedBlock retrieve(UNUSED uptr MaxAllowedFragmentedBytes, UNUSED uptr Size,
+                       UNUSED uptr Alignment, UNUSED uptr HeadersSize,
+                       UNUSED uptr &EntryHeaderPos) {
     return {};
   }
   void store(UNUSED Options Options, UNUSED uptr CommitBase,
@@ -334,13 +343,14 @@ class MapAllocatorCache {
     }
   }
 
-  CachedBlock retrieve(uptr Size, uptr Alignment, uptr HeadersSize,
-                       uptr &EntryHeaderPos) EXCLUDES(Mutex) {
+  CachedBlock retrieve(uptr MaxAllowedFragmentedBytes, uptr Size,
+                       uptr Alignment, uptr HeadersSize, uptr &EntryHeaderPos)
+      EXCLUDES(Mutex) {
     const uptr PageSize = getPageSizeCached();
     // 10% of the requested size proved to be the optimal choice for
     // retrieving cached blocks after testing several options.
     constexpr u32 FragmentedBytesDivisor = 10;
-    bool Found = false;
+    bool FoundOptimalFit = false;
     CachedBlock Entry;
     EntryHeaderPos = 0;
     {
@@ -348,47 +358,88 @@ class MapAllocatorCache {
       CallsToRetrieve++;
       if (EntriesCount == 0)
         return {};
-      u32 OptimalFitIndex = 0;
+      u16 RetrievedIndex = CachedBlock::InvalidEntry;
       uptr MinDiff = UINTPTR_MAX;
-      for (u32 I = LRUHead; I != CachedBlock::InvalidEntry;
+
+      //  Since allocation sizes don't always match cached memory chunk sizes
+      //  we allow some memory to be unused (called fragmented bytes). The
+      //  amount of unused bytes is exactly EntryHeaderPos - CommitBase.
+      //
+      //        CommitBase                CommitBase + CommitSize
+      //          V                              V
+      //      +---+------------+-----------------+---+
+      //      |   |            |                 |   |
+      //      +---+------------+-----------------+---+
+      //      ^                ^                     ^
+      //    Guard         EntryHeaderPos          Guard-page-end
+      //    page-begin
+      //
+      //  [EntryHeaderPos, CommitBase + CommitSize) contains the user data as
+      //  well as the header metadata. If EntryHeaderPos - CommitBase exceeds
+      //  MaxAllowedFragmentedBytes, the cached memory chunk is not considered
+      //  valid for retrieval.
+      for (u16 I = LRUHead; I != CachedBlock::InvalidEntry;
            I = Entries[I].Next) {
         const uptr CommitBase = Entries[I].CommitBase;
         const uptr CommitSize = Entries[I].CommitSize;
         const uptr AllocPos =
             roundDown(CommitBase + CommitSize - Size, Alignment);
         const uptr HeaderPos = AllocPos - HeadersSize;
-        if (HeaderPos > CommitBase + CommitSize)
+        if (HeaderPos > CommitBase + CommitSize || HeaderPos < CommitBase)
           continue;
-        if (HeaderPos < CommitBase ||
-            AllocPos > CommitBase + PageSize * MaxUnusedCachePages) {
+
+        const uptr Diff = roundDown(HeaderPos, PageSize) - CommitBase;
+
+        if (Diff > MaxAllowedFragmentedBytes || Diff >= MinDiff)
           continue;
-        }
-        Found = true;
-        const uptr Diff = HeaderPos - CommitBase;
-        // immediately use a cached block if it's size is close enough to the
-        // requested size.
-        const uptr MaxAllowedFragmentedBytes =
+
+        MinDiff = Diff;
+        RetrievedIndex = I;
+        EntryHeaderPos = HeaderPos;
+
+        const uptr OptimalFitThesholdBytes =
             (CommitBase + CommitSize - HeaderPos) / FragmentedBytesDivisor;
-        if (Diff <= MaxAllowedFragmentedBytes) {
-          OptimalFitIndex = I;
-          EntryHeaderPos = HeaderPos;
+        if (Diff <= OptimalFitThesholdBytes) {
+          FoundOptimalFit = true;
           break;
         }
-        // keep track of the smallest cached block
-        // that is greater than (AllocSize + HeaderSize)
-        if (Diff > MinDiff)
-          continue;
-        OptimalFitIndex = I;
-        MinDiff = Diff;
-        EntryHeaderPos = HeaderPos;
       }
-      if (Found) {
-        Entry = Entries[OptimalFitIndex];
-        remove(OptimalFitIndex);
+      if (RetrievedIndex != CachedBlock::InvalidEntry) {
+        Entry = Entries[RetrievedIndex];
+        remove(RetrievedIndex);
         SuccessfulRetrieves++;
       }
     }
 
+    //  The difference between the retrieved memory chunk and the request
+    //  size is at most MaxAllowedFragmentedBytes
+    //
+    //  /     MaxAllowedFragmentedBytes      \
+    // +--------------------------+-----------+
+    // |                          |           |
+    // +--------------------------+-----------+
+    //  \ Bytes to be released   /      ^
+    //                                  |
+    //                         (may or may not be commited)
+    //
+    //   The maximum number of bytes released to the OS is capped by
+    //   ReleaseMemoryUpperBound
+    //
+    //   TODO : Consider making ReleaseMemoryUpperBound configurable since
+    //   the release to OS API can vary across systems.
+    if (!FoundOptimalFit && Entry.Time != 0) {
+      const uptr FragmentedBytes =
+          roundDown(EntryHeaderPos, PageSize) - Entry.CommitBase;
+      const uptr MaxUnusedCacheBytes = MaxUnusedCachePages * PageSize;
+      if (FragmentedBytes > MaxUnusedCacheBytes) {
+        uptr BytesToRelease =
+            roundUp(Min<uptr>(CachedBlock::ReleaseMemoryUpperBound,
+                              FragmentedBytes - MaxUnusedCacheBytes),
+                    PageSize);
+        Entry.MemMap.releaseAndZeroPagesToOS(Entry.CommitBase, BytesToRelease);
+      }
+    }
+
     return Entry;
   }
 
@@ -659,8 +710,18 @@ MapAllocator<Config>::tryAllocateFromCache(const Options &Options, uptr Size,
                                            FillContentsMode FillContents) {
   CachedBlock Entry;
   uptr EntryHeaderPos;
+  uptr MaxAllowedFragmentedBytes;
+  const uptr PageSize = getPageSizeCached();
+
+  if (LIKELY(!useMemoryTagging<Config>(Options))) {
+    MaxAllowedFragmentedBytes =
+        MaxUnusedCachePages * PageSize + CachedBlock::ReleaseMemoryUpperBound;
+  } else {
+    MaxAllowedFragmentedBytes = MaxUnusedCachePages * PageSize;
+  }
 
-  Entry = Cache.retrieve(Size, Alignment, getHeadersSize(), EntryHeaderPos);
+  Entry = Cache.retrieve(MaxAllowedFragmentedBytes, Size, Alignment,
+                         getHeadersSize(), EntryHeaderPos);
   if (!Entry.isValid())
     return nullptr;
 
diff --git a/compiler-rt/lib/scudo/standalone/tests/secondary_test.cpp b/compiler-rt/lib/scudo/standalone/tests/secondary_test.cpp
index e85b6abdb36d228..e33e93c9ff78d48 100644
--- a/compiler-rt/lib/scudo/standalone/tests/secondary_test.cpp
+++ b/compiler-rt/lib/scudo/standalone/tests/secondary_test.cpp
@@ -281,8 +281,8 @@ struct MapAllocatorCacheTest : public Test {
   std::unique_ptr<CacheT> Cache = std::make_unique<CacheT>();
 
   const scudo::uptr PageSize = scudo::getPageSizeCached();
-  // The current test allocation size is set to the minimum size
-  // needed for the scudo allocator to fall back to the secondary allocator
+  // The current test allocation size is set to the maximum
+  // cache entry size
   static constexpr scudo::uptr TestAllocSize =
       CacheConfig::getDefaultMaxEntrySize();
 
@@ -327,7 +327,8 @@ TEST_F(MapAllocatorCacheTest, CacheOrder) {
   for (scudo::uptr I = CacheConfig::getEntriesArraySize(); I > 0; I--) {
     scudo::uptr EntryHeaderPos;
     scudo::CachedBlock Entry =
-        Cache->retrieve(TestAllocSize, PageSize, 0, EntryHeaderPos);
+        Cache->retrieve(scudo::MaxUnusedCachePages * PageSize, TestAllocSize,
+                        PageSize, 0, EntryHeaderPos);
     EXPECT_EQ(Entry.MemMap.getBase(), MemMaps[I - 1].getBase());
   }
 
@@ -336,6 +337,32 @@ TEST_F(MapAllocatorCacheTest, CacheOrder) {
     MemMap.unmap();
 }
 
+TEST_F(MapAllocatorCacheTest, PartialChunkHeuristicRetrievalTest) {
+  const scudo::uptr MaxUnusedCacheBytes = PageSize;
+  const scudo::uptr FragmentedBytes =
+      MaxUnusedCacheBytes + scudo::CachedBlock::ReleaseMemoryUpperBound;
+
+  scudo::uptr EntryHeaderPos;
+  scudo::CachedBlock Entry;
+  scudo::MemMapT MemMap = allocate(PageSize + FragmentedBytes);
+  Cache->store(Options, MemMap.getBase(), MemMap.getCapacity(),
+               MemMap.getBase(), MemMap);
+
+  // FragmentedBytes > MaxAllowedFragmentedBytes so PageSize
+  // cannot be retrieved from the cache
+  Entry = Cache->retrieve(/*MaxAllowedFragmentedBytes=*/0, PageSize, PageSize,
+                          0, EntryHeaderPos);
+  EXPECT_FALSE(Entry.isValid());
+
+  // FragmentedBytes <= MaxAllowedFragmentedBytes so PageSize
+  // can be retrieved from the cache
+  Entry =
+      Cache->retrieve(FragmentedBytes, PageSize, PageSize, 0, EntryHeaderPos);
+  EXPECT_TRUE(Entry.isValid());
+
+  MemMap.unmap();
+}
+
 TEST_F(MapAllocatorCacheTest, MemoryLeakTest) {
   std::vector<scudo::MemMapT> MemMaps;
   // Fill the cache above MaxEntriesCount to force an eviction
@@ -351,7 +378,8 @@ TEST_F(MapAllocatorCacheTest, MemoryLeakTest) {
   for (scudo::uptr I = CacheConfig::getDefaultMaxEntriesCount(); I > 0; I--) {
     scudo::uptr EntryHeaderPos;
     RetrievedEntries.push_back(
-        Cache->retrieve(TestAllocSize, PageSize, 0, EntryHeaderPos));
+        Cache->retrieve(scudo::MaxUnusedCachePages * PageSize, TestAllocSize,
+                        PageSize, 0, EntryHeaderPos));
     EXPECT_EQ(MemMaps[I].getBase(), RetrievedEntries.back().MemMap.getBase());
   }
 

From baa6627a0af6d31ddf57036c6f05a7d0fb31f8f2 Mon Sep 17 00:00:00 2001
From: Billy Zhu <billyzhu@modular.com>
Date: Mon, 19 Aug 2024 18:04:16 -0700
Subject: [PATCH 431/441] [MLIR][Transforms] Fix dialect conversion inverse
 mapping (#104648)

Inverse mapping needs to be updated for the result that was remapped (it
was previously only updated halfway).
---
 mlir/lib/Transforms/Utils/DialectConversion.cpp       |  1 +
 .../test-legalize-target-materialization-no-uses.mlir | 11 ++++++++++-
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/mlir/lib/Transforms/Utils/DialectConversion.cpp b/mlir/lib/Transforms/Utils/DialectConversion.cpp
index 8a4c7463a69a950..bf2990c257bad2f 100644
--- a/mlir/lib/Transforms/Utils/DialectConversion.cpp
+++ b/mlir/lib/Transforms/Utils/DialectConversion.cpp
@@ -2527,6 +2527,7 @@ LogicalResult OperationConverter::legalizeConvertedOpResultTypes(
           opReplacement->getConverter());
       rewriterImpl.mapping.map(result, castValue);
       inverseMapping[castValue].push_back(result);
+      llvm::erase(inverseMapping[newValue], result);
     }
   }
 
diff --git a/mlir/test/Transforms/test-legalize-target-materialization-no-uses.mlir b/mlir/test/Transforms/test-legalize-target-materialization-no-uses.mlir
index 19dd2b2ee4b2f21..5e694a04af49a8e 100644
--- a/mlir/test/Transforms/test-legalize-target-materialization-no-uses.mlir
+++ b/mlir/test/Transforms/test-legalize-target-materialization-no-uses.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt -test-target-materialization-with-no-uses %s | FileCheck %s
+// RUN: mlir-opt -test-target-materialization-with-no-uses -split-input-file %s | FileCheck %s
 
 // The conversion is set up as follows:
 // - type_changer ops are illegal;
@@ -25,3 +25,12 @@ func.func @foo() {
   "test.type_consumer"(%2) : (i64) -> ()
   return
 }
+
+// -----
+
+// CHECK-LABEL: @direct_forward
+func.func @direct_forward(%arg0 : i16) -> i16 {
+  // CHECK-NEXT: return
+  %0 = "test.type_changer"(%arg0) : (i16) -> i16
+  return %0 : i16
+}

From b5f3e28eca9bae94412415e5a77f769cf1e4fc61 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Mon, 19 Aug 2024 18:05:06 -0700
Subject: [PATCH 432/441] [docs] Revise the doc for
 __builtin_allow_runtime_check

Fix list formatting, improve the wording, and fix the description when
both options (note: prefer "option" to "flag" when arguments are
supported) are specified.

Pull Request: https://github.com/llvm/llvm-project/pull/104886
---
 clang/docs/LanguageExtensions.rst | 41 ++++++++++++++-----------------
 1 file changed, 19 insertions(+), 22 deletions(-)

diff --git a/clang/docs/LanguageExtensions.rst b/clang/docs/LanguageExtensions.rst
index 114e742f3561b75..5b50ad615d2c99f 100644
--- a/clang/docs/LanguageExtensions.rst
+++ b/clang/docs/LanguageExtensions.rst
@@ -3540,7 +3540,7 @@ the debugging experience.
 ``__builtin_allow_runtime_check``
 ---------------------------------
 
-``__builtin_allow_runtime_check`` return true if the check at the current
+``__builtin_allow_runtime_check`` returns true if the check at the current
 program location should be executed. It is expected to be used to implement
 ``assert`` like checks which can be safely removed by optimizer.
 
@@ -3560,28 +3560,25 @@ program location should be executed. It is expected to be used to implement
 
 **Description**
 
-``__builtin_allow_runtime_check`` is lowered to ` ``llvm.allow.runtime.check``
+``__builtin_allow_runtime_check`` is lowered to the `llvm.allow.runtime.check
 <https://llvm.org/docs/LangRef.html#llvm-allow-runtime-check-intrinsic>`_
-builtin.
-
-The ``__builtin_allow_runtime_check()`` is expected to be used with control
-flow conditions such as in ``if`` to guard expensive runtime checks. The
-specific rules for selecting permitted checks can differ and are controlled by
-the compiler options.
-
-Flags to control checks:
-* ``-mllvm -lower-allow-check-percentile-cutoff-hot=N`` where N is PGO hotness
-cutoff in range ``[0, 999999]`` to disallow checks in hot code.
-* ``-mllvm -lower-allow-check-random-rate=P`` where P is number in range
-``[0.0, 1.0]`` representation probability of keeping a check.
-* If both flags are specified, ``-lower-allow-check-random-rate`` takes
-precedence.
-* If none is specified, ``__builtin_allow_runtime_check`` is lowered as
-``true``, allowing all checks.
-
-Parameter ``kind`` is a string literal representing a user selected kind for
-guarded check. It's unused now. It will enable kind-specific lowering in future.
-E.g. a higher hotness cutoff can be used for more expensive kind of check.
+intrinsic.
+
+The ``__builtin_allow_runtime_check()`` can be used within constrol structures
+like ``if`` to guard expensive runtime checks. The return value is determined
+by the following compiler options and may differ per call site:
+
+* ``-mllvm -lower-allow-check-percentile-cutoff-hot=N``: Disable checks in hot
+  code marked by the profile summary with a hotness cutoff in the range
+  ``[0, 999999]`` (a larger N disables more checks).
+* ``-mllvm -lower-allow-check-random-rate=P``: Keep a check with probability P,
+  a floating point number in the range ``[0.0, 1.0]``.
+* If both options are specified, a check is disabled if either condition is satisfied.
+* If neither is specified, all checks are allowed.
+
+Parameter ``kind``, currently unused, is a string literal specifying the check
+kind. Future compiler versions may use this to allow for more granular control,
+such as applying different hotness cutoffs to different check kinds.
 
 Query for this feature with ``__has_builtin(__builtin_allow_runtime_check)``.
 

From d3864d946a178807bb5a87d3d1690e213c89e5be Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Mon, 19 Aug 2024 18:20:02 -0700
Subject: [PATCH 433/441] [Driver] Default -msmall-data-limit= to 0 and clean
 up code

D57497 added -msmall-data-limit= as an alias for -G and defaulted it to 8 for
-fno-pic/-fpie.

The behavior is already different from GCC in a few ways:

* GCC doesn't accept -G.
* GCC -fpie doesn't seem to use -msmall-data-limit=.
* GCC emits .srodata.cst* that we don't use (#82214). Writable contents
  caused confusion (https://bugs.chromium.org/p/llvm/issues/detail?id=61)

In addition,

* claiming `-shared` means we don't get a desired `-Wunused-command-line-argument` for `clang --target=riscv64-linux-gnu -fpic -c -shared a.c`.
* -mcmodel=large doesn't work for RISC-V yet, so the special case is strange.
* It's quite unusual to emit a warning when an option (unrelated to relocation model) is used with -fpic.
* We don't want future configurations (Android) to continue adding customization to `SetRISCVSmallDataLimit`.

I believe the extra code just doesn't pull its weight and should be
cleaned up. This patch also changes the default to 0. GP relaxation
users are encouraged to specify these customization options explicitly.

Pull Request: https://github.com/llvm/llvm-project/pull/83093
---
 .../clang/Basic/DiagnosticDriverKinds.td      |  3 --
 clang/lib/Driver/ToolChains/Clang.cpp         | 41 ++-----------------
 .../CodeGen/RISCV/riscv-sdata-module-flag.c   | 11 +----
 clang/test/Driver/riscv-sdata-warning.c       |  4 --
 clang/test/Driver/riscv-sdata.c               |  5 +++
 llvm/lib/Target/RISCV/RISCVTargetObjectFile.h |  2 +-
 6 files changed, 12 insertions(+), 54 deletions(-)
 delete mode 100644 clang/test/Driver/riscv-sdata-warning.c
 create mode 100644 clang/test/Driver/riscv-sdata.c

diff --git a/clang/include/clang/Basic/DiagnosticDriverKinds.td b/clang/include/clang/Basic/DiagnosticDriverKinds.td
index 92a602829933ced..ba90742fbdaabcf 100644
--- a/clang/include/clang/Basic/DiagnosticDriverKinds.td
+++ b/clang/include/clang/Basic/DiagnosticDriverKinds.td
@@ -604,9 +604,6 @@ def warn_drv_unsupported_gpopt : Warning<
   "ignoring '-mgpopt' option as it cannot be used with %select{|the implicit"
   " usage of }0-mabicalls">,
   InGroup<UnsupportedGPOpt>;
-def warn_drv_unsupported_sdata : Warning<
-  "ignoring '-msmall-data-limit=' with -mcmodel=large for -fpic or RV64">,
-  InGroup<OptionIgnored>;
 def warn_drv_unsupported_longcalls : Warning<
   "ignoring '-mlong-calls' option as it is not currently supported with "
   "%select{|the implicit usage of }0-mabicalls">,
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index a5a87db97e96b48..a8606a866bdff64 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -2126,42 +2126,6 @@ void Clang::AddPPCTargetArgs(const ArgList &Args,
   }
 }
 
-static void SetRISCVSmallDataLimit(const ToolChain &TC, const ArgList &Args,
-                                   ArgStringList &CmdArgs) {
-  const Driver &D = TC.getDriver();
-  const llvm::Triple &Triple = TC.getTriple();
-  // Default small data limitation is eight.
-  const char *SmallDataLimit = "8";
-  // Get small data limitation.
-  if (Args.getLastArg(options::OPT_shared, options::OPT_fpic,
-                      options::OPT_fPIC)) {
-    // Not support linker relaxation for PIC.
-    SmallDataLimit = "0";
-    if (Args.hasArg(options::OPT_G)) {
-      D.Diag(diag::warn_drv_unsupported_sdata);
-    }
-  } else if (Args.getLastArgValue(options::OPT_mcmodel_EQ)
-                 .equals_insensitive("large") &&
-             (Triple.getArch() == llvm::Triple::riscv64)) {
-    // Not support linker relaxation for RV64 with large code model.
-    SmallDataLimit = "0";
-    if (Args.hasArg(options::OPT_G)) {
-      D.Diag(diag::warn_drv_unsupported_sdata);
-    }
-  } else if (Triple.isAndroid()) {
-    // GP relaxation is not supported on Android.
-    SmallDataLimit = "0";
-    if (Args.hasArg(options::OPT_G)) {
-      D.Diag(diag::warn_drv_unsupported_sdata);
-    }
-  } else if (Arg *A = Args.getLastArg(options::OPT_G)) {
-    SmallDataLimit = A->getValue();
-  }
-  // Forward the -msmall-data-limit= option.
-  CmdArgs.push_back("-msmall-data-limit");
-  CmdArgs.push_back(SmallDataLimit);
-}
-
 void Clang::AddRISCVTargetArgs(const ArgList &Args,
                                ArgStringList &CmdArgs) const {
   const llvm::Triple &Triple = getToolChain().getTriple();
@@ -2170,7 +2134,10 @@ void Clang::AddRISCVTargetArgs(const ArgList &Args,
   CmdArgs.push_back("-target-abi");
   CmdArgs.push_back(ABIName.data());
 
-  SetRISCVSmallDataLimit(getToolChain(), Args, CmdArgs);
+  if (Arg *A = Args.getLastArg(options::OPT_G)) {
+    CmdArgs.push_back("-msmall-data-limit");
+    CmdArgs.push_back(A->getValue());
+  }
 
   if (!Args.hasFlag(options::OPT_mimplicit_float,
                     options::OPT_mno_implicit_float, true))
diff --git a/clang/test/CodeGen/RISCV/riscv-sdata-module-flag.c b/clang/test/CodeGen/RISCV/riscv-sdata-module-flag.c
index 9bd69e7995877e4..83beabe85b1f4dd 100644
--- a/clang/test/CodeGen/RISCV/riscv-sdata-module-flag.c
+++ b/clang/test/CodeGen/RISCV/riscv-sdata-module-flag.c
@@ -21,23 +21,19 @@
 // RUN:   | FileCheck %s -check-prefix=RV64-S2G4
 // RUN: %clang --target=riscv64-unknown-elf %s -S -emit-llvm -msmall-data-threshold=16 -o - \
 // RUN:   | FileCheck %s -check-prefix=RV64-T16
-// RUN: %clang --target=riscv64-linux-android %s -S -emit-llvm -o - \
-// RUN:   | FileCheck %s -check-prefix=RV64-ANDROID
-// RUN: %clang --target=riscv64-linux-android %s -S -emit-llvm -msmall-data-limit=8 -o - \
-// RUN:   | FileCheck %s -check-prefix=RV64-ANDROID
 // RUN: %clang --target=riscv64-unknown-elf %s -S -emit-llvm -fpic -o - \
 // RUN:   | FileCheck %s -check-prefix=RV64-PIC
 
 void test(void) {}
 
-// RV32-DEFAULT: !{i32 8, !"SmallDataLimit", i32 8}
+// RV32-DEFAULT: !{i32 8, !"SmallDataLimit", i32 0}
 // RV32-G4:      !{i32 8, !"SmallDataLimit", i32 4}
 // RV32-S0:      !{i32 8, !"SmallDataLimit", i32 0}
 // RV32-S2G4:    !{i32 8, !"SmallDataLimit", i32 4}
 // RV32-T16:     !{i32 8, !"SmallDataLimit", i32 16}
 // RV32-PIC:     !{i32 8, !"SmallDataLimit", i32 0}
 
-// RV64-DEFAULT: !{i32 8, !"SmallDataLimit", i32 8}
+// RV64-DEFAULT: !{i32 8, !"SmallDataLimit", i32 0}
 // RV64-G4:      !{i32 8, !"SmallDataLimit", i32 4}
 // RV64-S0:      !{i32 8, !"SmallDataLimit", i32 0}
 // RV64-S2G4:    !{i32 8, !"SmallDataLimit", i32 4}
@@ -48,6 +44,3 @@ void test(void) {}
 // The value will be passed by module flag instead of target feature.
 // RV32-S0-NOT: +small-data-limit=
 // RV64-S0-NOT: +small-data-limit=
-
-// RV64-ANDROID-NOT: small-data-limit
-// RV64-ANDROID:   !{i32 8, !"SmallDataLimit", i32 0}
diff --git a/clang/test/Driver/riscv-sdata-warning.c b/clang/test/Driver/riscv-sdata-warning.c
deleted file mode 100644
index ace9a32ee116fd8..000000000000000
--- a/clang/test/Driver/riscv-sdata-warning.c
+++ /dev/null
@@ -1,4 +0,0 @@
-// REQUIRES: riscv-registered-target
-// RUN: %clang -S --target=riscv32-unknown-elf -fpic -msmall-data-limit=8 %s 2>&1 \
-// RUN:   | FileCheck -check-prefix=CHECK-PIC-SDATA %s
-// CHECK-PIC-SDATA: warning: ignoring '-msmall-data-limit=' with -mcmodel=large for -fpic or RV64
diff --git a/clang/test/Driver/riscv-sdata.c b/clang/test/Driver/riscv-sdata.c
new file mode 100644
index 000000000000000..84cd109813db4ca
--- /dev/null
+++ b/clang/test/Driver/riscv-sdata.c
@@ -0,0 +1,5 @@
+// RUN: %clang -### -S --target=riscv64 %s 2>&1 | FileCheck %s
+// RUN: %clang -### -S --target=riscv64 -msmall-data-limit=8 %s 2>&1 | FileCheck %s --check-prefix=EIGHT
+
+// CHECK-NOT: "-msmall-data-limit"
+// EIGHT: "-msmall-data-limit" "8"
diff --git a/llvm/lib/Target/RISCV/RISCVTargetObjectFile.h b/llvm/lib/Target/RISCV/RISCVTargetObjectFile.h
index 05e61ac874abba2..ff7e3e4c752c330 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetObjectFile.h
+++ b/llvm/lib/Target/RISCV/RISCVTargetObjectFile.h
@@ -22,7 +22,7 @@ class RISCVELFTargetObjectFile : public TargetLoweringObjectFileELF {
   MCSection *SmallROData16Section;
   MCSection *SmallROData32Section;
   MCSection *SmallBSSSection;
-  unsigned SSThreshold = 8;
+  unsigned SSThreshold = 0;
 
 public:
   unsigned getTextSectionAlignment() const override;

From 52bfb2611f8d30fae3306c652af7ba5c7e88c147 Mon Sep 17 00:00:00 2001
From: Dmitry Yanovsky <kerambyte@gmail.com>
Date: Tue, 20 Aug 2024 02:53:41 +0100
Subject: [PATCH 434/441] [ADT] Fix a minor build error (#104840)

A quick follow-up fix for
https://github.com/llvm/llvm-project/pull/99403

Buildbot
[reported](https://lab.llvm.org/buildbot/#/builders/168/builds/2330) an
error:

```
/home/buildbots/llvm-external-buildbots/workers/ppc64le-lld-multistage-test/ppc64le-lld-multistage-test/llvm-project/llvm/unittests/ADT/FunctionExtrasTest.cpp:320:8: error: variable 'ptr' is uninitialized when used here [-Werror,-Wuninitialized]
  320 |       [ptr](void *self) {
      |        ^~~
/home/buildbots/llvm-external-buildbots/workers/ppc64le-lld-multistage-test/ppc64le-lld-multistage-test/llvm-project/llvm/unittests/ADT/FunctionExtrasTest.cpp:318:12: note: initialize the variable 'ptr' to silence this warning
  318 |   void *ptr;
      |            ^
      |             = nullptr
1 error generated.
```

So that PR does exactly what's sugested.
---
 llvm/unittests/ADT/FunctionExtrasTest.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/unittests/ADT/FunctionExtrasTest.cpp b/llvm/unittests/ADT/FunctionExtrasTest.cpp
index f64b888dbe1591c..9809a92daac7246 100644
--- a/llvm/unittests/ADT/FunctionExtrasTest.cpp
+++ b/llvm/unittests/ADT/FunctionExtrasTest.cpp
@@ -316,7 +316,7 @@ TEST(UniqueFunctionTest, InlineStorageWorks) {
   // We do assume a couple of implementation details of the unique_function here:
   //  - It can store certain small-enough payload inline
   //  - Inline storage size is at least >= sizeof(void*)
-  void *ptr;
+  void *ptr = nullptr;
   unique_function<void(void *)> UniqueFunctionWithInlineStorage{
       [ptr](void *self) {
         auto mid = reinterpret_cast<uintptr_t>(&ptr);

From 06fd808654ff814c2f79c2619a5b2be50c56521b Mon Sep 17 00:00:00 2001
From: Christopher Bate <cbate@nvidia.com>
Date: Tue, 20 Aug 2024 03:46:29 +0000
Subject: [PATCH 435/441] Revert "[mlir] NFC: fix dependence of
 (Tensor|Linalg|MemRef|Complex) dialects on LLVM Dialect and LLVM Core in
 CMake build (#104832)"

This reverts commit 43b508566799751aa180f1eaaafc5be693f2f1ae since it
caused the build to break with BUILD_SHARED_LIBS=ON.
---
 mlir/lib/Conversion/AffineToStandard/CMakeLists.txt       | 3 +++
 mlir/lib/Conversion/ControlFlowToSCF/CMakeLists.txt       | 6 +++++-
 mlir/lib/Conversion/ControlFlowToSCF/ControlFlowToSCF.cpp | 1 +
 mlir/lib/Conversion/SCFToControlFlow/CMakeLists.txt       | 5 ++++-
 mlir/lib/Conversion/SCFToControlFlow/SCFToControlFlow.cpp | 5 +++--
 mlir/lib/Conversion/VectorToSCF/CMakeLists.txt            | 4 ++++
 mlir/lib/Dialect/Affine/Transforms/CMakeLists.txt         | 1 +
 mlir/lib/Dialect/Complex/IR/CMakeLists.txt                | 1 +
 mlir/lib/Dialect/Linalg/Transforms/CMakeLists.txt         | 2 ++
 mlir/lib/Dialect/MemRef/IR/CMakeLists.txt                 | 3 +++
 mlir/lib/Dialect/MemRef/TransformOps/CMakeLists.txt       | 1 -
 mlir/lib/Dialect/Tensor/IR/CMakeLists.txt                 | 3 +++
 12 files changed, 30 insertions(+), 5 deletions(-)

diff --git a/mlir/lib/Conversion/AffineToStandard/CMakeLists.txt b/mlir/lib/Conversion/AffineToStandard/CMakeLists.txt
index cd8193aee8da326..f41e3ca27ee4dde 100644
--- a/mlir/lib/Conversion/AffineToStandard/CMakeLists.txt
+++ b/mlir/lib/Conversion/AffineToStandard/CMakeLists.txt
@@ -7,6 +7,9 @@ add_mlir_conversion_library(MLIRAffineToStandard
   DEPENDS
   MLIRConversionPassIncGen
 
+  LINK_COMPONENTS
+  Core
+
   LINK_LIBS PUBLIC
   MLIRAffineDialect
   MLIRAffineTransforms
diff --git a/mlir/lib/Conversion/ControlFlowToSCF/CMakeLists.txt b/mlir/lib/Conversion/ControlFlowToSCF/CMakeLists.txt
index e7024dfbd5ee325..679eca3dfa988c5 100644
--- a/mlir/lib/Conversion/ControlFlowToSCF/CMakeLists.txt
+++ b/mlir/lib/Conversion/ControlFlowToSCF/CMakeLists.txt
@@ -5,7 +5,11 @@ add_mlir_conversion_library(MLIRControlFlowToSCF
   ${MLIR_MAIN_INCLUDE_DIR}/mlir/Conversion/ControlFlowToSCF
 
   DEPENDS
-  MLIRConversionPassIncGen  
+  MLIRConversionPassIncGen
+  intrinsics_gen
+
+  LINK_COMPONENTS
+  Core
 
   LINK_LIBS PUBLIC
   MLIRAnalysis
diff --git a/mlir/lib/Conversion/ControlFlowToSCF/ControlFlowToSCF.cpp b/mlir/lib/Conversion/ControlFlowToSCF/ControlFlowToSCF.cpp
index 1c592d665f3e4c5..d3ee89743da9db5 100644
--- a/mlir/lib/Conversion/ControlFlowToSCF/ControlFlowToSCF.cpp
+++ b/mlir/lib/Conversion/ControlFlowToSCF/ControlFlowToSCF.cpp
@@ -16,6 +16,7 @@
 #include "mlir/Dialect/ControlFlow/IR/ControlFlow.h"
 #include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
+#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/Dialect/SCF/IR/SCF.h"
 #include "mlir/Dialect/UB/IR/UBOps.h"
 #include "mlir/Pass/Pass.h"
diff --git a/mlir/lib/Conversion/SCFToControlFlow/CMakeLists.txt b/mlir/lib/Conversion/SCFToControlFlow/CMakeLists.txt
index ff64b8fcca0830e..63c5199af9290c9 100644
--- a/mlir/lib/Conversion/SCFToControlFlow/CMakeLists.txt
+++ b/mlir/lib/Conversion/SCFToControlFlow/CMakeLists.txt
@@ -5,7 +5,10 @@ add_mlir_conversion_library(MLIRSCFToControlFlow
   ${MLIR_MAIN_INCLUDE_DIR}/mlir/Conversion/SCFToControlFlow
 
   DEPENDS
-  MLIRConversionPassIncGen  
+  MLIRConversionPassIncGen
+
+  LINK_COMPONENTS
+  Core
 
   LINK_LIBS PUBLIC
   MLIRArithDialect
diff --git a/mlir/lib/Conversion/SCFToControlFlow/SCFToControlFlow.cpp b/mlir/lib/Conversion/SCFToControlFlow/SCFToControlFlow.cpp
index cef96c206dda5c8..2372ab5b82a772e 100644
--- a/mlir/lib/Conversion/SCFToControlFlow/SCFToControlFlow.cpp
+++ b/mlir/lib/Conversion/SCFToControlFlow/SCFToControlFlow.cpp
@@ -15,6 +15,7 @@
 
 #include "mlir/Dialect/Arith/IR/Arith.h"
 #include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h"
+#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/Dialect/SCF/IR/SCF.h"
 #include "mlir/Dialect/SCF/Transforms/Transforms.h"
 #include "mlir/IR/Builders.h"
@@ -378,8 +379,8 @@ LogicalResult ForLowering::matchAndRewrite(ForOp forOp,
   // llvm.loop_annotation attribute.
   SmallVector<NamedAttribute> llvmAttrs;
   llvm::copy_if(forOp->getAttrs(), std::back_inserter(llvmAttrs),
-                [](NamedAttribute attr) {
-                  return attr.getValue().getDialect().getNamespace() == "llvm";
+                [](auto attr) {
+                  return isa<LLVM::LLVMDialect>(attr.getValue().getDialect());
                 });
   condBranchOp->setDiscardableAttrs(llvmAttrs);
   // The result of the loop operation is the values of the condition block
diff --git a/mlir/lib/Conversion/VectorToSCF/CMakeLists.txt b/mlir/lib/Conversion/VectorToSCF/CMakeLists.txt
index 0915270e453dfdb..83bbc4a4bc395a6 100644
--- a/mlir/lib/Conversion/VectorToSCF/CMakeLists.txt
+++ b/mlir/lib/Conversion/VectorToSCF/CMakeLists.txt
@@ -4,8 +4,12 @@ add_mlir_conversion_library(MLIRVectorToSCF
   ADDITIONAL_HEADER_DIRS
   ${MLIR_MAIN_INCLUDE_DIR}/mlir/Conversion/VectorToSCF
 
+  LINK_COMPONENTS
+  Core
+
   LINK_LIBS PUBLIC
   MLIRArithDialect
+  MLIRLLVMDialect
   MLIRMemRefDialect
   MLIRTransforms
   MLIRVectorDialect
diff --git a/mlir/lib/Dialect/Affine/Transforms/CMakeLists.txt b/mlir/lib/Dialect/Affine/Transforms/CMakeLists.txt
index 772f15335d907f2..60d14a373348d38 100644
--- a/mlir/lib/Dialect/Affine/Transforms/CMakeLists.txt
+++ b/mlir/lib/Dialect/Affine/Transforms/CMakeLists.txt
@@ -39,5 +39,6 @@ add_mlir_dialect_library(MLIRAffineTransforms
   MLIRValueBoundsOpInterface
   MLIRVectorDialect
   MLIRVectorUtils
+  MLIRVectorToLLVM
   )
 
diff --git a/mlir/lib/Dialect/Complex/IR/CMakeLists.txt b/mlir/lib/Dialect/Complex/IR/CMakeLists.txt
index a90f34ec1684d90..3ee0d26f3225f72 100644
--- a/mlir/lib/Dialect/Complex/IR/CMakeLists.txt
+++ b/mlir/lib/Dialect/Complex/IR/CMakeLists.txt
@@ -10,6 +10,7 @@ add_mlir_dialect_library(MLIRComplexDialect
   MLIRComplexAttributesIncGen
 
   LINK_LIBS PUBLIC
+  MLIRArithAttrToLLVMConversion
   MLIRArithDialect
   MLIRDialect
   MLIRInferTypeOpInterface
diff --git a/mlir/lib/Dialect/Linalg/Transforms/CMakeLists.txt b/mlir/lib/Dialect/Linalg/Transforms/CMakeLists.txt
index 47af392def94ac0..a7dcc29b5b9be18 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/CMakeLists.txt
+++ b/mlir/lib/Dialect/Linalg/Transforms/CMakeLists.txt
@@ -60,6 +60,7 @@ add_mlir_dialect_library(MLIRLinalgTransforms
   MLIRDestinationStyleOpInterface
   MLIRDialectUtils
   MLIRFuncDialect
+  MLIRFuncToLLVM
   MLIRFuncTransforms
   MLIRIndexDialect
   MLIRInferTypeOpInterface
@@ -86,5 +87,6 @@ add_mlir_dialect_library(MLIRLinalgTransforms
   MLIRVectorDialect
   MLIRVectorTransforms
   MLIRVectorUtils
+  MLIRX86VectorTransforms
   MLIRVectorToSCF
 )
diff --git a/mlir/lib/Dialect/MemRef/IR/CMakeLists.txt b/mlir/lib/Dialect/MemRef/IR/CMakeLists.txt
index 845914ebd107a26..fd2fed28badd63f 100644
--- a/mlir/lib/Dialect/MemRef/IR/CMakeLists.txt
+++ b/mlir/lib/Dialect/MemRef/IR/CMakeLists.txt
@@ -10,6 +10,9 @@ add_mlir_dialect_library(MLIRMemRefDialect
   DEPENDS
   MLIRMemRefOpsIncGen
 
+  LINK_COMPONENTS
+  Core
+
   LINK_LIBS PUBLIC
   MLIRArithDialect
   MLIRArithUtils
diff --git a/mlir/lib/Dialect/MemRef/TransformOps/CMakeLists.txt b/mlir/lib/Dialect/MemRef/TransformOps/CMakeLists.txt
index 4c9e3d3c813f7a6..e379663f3b0a01f 100644
--- a/mlir/lib/Dialect/MemRef/TransformOps/CMakeLists.txt
+++ b/mlir/lib/Dialect/MemRef/TransformOps/CMakeLists.txt
@@ -19,5 +19,4 @@ add_mlir_dialect_library(MLIRMemRefTransformOps
   MLIRNVGPUDialect
   MLIRTransformDialect
   MLIRVectorDialect
-  MLIRVectorTransforms
 )
diff --git a/mlir/lib/Dialect/Tensor/IR/CMakeLists.txt b/mlir/lib/Dialect/Tensor/IR/CMakeLists.txt
index cfdd3847761a49a..549b9f10388bd47 100644
--- a/mlir/lib/Dialect/Tensor/IR/CMakeLists.txt
+++ b/mlir/lib/Dialect/Tensor/IR/CMakeLists.txt
@@ -17,6 +17,9 @@ add_mlir_dialect_library(MLIRTensorDialect
   DEPENDS
   MLIRTensorOpsIncGen
 
+  LINK_COMPONENTS
+  Core
+
   LINK_LIBS PUBLIC
   MLIRAffineDialect
   MLIRArithDialect

From ca148b21505e97f07787c13ec00ffc086d4658d0 Mon Sep 17 00:00:00 2001
From: Timm Baeder <tbaeder@redhat.com>
Date: Tue, 20 Aug 2024 06:08:53 +0200
Subject: [PATCH 436/441] [clang][bytecode] Support ObjC blocks (#104551)

I started out by adding a new pointer type for blocks, and I was fully
prepared to compile their AST to bytecode and later call them.

... then I found out that the current interpreter doesn't support
calling blocks at all. So we reuse `Function` to support sources other
than `FunctionDecl`s and classify `BlockPointerType` as `PT_FnPtr`.
---
 clang/lib/AST/ByteCode/ByteCodeEmitter.cpp | 42 +++++++++++++++++++
 clang/lib/AST/ByteCode/ByteCodeEmitter.h   |  1 +
 clang/lib/AST/ByteCode/Compiler.cpp        | 15 +++++--
 clang/lib/AST/ByteCode/Compiler.h          |  1 +
 clang/lib/AST/ByteCode/Context.cpp         |  2 +-
 clang/lib/AST/ByteCode/Function.cpp        | 14 ++++---
 clang/lib/AST/ByteCode/Function.h          | 49 ++++++++++++++++------
 clang/lib/AST/ByteCode/FunctionPointer.h   |  7 +++-
 clang/lib/AST/ByteCode/Interp.h            |  2 +-
 clang/test/Sema/block-misc.c               |  1 +
 clang/test/Sema/block-return.c             |  1 +
 clang/test/SemaCXX/consteval-cleanup.cpp   |  1 +
 12 files changed, 110 insertions(+), 26 deletions(-)

diff --git a/clang/lib/AST/ByteCode/ByteCodeEmitter.cpp b/clang/lib/AST/ByteCode/ByteCodeEmitter.cpp
index 1f6bdaff50b5bff..35ae1547939fdd5 100644
--- a/clang/lib/AST/ByteCode/ByteCodeEmitter.cpp
+++ b/clang/lib/AST/ByteCode/ByteCodeEmitter.cpp
@@ -195,6 +195,48 @@ Function *ByteCodeEmitter::compileFunc(const FunctionDecl *FuncDecl) {
   return Func;
 }
 
+/// Compile an ObjC block, i.e. ^(){}, that thing.
+///
+/// FIXME: We do not support calling the block though, so we create a function
+/// here but do not compile any code for it.
+Function *ByteCodeEmitter::compileObjCBlock(const BlockExpr *BE) {
+  const BlockDecl *BD = BE->getBlockDecl();
+  // Set up argument indices.
+  unsigned ParamOffset = 0;
+  SmallVector<PrimType, 8> ParamTypes;
+  SmallVector<unsigned, 8> ParamOffsets;
+  llvm::DenseMap<unsigned, Function::ParamDescriptor> ParamDescriptors;
+
+  // Assign descriptors to all parameters.
+  // Composite objects are lowered to pointers.
+  for (const ParmVarDecl *PD : BD->parameters()) {
+    std::optional<PrimType> T = Ctx.classify(PD->getType());
+    PrimType PT = T.value_or(PT_Ptr);
+    Descriptor *Desc = P.createDescriptor(PD, PT);
+    ParamDescriptors.insert({ParamOffset, {PT, Desc}});
+    Params.insert({PD, {ParamOffset, T != std::nullopt}});
+    ParamOffsets.push_back(ParamOffset);
+    ParamOffset += align(primSize(PT));
+    ParamTypes.push_back(PT);
+  }
+
+  if (BD->hasCaptures())
+    return nullptr;
+
+  // Create a handle over the emitted code.
+  Function *Func =
+      P.createFunction(BE, ParamOffset, std::move(ParamTypes),
+                       std::move(ParamDescriptors), std::move(ParamOffsets),
+                       /*HasThisPointer=*/false, /*HasRVO=*/false,
+                       /*IsUnevaluatedBuiltin=*/false);
+
+  assert(Func);
+  Func->setDefined(true);
+  // We don't compile the BlockDecl code at all right now.
+  Func->setIsFullyCompiled(true);
+  return Func;
+}
+
 Scope::Local ByteCodeEmitter::createLocal(Descriptor *D) {
   NextLocalOffset += sizeof(Block);
   unsigned Location = NextLocalOffset;
diff --git a/clang/lib/AST/ByteCode/ByteCodeEmitter.h b/clang/lib/AST/ByteCode/ByteCodeEmitter.h
index 59e12defd8e52b5..7cbbe651699b34b 100644
--- a/clang/lib/AST/ByteCode/ByteCodeEmitter.h
+++ b/clang/lib/AST/ByteCode/ByteCodeEmitter.h
@@ -32,6 +32,7 @@ class ByteCodeEmitter {
 public:
   /// Compiles the function into the module.
   Function *compileFunc(const FunctionDecl *FuncDecl);
+  Function *compileObjCBlock(const BlockExpr *BE);
 
 protected:
   ByteCodeEmitter(Context &Ctx, Program &P) : Ctx(Ctx), P(P) {}
diff --git a/clang/lib/AST/ByteCode/Compiler.cpp b/clang/lib/AST/ByteCode/Compiler.cpp
index c839364d4c8d25c..131b83ae8eb3976 100644
--- a/clang/lib/AST/ByteCode/Compiler.cpp
+++ b/clang/lib/AST/ByteCode/Compiler.cpp
@@ -391,8 +391,6 @@ bool Compiler<Emitter>::VisitCastExpr(const CastExpr *CE) {
       return this->emitPop(T, CE);
 
     QualType PtrType = CE->getType();
-    assert(PtrType->isPointerType());
-
     const Descriptor *Desc;
     if (std::optional<PrimType> T = classify(PtrType->getPointeeType()))
       Desc = P.createDescriptor(SubExpr, *T);
@@ -2242,8 +2240,6 @@ bool Compiler<Emitter>::VisitExprWithCleanups(const ExprWithCleanups *E) {
   LocalScope<Emitter> ES(this);
   const Expr *SubExpr = E->getSubExpr();
 
-  assert(E->getNumObjects() == 0 && "TODO: Implement cleanups");
-
   return this->delegate(SubExpr) && ES.destroyLocals(E);
 }
 
@@ -2913,6 +2909,17 @@ bool Compiler<Emitter>::VisitCXXDeleteExpr(const CXXDeleteExpr *E) {
   return this->emitFree(E->isArrayForm(), E);
 }
 
+template <class Emitter>
+bool Compiler<Emitter>::VisitBlockExpr(const BlockExpr *E) {
+  const Function *Func = nullptr;
+  if (auto F = Compiler<ByteCodeEmitter>(Ctx, P).compileObjCBlock(E))
+    Func = F;
+
+  if (!Func)
+    return false;
+  return this->emitGetFnPtr(Func, E);
+}
+
 template <class Emitter>
 bool Compiler<Emitter>::VisitExpressionTraitExpr(const ExpressionTraitExpr *E) {
   assert(Ctx.getLangOpts().CPlusPlus);
diff --git a/clang/lib/AST/ByteCode/Compiler.h b/clang/lib/AST/ByteCode/Compiler.h
index 74bfce5f241f28e..5acfe3c41796c45 100644
--- a/clang/lib/AST/ByteCode/Compiler.h
+++ b/clang/lib/AST/ByteCode/Compiler.h
@@ -199,6 +199,7 @@ class Compiler : public ConstStmtVisitor<Compiler<Emitter>, bool>,
   bool VisitStmtExpr(const StmtExpr *E);
   bool VisitCXXNewExpr(const CXXNewExpr *E);
   bool VisitCXXDeleteExpr(const CXXDeleteExpr *E);
+  bool VisitBlockExpr(const BlockExpr *E);
 
   // Statements.
   bool visitCompoundStmt(const CompoundStmt *S);
diff --git a/clang/lib/AST/ByteCode/Context.cpp b/clang/lib/AST/ByteCode/Context.cpp
index 48ae363f651f5a4..e682d87b703ad77 100644
--- a/clang/lib/AST/ByteCode/Context.cpp
+++ b/clang/lib/AST/ByteCode/Context.cpp
@@ -179,7 +179,7 @@ std::optional<PrimType> Context::classify(QualType T) const {
     return PT_MemberPtr;
 
   if (T->isFunctionPointerType() || T->isFunctionReferenceType() ||
-      T->isFunctionType())
+      T->isFunctionType() || T->isBlockPointerType())
     return PT_FnPtr;
 
   if (T->isPointerOrReferenceType() || T->isObjCObjectPointerType())
diff --git a/clang/lib/AST/ByteCode/Function.cpp b/clang/lib/AST/ByteCode/Function.cpp
index fc3345cbe123f88..e3fab3f6720b41e 100644
--- a/clang/lib/AST/ByteCode/Function.cpp
+++ b/clang/lib/AST/ByteCode/Function.cpp
@@ -16,15 +16,18 @@
 using namespace clang;
 using namespace clang::interp;
 
-Function::Function(Program &P, const FunctionDecl *F, unsigned ArgSize,
+Function::Function(Program &P, FunctionDeclTy Source, unsigned ArgSize,
                    llvm::SmallVectorImpl<PrimType> &&ParamTypes,
                    llvm::DenseMap<unsigned, ParamDescriptor> &&Params,
                    llvm::SmallVectorImpl<unsigned> &&ParamOffsets,
                    bool HasThisPointer, bool HasRVO, bool UnevaluatedBuiltin)
-    : P(P), F(F), ArgSize(ArgSize), ParamTypes(std::move(ParamTypes)),
+    : P(P), Source(Source), ArgSize(ArgSize), ParamTypes(std::move(ParamTypes)),
       Params(std::move(Params)), ParamOffsets(std::move(ParamOffsets)),
-      HasThisPointer(HasThisPointer), HasRVO(HasRVO), Variadic(F->isVariadic()),
-      IsUnevaluatedBuiltin(UnevaluatedBuiltin) {}
+      HasThisPointer(HasThisPointer), HasRVO(HasRVO),
+      IsUnevaluatedBuiltin(UnevaluatedBuiltin) {
+  if (const auto *F = Source.dyn_cast<const FunctionDecl *>())
+    Variadic = F->isVariadic();
+}
 
 Function::ParamDescriptor Function::getParamDescriptor(unsigned Offset) const {
   auto It = Params.find(Offset);
@@ -45,7 +48,8 @@ SourceInfo Function::getSource(CodePtr PC) const {
 }
 
 bool Function::isVirtual() const {
-  if (const auto *M = dyn_cast<CXXMethodDecl>(F))
+  if (const auto *M = dyn_cast_if_present<CXXMethodDecl>(
+          Source.dyn_cast<const FunctionDecl *>()))
     return M->isVirtual();
   return false;
 }
diff --git a/clang/lib/AST/ByteCode/Function.h b/clang/lib/AST/ByteCode/Function.h
index 625d5af0407a721..f254db20d4f5946 100644
--- a/clang/lib/AST/ByteCode/Function.h
+++ b/clang/lib/AST/ByteCode/Function.h
@@ -20,6 +20,7 @@
 #include "clang/AST/ASTLambda.h"
 #include "clang/AST/Attr.h"
 #include "clang/AST/Decl.h"
+#include "llvm/ADT/PointerUnion.h"
 #include "llvm/Support/raw_ostream.h"
 
 namespace clang {
@@ -55,6 +56,9 @@ class Scope final {
   LocalVectorTy Descriptors;
 };
 
+using FunctionDeclTy =
+    llvm::PointerUnion<const FunctionDecl *, const BlockExpr *>;
+
 /// Bytecode function.
 ///
 /// Contains links to the bytecode of the function, as well as metadata
@@ -89,15 +93,20 @@ class Function final {
   CodePtr getCodeEnd() const { return Code.data() + Code.size(); }
 
   /// Returns the original FunctionDecl.
-  const FunctionDecl *getDecl() const { return F; }
+  const FunctionDecl *getDecl() const {
+    return Source.dyn_cast<const FunctionDecl *>();
+  }
+  const BlockExpr *getExpr() const {
+    return Source.dyn_cast<const BlockExpr *>();
+  }
 
   /// Returns the name of the function decl this code
   /// was generated for.
   const std::string getName() const {
-    if (!F)
+    if (!Source)
       return "<<expr>>";
 
-    return F->getQualifiedNameAsString();
+    return Source.get<const FunctionDecl *>()->getQualifiedNameAsString();
   }
 
   /// Returns a parameter descriptor.
@@ -135,13 +144,20 @@ class Function final {
   bool isVirtual() const;
 
   /// Checks if the function is a constructor.
-  bool isConstructor() const { return isa<CXXConstructorDecl>(F); }
+  bool isConstructor() const {
+    return isa_and_nonnull<CXXConstructorDecl>(
+        Source.dyn_cast<const FunctionDecl *>());
+  }
   /// Checks if the function is a destructor.
-  bool isDestructor() const { return isa<CXXDestructorDecl>(F); }
+  bool isDestructor() const {
+    return isa_and_nonnull<CXXDestructorDecl>(
+        Source.dyn_cast<const FunctionDecl *>());
+  }
 
   /// Returns the parent record decl, if any.
   const CXXRecordDecl *getParentDecl() const {
-    if (const auto *MD = dyn_cast<CXXMethodDecl>(F))
+    if (const auto *MD = dyn_cast_if_present<CXXMethodDecl>(
+            Source.dyn_cast<const FunctionDecl *>()))
       return MD->getParent();
     return nullptr;
   }
@@ -149,7 +165,8 @@ class Function final {
   /// Returns whether this function is a lambda static invoker,
   /// which we generate custom byte code for.
   bool isLambdaStaticInvoker() const {
-    if (const auto *MD = dyn_cast<CXXMethodDecl>(F))
+    if (const auto *MD = dyn_cast_if_present<CXXMethodDecl>(
+            Source.dyn_cast<const FunctionDecl *>()))
       return MD->isLambdaStaticInvoker();
     return false;
   }
@@ -157,7 +174,8 @@ class Function final {
   /// Returns whether this function is the call operator
   /// of a lambda record decl.
   bool isLambdaCallOperator() const {
-    if (const auto *MD = dyn_cast<CXXMethodDecl>(F))
+    if (const auto *MD = dyn_cast_if_present<CXXMethodDecl>(
+            Source.dyn_cast<const FunctionDecl *>()))
       return clang::isLambdaCallOperator(MD);
     return false;
   }
@@ -175,9 +193,13 @@ class Function final {
 
   bool isVariadic() const { return Variadic; }
 
-  unsigned getBuiltinID() const { return F->getBuiltinID(); }
+  unsigned getBuiltinID() const {
+    return Source.get<const FunctionDecl *>()->getBuiltinID();
+  }
 
-  bool isBuiltin() const { return F->getBuiltinID() != 0; }
+  bool isBuiltin() const {
+    return Source.get<const FunctionDecl *>()->getBuiltinID() != 0;
+  }
 
   bool isUnevaluatedBuiltin() const { return IsUnevaluatedBuiltin; }
 
@@ -194,7 +216,8 @@ class Function final {
   }
 
   bool isThisPointerExplicit() const {
-    if (const auto *MD = dyn_cast<CXXMethodDecl>(F))
+    if (const auto *MD = dyn_cast_if_present<CXXMethodDecl>(
+            Source.dyn_cast<const FunctionDecl *>()))
       return MD->isExplicitObjectMemberFunction();
     return false;
   }
@@ -205,7 +228,7 @@ class Function final {
 
 private:
   /// Construct a function representing an actual function.
-  Function(Program &P, const FunctionDecl *F, unsigned ArgSize,
+  Function(Program &P, FunctionDeclTy Source, unsigned ArgSize,
            llvm::SmallVectorImpl<PrimType> &&ParamTypes,
            llvm::DenseMap<unsigned, ParamDescriptor> &&Params,
            llvm::SmallVectorImpl<unsigned> &&ParamOffsets, bool HasThisPointer,
@@ -233,7 +256,7 @@ class Function final {
   /// Program reference.
   Program &P;
   /// Declaration this function was compiled from.
-  const FunctionDecl *F;
+  FunctionDeclTy Source;
   /// Local area size: storage + metadata.
   unsigned FrameSize = 0;
   /// Size of the argument stack.
diff --git a/clang/lib/AST/ByteCode/FunctionPointer.h b/clang/lib/AST/ByteCode/FunctionPointer.h
index d92cd32933fcdce..c9bdfbee55441a3 100644
--- a/clang/lib/AST/ByteCode/FunctionPointer.h
+++ b/clang/lib/AST/ByteCode/FunctionPointer.h
@@ -33,7 +33,7 @@ class FunctionPointer final {
   bool isZero() const { return !Func; }
   bool isValid() const { return Valid; }
   bool isWeak() const {
-    if (!Func || !Valid)
+    if (!Func || !Valid || !Func->getDecl())
       return false;
 
     return Func->getDecl()->isWeak();
@@ -49,7 +49,10 @@ class FunctionPointer final {
                      CharUnits::fromQuantity(getIntegerRepresentation()), {},
                      /*OnePastTheEnd=*/false, /*IsNull=*/false);
 
-    return APValue(Func->getDecl(), CharUnits::Zero(), {},
+    if (Func->getDecl())
+      return APValue(Func->getDecl(), CharUnits::Zero(), {},
+                     /*OnePastTheEnd=*/false, /*IsNull=*/false);
+    return APValue(Func->getExpr(), CharUnits::Zero(), {},
                    /*OnePastTheEnd=*/false, /*IsNull=*/false);
   }
 
diff --git a/clang/lib/AST/ByteCode/Interp.h b/clang/lib/AST/ByteCode/Interp.h
index 5aa3d24ebc75827..bad46e983048455 100644
--- a/clang/lib/AST/ByteCode/Interp.h
+++ b/clang/lib/AST/ByteCode/Interp.h
@@ -2725,7 +2725,7 @@ inline bool CallPtr(InterpState &S, CodePtr OpPC, uint32_t ArgSize,
     return false;
   }
 
-  if (!FuncPtr.isValid())
+  if (!FuncPtr.isValid() || !F->getDecl())
     return Invalid(S, OpPC);
 
   assert(F);
diff --git a/clang/test/Sema/block-misc.c b/clang/test/Sema/block-misc.c
index aea44d55a606a41..c8a34b7f3c9fd55 100644
--- a/clang/test/Sema/block-misc.c
+++ b/clang/test/Sema/block-misc.c
@@ -1,4 +1,5 @@
 // RUN: %clang_cc1 -fsyntax-only -Wno-strict-prototypes -verify %s -fblocks
+// RUN: %clang_cc1 -fsyntax-only -Wno-strict-prototypes -verify %s -fblocks -fexperimental-new-constant-interpreter
 void donotwarn(void);
 
 int (^IFP) ();
diff --git a/clang/test/Sema/block-return.c b/clang/test/Sema/block-return.c
index d3d70511b185138..126fc6f953dea2e 100644
--- a/clang/test/Sema/block-return.c
+++ b/clang/test/Sema/block-return.c
@@ -1,4 +1,5 @@
 // RUN: %clang_cc1 -Wno-int-to-pointer-cast -Wno-pointer-to-int-cast -pedantic -fsyntax-only %s -verify -fblocks
+// RUN: %clang_cc1 -Wno-int-to-pointer-cast -Wno-pointer-to-int-cast -pedantic -fsyntax-only %s -verify -fblocks -fexperimental-new-constant-interpreter
 
 extern int printf(const char *, ...);
 
diff --git a/clang/test/SemaCXX/consteval-cleanup.cpp b/clang/test/SemaCXX/consteval-cleanup.cpp
index 499c45db5017702..f7d033b2ecafa46 100644
--- a/clang/test/SemaCXX/consteval-cleanup.cpp
+++ b/clang/test/SemaCXX/consteval-cleanup.cpp
@@ -1,4 +1,5 @@
 // RUN: %clang_cc1 -fblocks -Wno-unused-value -std=c++20 -ast-dump -verify %s -ast-dump | FileCheck %s
+// RUN: %clang_cc1 -fblocks -Wno-unused-value -std=c++20 -ast-dump -verify %s -ast-dump -fexperimental-new-constant-interpreter | FileCheck %s
 
 // expected-no-diagnostics
 

From 5403123197f8e331d3c2c24d82439ca1fe7e702d Mon Sep 17 00:00:00 2001
From: Sam Clegg <sbc@chromium.org>
Date: Mon, 19 Aug 2024 21:51:40 -0700
Subject: [PATCH 437/441] [lld][WebAssembly] Ignore local symbols when parsing
 lazy object files. (#104876)

This was broken back in #78658 when we transitioned away from archive
indexes to parsing lazy object files.

Fixes: #94077
Fixes: https://github.com/emscripten-core/emscripten/issues/22008
---
 lld/test/wasm/archive-local-sym.s | 24 ++++++++++++++++++++++++
 lld/wasm/InputFiles.cpp           |  2 +-
 2 files changed, 25 insertions(+), 1 deletion(-)
 create mode 100644 lld/test/wasm/archive-local-sym.s

diff --git a/lld/test/wasm/archive-local-sym.s b/lld/test/wasm/archive-local-sym.s
new file mode 100644
index 000000000000000..53e95b9bbb10567
--- /dev/null
+++ b/lld/test/wasm/archive-local-sym.s
@@ -0,0 +1,24 @@
+## Test that local symbols in archive files are ignored.
+# RUN: split-file %s %t
+# RUN: llvm-mc -filetype=obj -triple=wasm32-unknown-unknown -o %t/foo.o %t/foo.s
+# RUN: llvm-mc -filetype=obj -triple=wasm32-unknown-unknown -o %t/main.o %t/main.s
+# RUN: rm -f %t/libfoo.a
+# RUN: llvm-ar rcs %t/libfoo.a %t/foo.o
+# RUN: not wasm-ld %t/libfoo.a %t/main.o -o out.wasm 2>&1 | FileCheck %s
+
+#--- main.s
+
+.functype foo () -> ()
+
+.globl _start
+_start:
+  .functype _start () -> ()
+  call foo
+# CHECK: main.o: undefined symbol: foo
+  end_function
+
+#--- foo.s
+
+foo:
+  .functype foo () -> ()
+  end_function
diff --git a/lld/wasm/InputFiles.cpp b/lld/wasm/InputFiles.cpp
index 706ee25d5aae278..de8e707ab2b497d 100644
--- a/lld/wasm/InputFiles.cpp
+++ b/lld/wasm/InputFiles.cpp
@@ -392,7 +392,7 @@ void ObjFile::parseLazy() {
                     << wasmObj.get() << "\n");
   for (const SymbolRef &sym : wasmObj->symbols()) {
     const WasmSymbol &wasmSym = wasmObj->getWasmSymbol(sym.getRawDataRefImpl());
-    if (!wasmSym.isDefined())
+    if (wasmSym.isUndefined() || wasmSym.isBindingLocal())
       continue;
     symtab->addLazy(wasmSym.Info.Name, this);
     // addLazy() may trigger this->extract() if an existing symbol is an

From 2a6136e552d24b6bf665c42a6e32efc0b2d88fbf Mon Sep 17 00:00:00 2001
From: Sam Clegg <sbc@chromium.org>
Date: Mon, 19 Aug 2024 21:52:17 -0700
Subject: [PATCH 438/441] [llvm-objcopy][WebAssembly] Allow --strip-debug to
 operate on relocatable files. (#102978)

This change is enough to allow `--strip-debug` to work on object files,
without breaking the relocation information or symbol table.

A more complete version of this change would instead reconstruct the
symbol table and relocation sections, but that is much larger change.

Bug: #102002
---
 lld/wasm/Writer.cpp                           |  5 ++
 llvm/lib/ObjCopy/wasm/WasmObjcopy.cpp         |  2 +-
 llvm/lib/ObjCopy/wasm/WasmObject.cpp          | 18 ++++-
 llvm/lib/ObjCopy/wasm/WasmObject.h            |  1 +
 llvm/lib/ObjCopy/wasm/WasmReader.cpp          |  1 +
 llvm/lib/ObjectYAML/WasmEmitter.cpp           |  3 +-
 .../wasm/invalid_section_order.yaml           |  2 +-
 .../llvm-objcopy/wasm/basic-only-section.test | 16 +++--
 .../llvm-objcopy/wasm/only-keep-debug.test    | 38 +++++-----
 .../tools/llvm-objcopy/wasm/strip-all.test    |  6 ++
 .../tools/llvm-objcopy/wasm/strip-debug.test  | 71 +++++++++++++++++--
 .../tools/llvm-objcopy/wasm/strip-reloc.test  |  4 +-
 12 files changed, 132 insertions(+), 35 deletions(-)

diff --git a/lld/wasm/Writer.cpp b/lld/wasm/Writer.cpp
index 6a66a29d2498c53..6beef81d39a3818 100644
--- a/lld/wasm/Writer.cpp
+++ b/lld/wasm/Writer.cpp
@@ -138,6 +138,11 @@ void Writer::calculateCustomSections() {
       // Exclude COMDAT sections that are not selected for inclusion
       if (section->discarded)
         continue;
+      // Ignore empty custom sections.  In particular objcopy/strip will
+      // sometimes replace stripped sections with empty custom sections to
+      // avoid section re-numbering.
+      if (section->getSize() == 0)
+        continue;
       StringRef name = section->name;
       // These custom sections are known the linker and synthesized rather than
       // blindly copied.
diff --git a/llvm/lib/ObjCopy/wasm/WasmObjcopy.cpp b/llvm/lib/ObjCopy/wasm/WasmObjcopy.cpp
index 5bba1dea9adf0e3..cf3d884bee3bd8e 100644
--- a/llvm/lib/ObjCopy/wasm/WasmObjcopy.cpp
+++ b/llvm/lib/ObjCopy/wasm/WasmObjcopy.cpp
@@ -22,7 +22,7 @@ using namespace object;
 using SectionPred = std::function<bool(const Section &Sec)>;
 
 static bool isDebugSection(const Section &Sec) {
-  return Sec.Name.starts_with(".debug");
+  return Sec.Name.starts_with(".debug") || Sec.Name.starts_with("reloc..debug");
 }
 
 static bool isLinkerSection(const Section &Sec) {
diff --git a/llvm/lib/ObjCopy/wasm/WasmObject.cpp b/llvm/lib/ObjCopy/wasm/WasmObject.cpp
index 28a2de6e6e4f124..0b543b793f049ed 100644
--- a/llvm/lib/ObjCopy/wasm/WasmObject.cpp
+++ b/llvm/lib/ObjCopy/wasm/WasmObject.cpp
@@ -25,8 +25,22 @@ void Object::addSectionWithOwnedContents(
 }
 
 void Object::removeSections(function_ref<bool(const Section &)> ToRemove) {
-  // TODO: remove reloc sections for the removed section, handle symbols, etc.
-  llvm::erase_if(Sections, ToRemove);
+  if (isRelocatableObject) {
+    // For relocatable objects, avoid actually removing any sections,
+    // since that can invalidate the symbol table and relocation sections.
+    // TODO: Allow removal of sections by re-generating symbol table and
+    // relocation sections here instead.
+    for (auto &Sec : Sections) {
+      if (ToRemove(Sec)) {
+        Sec.Name = ".objcopy.removed";
+        Sec.SectionType = wasm::WASM_SEC_CUSTOM;
+        Sec.Contents = {};
+        Sec.HeaderSecSizeEncodingLen = std::nullopt;
+      }
+    }
+  } else {
+    llvm::erase_if(Sections, ToRemove);
+  }
 }
 
 } // end namespace wasm
diff --git a/llvm/lib/ObjCopy/wasm/WasmObject.h b/llvm/lib/ObjCopy/wasm/WasmObject.h
index f860ec697e5684d..374e523cd879b65 100644
--- a/llvm/lib/ObjCopy/wasm/WasmObject.h
+++ b/llvm/lib/ObjCopy/wasm/WasmObject.h
@@ -32,6 +32,7 @@ struct Object {
   llvm::wasm::WasmObjectHeader Header;
   // For now don't discriminate between kinds of sections.
   std::vector<Section> Sections;
+  bool isRelocatableObject = false;
 
   void addSectionWithOwnedContents(Section NewSection,
                                    std::unique_ptr<MemoryBuffer> &&Content);
diff --git a/llvm/lib/ObjCopy/wasm/WasmReader.cpp b/llvm/lib/ObjCopy/wasm/WasmReader.cpp
index 578e78955af3e45..420d17f9864323f 100644
--- a/llvm/lib/ObjCopy/wasm/WasmReader.cpp
+++ b/llvm/lib/ObjCopy/wasm/WasmReader.cpp
@@ -18,6 +18,7 @@ using namespace llvm::wasm;
 Expected<std::unique_ptr<Object>> Reader::create() const {
   auto Obj = std::make_unique<Object>();
   Obj->Header = WasmObj.getHeader();
+  Obj->isRelocatableObject = WasmObj.isRelocatableObject();
   std::vector<Section> Sections;
   Obj->Sections.reserve(WasmObj.getNumSections());
   for (const SectionRef &Sec : WasmObj.sections()) {
diff --git a/llvm/lib/ObjectYAML/WasmEmitter.cpp b/llvm/lib/ObjectYAML/WasmEmitter.cpp
index 9b8fd11f85437e9..4f384928b89f1df 100644
--- a/llvm/lib/ObjectYAML/WasmEmitter.cpp
+++ b/llvm/lib/ObjectYAML/WasmEmitter.cpp
@@ -604,7 +604,8 @@ bool WasmWriter::writeWasm(raw_ostream &OS) {
     if (auto S = dyn_cast<WasmYAML::CustomSection>(Sec.get()))
       SecName = S->Name;
     if (!Checker.isValidSectionOrder(Sec->Type, SecName)) {
-      reportError("out of order section type: " + Twine(Sec->Type));
+      reportError("out of order section type: " +
+                  wasm::sectionTypeToString(Sec->Type));
       return false;
     }
     encodeULEB128(Sec->Type, OS);
diff --git a/llvm/test/ObjectYAML/wasm/invalid_section_order.yaml b/llvm/test/ObjectYAML/wasm/invalid_section_order.yaml
index 71767576979b0a3..08319e500e11158 100644
--- a/llvm/test/ObjectYAML/wasm/invalid_section_order.yaml
+++ b/llvm/test/ObjectYAML/wasm/invalid_section_order.yaml
@@ -14,7 +14,7 @@ Sections:
       - Index:           0
         Locals:          []
         Body:            0B
-  # CHECK: yaml2obj: error: out of order section type: 3
+  # CHECK: yaml2obj: error: out of order section type: FUNCTION
   - Type:            FUNCTION
     FunctionTypes:   [ 0 ]
 ...
diff --git a/llvm/test/tools/llvm-objcopy/wasm/basic-only-section.test b/llvm/test/tools/llvm-objcopy/wasm/basic-only-section.test
index fe47d5c027733a8..f30743c632bc863 100644
--- a/llvm/test/tools/llvm-objcopy/wasm/basic-only-section.test
+++ b/llvm/test/tools/llvm-objcopy/wasm/basic-only-section.test
@@ -1,15 +1,15 @@
 ## Test --only-section.
 # RUN: yaml2obj %s -o %t
 # RUN: llvm-objcopy --only-section=foo %t %t2
-# RUN: obj2yaml %t2 | FileCheck --implicit-check-not TYPE --implicit-check-not linking %s
+# RUN: obj2yaml %t2 | FileCheck --implicit-check-not TYPE --implicit-check-not producers %s
 
 ## Test that it's the same with only-section + keep-section (for the same section).
 # RUN: llvm-objcopy --only-section=foo --keep-section=foo %t %t2
-# RUN: obj2yaml %t2 | FileCheck --implicit-check-not TYPE --implicit-check-not linking %s
+# RUN: obj2yaml %t2 | FileCheck --implicit-check-not TYPE --implicit-check-not producers %s
 
 ## Also test that only-section overrides remove-section.
 # RUN: llvm-objcopy --only-section=foo --remove-section=foo %t %t2
-# RUN: obj2yaml %t2 | FileCheck --implicit-check-not linking %s
+# RUN: obj2yaml %t2 | FileCheck --implicit-check-not producers %s
 
 ## This file has both known and custom sections. Check that only the foo section is left.
 # CHECK:      Sections:
@@ -19,10 +19,10 @@
 # CHECK-NEXT: ...
 
 ## Test that only-section + keep-section keeps both sections.
-# RUN: llvm-objcopy --only-section=foo --keep-section=linking %t %t2
+# RUN: llvm-objcopy --only-section=foo --keep-section=producers %t %t2
 # RUN: obj2yaml %t2 | FileCheck --implicit-check-not=TYPE --check-prefix=KEEP %s
 # KEEP: Name: foo
-# KEEP: Name: linking
+# KEEP: Name: producers
 
 --- !WASM
 FileHeader:
@@ -39,5 +39,7 @@ Sections:
         ReturnTypes:
           - F32
   - Type: CUSTOM
-    Name: linking
-    Version: 2
+    Name: producers
+    Tools:
+      - Name:   clang
+        Version: 9.0.0
diff --git a/llvm/test/tools/llvm-objcopy/wasm/only-keep-debug.test b/llvm/test/tools/llvm-objcopy/wasm/only-keep-debug.test
index fc57329bdd74c2f..c7a5da99b1edf70 100644
--- a/llvm/test/tools/llvm-objcopy/wasm/only-keep-debug.test
+++ b/llvm/test/tools/llvm-objcopy/wasm/only-keep-debug.test
@@ -1,26 +1,32 @@
 ## Test that only debug sections are kept with --only-keep-debug.
-# RUN: yaml2obj %s -o %t
-# RUN: llvm-strip --only-keep-debug %t
-# RUN: obj2yaml %t | FileCheck %s
+# RUN: yaml2obj %s -o %t.wasm
+# RUN: llvm-strip --only-keep-debug %t.wasm -o %t2.wasm
+# RUN: obj2yaml %t2.wasm | FileCheck %s -check-prefixes=CHECK,STRIPTYPE,STRIPFOO
 
 ## Test that keep-section overrides only-keep-debug.
-# RUN: yaml2obj %s -o %t
-# RUN: llvm-strip --only-keep-debug --keep-section=foo %t
-# RUN: obj2yaml %t | FileCheck --implicit-check-not=Name --check-prefix=CHECK --check-prefix=KEEP %s
+# RUN: llvm-strip --only-keep-debug --keep-section=foo %t.wasm -o %t3.wasm
+# RUN: obj2yaml %t3.wasm | FileCheck --implicit-check-not=Name --check-prefixes=CHECK,STRIPTYPE,KEEPFOO %s
 
 ## Test that keep-section overrides only-keep-debug, even for known sections.
-# RUN: yaml2obj %s -o %t
-# RUN: llvm-strip --only-keep-debug --keep-section=TYPE %t
-# RUN: obj2yaml %t | FileCheck --implicit-check-not=Name --check-prefix=CHECK --check-prefix=KEEPTYPE %s
+# RUN: llvm-strip --only-keep-debug --keep-section=TYPE %t.wasm -o %t4.wasm
+# RUN: obj2yaml %t4.wasm | FileCheck --implicit-check-not=Name --check-prefixes=CHECK,KEEPTYPE,STRIPFOO %s
 
 # CHECK:      Sections:
-# KEEPTYPE:     - Type: TYPE
-# CHECK-NOT:    - Type: TYPE
-# CHECK:        - Type: CUSTOM
-# CHECK-NEXT:     Name: .debug_info
-# CHECK:        - Type: CUSTOM
-# CHECK-NEXT:     Name: .debug_line
-# KEEP:           Name: foo
+# KEEPTYPE:       - Type: TYPE
+# STRIPTYPE-NOT:  - Type: TYPE
+# STRIPTYPE:      - Type: CUSTOM
+# STRIPTYPE-NEXT:   Name: .objcopy.removed
+# CHECK:          - Type: CUSTOM
+# CHECK-NEXT:       Name: .debug_info
+# CHECK:          - Type: CUSTOM
+# CHECK-NEXT:       Name: .objcopy.removed
+# CHECK:          - Type: CUSTOM
+# CHECK-NEXT:       Name: .objcopy.removed
+# CHECK:          - Type: CUSTOM
+# CHECK-NEXT:       Name: .debug_line
+# CHECK:          - Type: CUSTOM
+# KEEPFOO-NEXT:     Name: foo
+# STRIPFOO-NEXT:    Name: .objcopy.removed
 
 ## Test that remove-section overrides only-keep-debug.
 # RUN: yaml2obj %s -o %t
diff --git a/llvm/test/tools/llvm-objcopy/wasm/strip-all.test b/llvm/test/tools/llvm-objcopy/wasm/strip-all.test
index 9a9015713242c81..ef7c1a90e610752 100644
--- a/llvm/test/tools/llvm-objcopy/wasm/strip-all.test
+++ b/llvm/test/tools/llvm-objcopy/wasm/strip-all.test
@@ -7,6 +7,12 @@
 # CHECK:      Sections:
 # CHECK-NEXT:   - Type: TYPE
 # CHECK:        - Type: CUSTOM
+# CHECK-NEXT:        Name: .objcopy.removed
+# CHECK:        - Type: CUSTOM
+# CHECK-NEXT:        Name: .objcopy.removed
+# CHECK:        - Type: CUSTOM
+# CHECK-NEXT:        Name: .objcopy.removed
+# CHECK:        - Type: CUSTOM
 # CHECK-NEXT:        Name: foo
 
 --- !WASM
diff --git a/llvm/test/tools/llvm-objcopy/wasm/strip-debug.test b/llvm/test/tools/llvm-objcopy/wasm/strip-debug.test
index 2747c3bab742ea4..53e8153dd3539ab 100644
--- a/llvm/test/tools/llvm-objcopy/wasm/strip-debug.test
+++ b/llvm/test/tools/llvm-objcopy/wasm/strip-debug.test
@@ -1,14 +1,65 @@
-## Test that debug sections (but not linking or names) are stripped with --strip-debug
 # RUN: yaml2obj %s -o %t
-# RUN: llvm-strip --strip-debug %t
-# RUN: obj2yaml %t | FileCheck --implicit-check-not=.debug %s
+# RUN: cp %t %t3
+# RUN: llvm-objcopy --strip-debug %t %t2
+## Test that debug sections (but not linking or names) are stripped with --strip-debug
+# RUN: obj2yaml %t2 | FileCheck --implicit-check-not=.debug %s
+#
+# RUN: llvm-objcopy -g %t %t2g
+# Verify that --strip-debug and -g produce the same output
+# RUN: cmp %t2 %t2g
+
+# RUN: llvm-strip --strip-debug %t3
+# RUN: cmp %t2 %t3
+
+# RUN: cp %t %t4
+# RUN: llvm-strip -d %t4
+# RUN: cmp %t2 %t4
+
+# RUN: cp %t %t5
+# RUN: llvm-strip -g %t5
+# RUN: cmp %t2 %t5
+
+# RUN: cp %t %t6
+# RUN: llvm-strip -S %t6
+# RUN: cmp %t2 %t6
+
+# Verify that an archive with multiple object files is handled correctly.
+# RUN: cp %t %t.duplicate
+# RUN: cp %t2 %t.duplicate.stripped
+# RUN: rm -f %t.multiple-stripped-obj.a
+# RUN: llvm-ar crs %t.multiple-stripped-obj.a %t2 %t.duplicate.stripped
+# RUN: rm -f %t.multiple-obj.a
+# RUN: llvm-ar crs %t.multiple-obj.a %t %t.duplicate
+# RUN: llvm-objcopy --strip-debug %t.multiple-obj.a %t.multiple-obj.stripped.a
+# RUN: llvm-ar p %t.multiple-stripped-obj.a > %t.multiple-stripped-obj.a.dump
+# RUN: llvm-ar p %t.multiple-obj.stripped.a > %t.multiple-obj.stripped.a.dump
+# RUN: cmp %t.multiple-stripped-obj.a.dump %t.multiple-obj.stripped.a.dump
 
 # CHECK:      Sections:
 # CHECK-NEXT:   - Type: TYPE
-# CHECK:          Name: linking
+# CHECK:        - Type: CUSTOM
+## We expect the linking section to be preceeded by the removed `.debug_info`
+## section.
+# CHECK-NEXT:     Name:            .objcopy.removed
+# CHECK-NEXT:     Payload:         ''
+# CHECK-NEXT:   - Type: CUSTOM
+# CHECK-NEXT:     Name: linking
 # CHECK:          Name: name
 # CHECK-NEXT:     FunctionNames:
 # CHECK:          Name: producers
+## Following the producers section we expect to find three removed sections.
+## The `.debug_line` section that two reloction section corresponding to the
+## two debug sections.
+# CHECK:        - Type:            CUSTOM
+# CHECK-NEXT:     Name:            .objcopy.removed
+# CHECK-NEXT:     Payload:         ''
+# CHECK-NEXT:   - Type:            CUSTOM
+# CHECK-NEXT:     Name:            .objcopy.removed
+# CHECK-NEXT:     Payload:         ''
+# CHECK-NEXT:   - Type:            CUSTOM
+# CHECK-NEXT:     Name:            .objcopy.removed
+# CHECK-NEXT:     Payload:         ''
+
 
 --- !WASM
 FileHeader:
@@ -28,7 +79,11 @@ Sections:
         Body: 0B
   - Type: CUSTOM
     Name: .debug_info
-    Payload: CAFE1234
+    Payload: 'CAFE123456'
+    Relocations:
+      - Type:   R_WASM_FUNCTION_INDEX_LEB
+        Index:  0
+        Offset: 0x0000000
   - Type:            CUSTOM
     Name:            linking
     Version:         2
@@ -50,4 +105,8 @@ Sections:
         Version: 9.0.0
   - Type: CUSTOM
     Name: .debug_line
-    Payload: DEADBEEF
+    Payload: 'DEADBEEF01'
+    Relocations:
+      - Type:   R_WASM_FUNCTION_INDEX_LEB
+        Index:  0
+        Offset: 0x0000000
diff --git a/llvm/test/tools/llvm-objcopy/wasm/strip-reloc.test b/llvm/test/tools/llvm-objcopy/wasm/strip-reloc.test
index 4c8d1ba670447f8..d072bf70c9db76b 100644
--- a/llvm/test/tools/llvm-objcopy/wasm/strip-reloc.test
+++ b/llvm/test/tools/llvm-objcopy/wasm/strip-reloc.test
@@ -5,13 +5,15 @@
 
 # RUN: yaml2obj %s -o %t
 # RUN: llvm-objcopy --strip-all %t %t2
-# RUN: obj2yaml %t2 | FileCheck --implicit-check-not=Type: %s
+# RUN: obj2yaml %t2 | FileCheck %s
 
 ## Check that the known sections are still present.
 # CHECK: Sections:
 # CHECK: - Type: TYPE
 # CHECK: - Type: FUNCTION
 # CHECK: - Type: CODE
+# CHECK-NOT: Relocations
+# CHECK-NOT: linking
 ## Check that there are still functions in the code section.
 # CHECK: Functions:
 

From ea1f05e6189fbe2b567e795677e1d64acbadc0a2 Mon Sep 17 00:00:00 2001
From: David Majnemer <david.majnemer@gmail.com>
Date: Tue, 20 Aug 2024 02:55:13 +0000
Subject: [PATCH 439/441] [SelectionDAG] Fix lowering of IEEE 754 2019
 minimum/maximum

We used integer comparisons instead of floating point comparisons
resulting in very odd behavior.
---
 .../CodeGen/SelectionDAG/TargetLowering.cpp   |    6 +-
 llvm/test/CodeGen/X86/fminimum-fmaximum.ll    | 1100 +++++++++++++++++
 2 files changed, 1103 insertions(+), 3 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index ae9f11d02a8ac49..4e796289cff0a10 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -8517,7 +8517,7 @@ SDValue TargetLowering::expandFMINIMUM_FMAXIMUM(SDNode *N,
 
     // NaN (if exists) will be propagated later, so orderness doesn't matter.
     SDValue Compare =
-        DAG.getSetCC(DL, CCVT, LHS, RHS, IsMax ? ISD::SETGT : ISD::SETLT);
+        DAG.getSetCC(DL, CCVT, LHS, RHS, IsMax ? ISD::SETOGT : ISD::SETOLT);
     MinMax = DAG.getSelect(DL, VT, Compare, LHS, RHS, Flags);
   }
 
@@ -8534,7 +8534,7 @@ SDValue TargetLowering::expandFMINIMUM_FMAXIMUM(SDNode *N,
   if (!MinMaxMustRespectOrderedZero && !N->getFlags().hasNoSignedZeros() &&
       !DAG.isKnownNeverZeroFloat(RHS) && !DAG.isKnownNeverZeroFloat(LHS)) {
     SDValue IsZero = DAG.getSetCC(DL, CCVT, MinMax,
-                                  DAG.getConstantFP(0.0, DL, VT), ISD::SETEQ);
+                                  DAG.getConstantFP(0.0, DL, VT), ISD::SETOEQ);
     SDValue TestZero =
         DAG.getTargetConstant(IsMax ? fcPosZero : fcNegZero, DL, MVT::i32);
     SDValue LCmp = DAG.getSelect(
@@ -8823,7 +8823,7 @@ SDValue TargetLowering::expandIS_FPCLASS(EVT ResultVT, SDValue Op,
   // Split the value into sign bit and absolute value.
   SDValue AbsV = DAG.getNode(ISD::AND, DL, IntVT, OpAsInt, ValueMaskV);
   SDValue SignV = DAG.getSetCC(DL, ResultVT, OpAsInt,
-                               DAG.getConstant(0.0, DL, IntVT), ISD::SETLT);
+                               DAG.getConstant(0, DL, IntVT), ISD::SETLT);
 
   // Tests that involve more than one class should be processed first.
   SDValue PartialRes;
diff --git a/llvm/test/CodeGen/X86/fminimum-fmaximum.ll b/llvm/test/CodeGen/X86/fminimum-fmaximum.ll
index 55502f37da1a627..41d9a867c0a9609 100644
--- a/llvm/test/CodeGen/X86/fminimum-fmaximum.ll
+++ b/llvm/test/CodeGen/X86/fminimum-fmaximum.ll
@@ -11,6 +11,8 @@ declare float @llvm.minimum.f32(float, float)
 declare double @llvm.minimum.f64(double, double)
 declare <2 x double> @llvm.minimum.v2f64(<2 x double>, <2 x double>)
 declare <4 x float> @llvm.maximum.v4f32(<4 x float>, <4 x float>)
+declare <4 x half> @llvm.maximum.v4f16(<4 x half>, <4 x half>)
+declare <4 x bfloat> @llvm.maximum.v4bf16(<4 x bfloat>, <4 x bfloat>)
 
 ;
 ; fmaximum
@@ -1381,3 +1383,1101 @@ define <4 x float> @test_fmaximum_v4f32_splat(<4 x float> %x, float %y) {
   %r = call <4 x float> @llvm.maximum.v4f32(<4 x float> %x, <4 x float> %vec) readnone
   ret <4 x float> %r
 }
+
+define <4 x half> @test_fmaximum_v4f16(<4 x half> %x, <4 x half> %y) {
+; SSE2-LABEL: test_fmaximum_v4f16:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    subq $104, %rsp
+; SSE2-NEXT:    .cfi_def_cfa_offset 112
+; SSE2-NEXT:    movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE2-NEXT:    psrld $16, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE2-NEXT:    movdqa %xmm1, (%rsp) # 16-byte Spill
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    psrld $16, %xmm0
+; SSE2-NEXT:    callq __extendhfsf2@PLT
+; SSE2-NEXT:    movd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
+; SSE2-NEXT:    movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; SSE2-NEXT:    callq __extendhfsf2@PLT
+; SSE2-NEXT:    movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 4-byte Folded Reload
+; SSE2-NEXT:    # xmm4 = mem[0],zero,zero,zero
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    testl %eax, %eax
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    js .LBB33_2
+; SSE2-NEXT:  # %bb.1:
+; SSE2-NEXT:    movdqa %xmm4, %xmm2
+; SSE2-NEXT:  .LBB33_2:
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    cmpunordss %xmm2, %xmm0
+; SSE2-NEXT:    movaps %xmm0, %xmm3
+; SSE2-NEXT:    andps %xmm2, %xmm3
+; SSE2-NEXT:    js .LBB33_4
+; SSE2-NEXT:  # %bb.3:
+; SSE2-NEXT:    movdqa %xmm1, %xmm4
+; SSE2-NEXT:  .LBB33_4:
+; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,1,1]
+; SSE2-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE2-NEXT:    movaps (%rsp), %xmm1 # 16-byte Reload
+; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,1,1]
+; SSE2-NEXT:    movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE2-NEXT:    maxss %xmm4, %xmm2
+; SSE2-NEXT:    andnps %xmm2, %xmm0
+; SSE2-NEXT:    orps %xmm3, %xmm0
+; SSE2-NEXT:    callq __truncsfhf2@PLT
+; SSE2-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE2-NEXT:    movaps (%rsp), %xmm0 # 16-byte Reload
+; SSE2-NEXT:    callq __extendhfsf2@PLT
+; SSE2-NEXT:    movss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; SSE2-NEXT:    movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; SSE2-NEXT:    callq __extendhfsf2@PLT
+; SSE2-NEXT:    movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 4-byte Folded Reload
+; SSE2-NEXT:    # xmm4 = mem[0],zero,zero,zero
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    testl %eax, %eax
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    js .LBB33_6
+; SSE2-NEXT:  # %bb.5:
+; SSE2-NEXT:    movdqa %xmm4, %xmm2
+; SSE2-NEXT:  .LBB33_6:
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    cmpunordss %xmm2, %xmm0
+; SSE2-NEXT:    movaps %xmm0, %xmm3
+; SSE2-NEXT:    andps %xmm2, %xmm3
+; SSE2-NEXT:    js .LBB33_8
+; SSE2-NEXT:  # %bb.7:
+; SSE2-NEXT:    movdqa %xmm1, %xmm4
+; SSE2-NEXT:  .LBB33_8:
+; SSE2-NEXT:    movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; SSE2-NEXT:    psrlq $48, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE2-NEXT:    movdqa (%rsp), %xmm1 # 16-byte Reload
+; SSE2-NEXT:    psrlq $48, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, (%rsp) # 16-byte Spill
+; SSE2-NEXT:    maxss %xmm4, %xmm2
+; SSE2-NEXT:    andnps %xmm2, %xmm0
+; SSE2-NEXT:    orps %xmm3, %xmm0
+; SSE2-NEXT:    callq __truncsfhf2@PLT
+; SSE2-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE2-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; SSE2-NEXT:    callq __extendhfsf2@PLT
+; SSE2-NEXT:    movss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; SSE2-NEXT:    movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; SSE2-NEXT:    callq __extendhfsf2@PLT
+; SSE2-NEXT:    movd {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 4-byte Folded Reload
+; SSE2-NEXT:    # xmm4 = mem[0],zero,zero,zero
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    testl %eax, %eax
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    js .LBB33_10
+; SSE2-NEXT:  # %bb.9:
+; SSE2-NEXT:    movdqa %xmm4, %xmm2
+; SSE2-NEXT:  .LBB33_10:
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    cmpunordss %xmm2, %xmm1
+; SSE2-NEXT:    movaps %xmm1, %xmm3
+; SSE2-NEXT:    andps %xmm2, %xmm3
+; SSE2-NEXT:    js .LBB33_12
+; SSE2-NEXT:  # %bb.11:
+; SSE2-NEXT:    movdqa %xmm0, %xmm4
+; SSE2-NEXT:  .LBB33_12:
+; SSE2-NEXT:    maxss %xmm4, %xmm2
+; SSE2-NEXT:    andnps %xmm2, %xmm1
+; SSE2-NEXT:    orps %xmm3, %xmm1
+; SSE2-NEXT:    movaps %xmm1, %xmm0
+; SSE2-NEXT:    callq __truncsfhf2@PLT
+; SSE2-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE2-NEXT:    movaps (%rsp), %xmm0 # 16-byte Reload
+; SSE2-NEXT:    callq __extendhfsf2@PLT
+; SSE2-NEXT:    movss %xmm0, (%rsp) # 4-byte Spill
+; SSE2-NEXT:    movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; SSE2-NEXT:    callq __extendhfsf2@PLT
+; SSE2-NEXT:    movd (%rsp), %xmm4 # 4-byte Folded Reload
+; SSE2-NEXT:    # xmm4 = mem[0],zero,zero,zero
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    testl %eax, %eax
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    js .LBB33_14
+; SSE2-NEXT:  # %bb.13:
+; SSE2-NEXT:    movdqa %xmm4, %xmm2
+; SSE2-NEXT:  .LBB33_14:
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    cmpunordss %xmm2, %xmm0
+; SSE2-NEXT:    movaps %xmm0, %xmm3
+; SSE2-NEXT:    andps %xmm2, %xmm3
+; SSE2-NEXT:    js .LBB33_16
+; SSE2-NEXT:  # %bb.15:
+; SSE2-NEXT:    movdqa %xmm1, %xmm4
+; SSE2-NEXT:  .LBB33_16:
+; SSE2-NEXT:    maxss %xmm4, %xmm2
+; SSE2-NEXT:    andnps %xmm2, %xmm0
+; SSE2-NEXT:    orps %xmm3, %xmm0
+; SSE2-NEXT:    callq __truncsfhf2@PLT
+; SSE2-NEXT:    movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; SSE2-NEXT:    movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; SSE2-NEXT:    punpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; SSE2-NEXT:    # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT:    addq $104, %rsp
+; SSE2-NEXT:    .cfi_def_cfa_offset 8
+; SSE2-NEXT:    retq
+;
+; AVX1-LABEL: test_fmaximum_v4f16:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    subq $120, %rsp
+; AVX1-NEXT:    .cfi_def_cfa_offset 128
+; AVX1-NEXT:    vmovaps %xmm0, %xmm2
+; AVX1-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; AVX1-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm1[1,1,3,3]
+; AVX1-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT:    vpsrld $16, %xmm2, %xmm0
+; AVX1-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT:    vmovaps %xmm1, (%rsp) # 16-byte Spill
+; AVX1-NEXT:    vpsrld $16, %xmm1, %xmm0
+; AVX1-NEXT:    callq __extendhfsf2@PLT
+; AVX1-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX1-NEXT:    callq __extendhfsf2@PLT
+; AVX1-NEXT:    vmovd %xmm0, %eax
+; AVX1-NEXT:    testl %eax, %eax
+; AVX1-NEXT:    js .LBB33_1
+; AVX1-NEXT:  # %bb.2:
+; AVX1-NEXT:    vmovdqa %xmm0, %xmm1
+; AVX1-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
+; AVX1-NEXT:    jmp .LBB33_3
+; AVX1-NEXT:  .LBB33_1:
+; AVX1-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; AVX1-NEXT:    vmovdqa %xmm0, %xmm2
+; AVX1-NEXT:  .LBB33_3:
+; AVX1-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX1-NEXT:    vpsrlq $48, %xmm0, %xmm0
+; AVX1-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT:    vmovdqa (%rsp), %xmm0 # 16-byte Reload
+; AVX1-NEXT:    vpsrlq $48, %xmm0, %xmm0
+; AVX1-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT:    vmaxss %xmm1, %xmm2, %xmm0
+; AVX1-NEXT:    vcmpunordss %xmm2, %xmm2, %xmm1
+; AVX1-NEXT:    vblendvps %xmm1, %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    callq __truncsfhf2@PLT
+; AVX1-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
+; AVX1-NEXT:    callq __extendhfsf2@PLT
+; AVX1-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
+; AVX1-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX1-NEXT:    callq __extendhfsf2@PLT
+; AVX1-NEXT:    vmovd %xmm0, %eax
+; AVX1-NEXT:    testl %eax, %eax
+; AVX1-NEXT:    js .LBB33_4
+; AVX1-NEXT:  # %bb.5:
+; AVX1-NEXT:    vmovdqa %xmm0, %xmm1
+; AVX1-NEXT:    vmovdqa (%rsp), %xmm2 # 16-byte Reload
+; AVX1-NEXT:    jmp .LBB33_6
+; AVX1-NEXT:  .LBB33_4:
+; AVX1-NEXT:    vmovdqa (%rsp), %xmm1 # 16-byte Reload
+; AVX1-NEXT:    vmovdqa %xmm0, %xmm2
+; AVX1-NEXT:  .LBB33_6:
+; AVX1-NEXT:    vmaxss %xmm1, %xmm2, %xmm0
+; AVX1-NEXT:    vcmpunordss %xmm2, %xmm2, %xmm1
+; AVX1-NEXT:    vblendvps %xmm1, %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    callq __truncsfhf2@PLT
+; AVX1-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX1-NEXT:    callq __extendhfsf2@PLT
+; AVX1-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
+; AVX1-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX1-NEXT:    callq __extendhfsf2@PLT
+; AVX1-NEXT:    vmovd %xmm0, %eax
+; AVX1-NEXT:    testl %eax, %eax
+; AVX1-NEXT:    js .LBB33_7
+; AVX1-NEXT:  # %bb.8:
+; AVX1-NEXT:    vmovdqa %xmm0, %xmm1
+; AVX1-NEXT:    vmovdqa (%rsp), %xmm2 # 16-byte Reload
+; AVX1-NEXT:    jmp .LBB33_9
+; AVX1-NEXT:  .LBB33_7:
+; AVX1-NEXT:    vmovdqa (%rsp), %xmm1 # 16-byte Reload
+; AVX1-NEXT:    vmovdqa %xmm0, %xmm2
+; AVX1-NEXT:  .LBB33_9:
+; AVX1-NEXT:    vmaxss %xmm1, %xmm2, %xmm0
+; AVX1-NEXT:    vcmpunordss %xmm2, %xmm2, %xmm1
+; AVX1-NEXT:    vblendvps %xmm1, %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    callq __truncsfhf2@PLT
+; AVX1-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
+; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX1-NEXT:    callq __extendhfsf2@PLT
+; AVX1-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX1-NEXT:    callq __extendhfsf2@PLT
+; AVX1-NEXT:    vmovd %xmm0, %eax
+; AVX1-NEXT:    testl %eax, %eax
+; AVX1-NEXT:    js .LBB33_10
+; AVX1-NEXT:  # %bb.11:
+; AVX1-NEXT:    vmovdqa %xmm0, %xmm1
+; AVX1-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
+; AVX1-NEXT:    jmp .LBB33_12
+; AVX1-NEXT:  .LBB33_10:
+; AVX1-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; AVX1-NEXT:    vmovdqa %xmm0, %xmm2
+; AVX1-NEXT:  .LBB33_12:
+; AVX1-NEXT:    vmaxss %xmm1, %xmm2, %xmm0
+; AVX1-NEXT:    vcmpunordss %xmm2, %xmm2, %xmm1
+; AVX1-NEXT:    vblendvps %xmm1, %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    callq __truncsfhf2@PLT
+; AVX1-NEXT:    vmovdqa (%rsp), %xmm1 # 16-byte Reload
+; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; AVX1-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; AVX1-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload
+; AVX1-NEXT:    # xmm1 = xmm1[0],mem[0],xmm1[1],mem[1],xmm1[2],mem[2],xmm1[3],mem[3]
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],zero,zero
+; AVX1-NEXT:    addq $120, %rsp
+; AVX1-NEXT:    .cfi_def_cfa_offset 8
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: test_fmaximum_v4f16:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    pushq %rbp
+; AVX512-NEXT:    .cfi_def_cfa_offset 16
+; AVX512-NEXT:    pushq %r15
+; AVX512-NEXT:    .cfi_def_cfa_offset 24
+; AVX512-NEXT:    pushq %r14
+; AVX512-NEXT:    .cfi_def_cfa_offset 32
+; AVX512-NEXT:    pushq %r13
+; AVX512-NEXT:    .cfi_def_cfa_offset 40
+; AVX512-NEXT:    pushq %r12
+; AVX512-NEXT:    .cfi_def_cfa_offset 48
+; AVX512-NEXT:    pushq %rbx
+; AVX512-NEXT:    .cfi_def_cfa_offset 56
+; AVX512-NEXT:    .cfi_offset %rbx, -56
+; AVX512-NEXT:    .cfi_offset %r12, -48
+; AVX512-NEXT:    .cfi_offset %r13, -40
+; AVX512-NEXT:    .cfi_offset %r14, -32
+; AVX512-NEXT:    .cfi_offset %r15, -24
+; AVX512-NEXT:    .cfi_offset %rbp, -16
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
+; AVX512-NEXT:    vcvtph2ps %xmm2, %xmm2
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[3,3,3,3]
+; AVX512-NEXT:    vcvtph2ps %xmm3, %xmm3
+; AVX512-NEXT:    xorl %eax, %eax
+; AVX512-NEXT:    vucomiss %xmm2, %xmm3
+; AVX512-NEXT:    movl $65535, %ecx # imm = 0xFFFF
+; AVX512-NEXT:    movl $0, %edx
+; AVX512-NEXT:    cmovpl %ecx, %edx
+; AVX512-NEXT:    movl $0, %edi
+; AVX512-NEXT:    cmoval %ecx, %edi
+; AVX512-NEXT:    vpsrldq {{.*#+}} xmm2 = xmm1[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512-NEXT:    vcvtph2ps %xmm2, %xmm2
+; AVX512-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512-NEXT:    vcvtph2ps %xmm3, %xmm3
+; AVX512-NEXT:    vucomiss %xmm2, %xmm3
+; AVX512-NEXT:    movl $0, %esi
+; AVX512-NEXT:    cmovpl %ecx, %esi
+; AVX512-NEXT:    movl $0, %r9d
+; AVX512-NEXT:    cmoval %ecx, %r9d
+; AVX512-NEXT:    vshufpd {{.*#+}} xmm2 = xmm1[1,0]
+; AVX512-NEXT:    vcvtph2ps %xmm2, %xmm2
+; AVX512-NEXT:    vshufpd {{.*#+}} xmm3 = xmm0[1,0]
+; AVX512-NEXT:    vcvtph2ps %xmm3, %xmm3
+; AVX512-NEXT:    vucomiss %xmm2, %xmm3
+; AVX512-NEXT:    movl $0, %r8d
+; AVX512-NEXT:    cmovpl %ecx, %r8d
+; AVX512-NEXT:    movl $0, %r11d
+; AVX512-NEXT:    cmoval %ecx, %r11d
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm1[3,3,3,3,4,5,6,7]
+; AVX512-NEXT:    vcvtph2ps %xmm2, %xmm2
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm0[3,3,3,3,4,5,6,7]
+; AVX512-NEXT:    vcvtph2ps %xmm3, %xmm3
+; AVX512-NEXT:    vucomiss %xmm2, %xmm3
+; AVX512-NEXT:    movl $0, %r10d
+; AVX512-NEXT:    cmovpl %ecx, %r10d
+; AVX512-NEXT:    movl $0, %ebp
+; AVX512-NEXT:    cmoval %ecx, %ebp
+; AVX512-NEXT:    vmovshdup {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; AVX512-NEXT:    vcvtph2ps %xmm2, %xmm2
+; AVX512-NEXT:    vmovshdup {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; AVX512-NEXT:    vcvtph2ps %xmm3, %xmm3
+; AVX512-NEXT:    vucomiss %xmm2, %xmm3
+; AVX512-NEXT:    movl $0, %ebx
+; AVX512-NEXT:    cmovpl %ecx, %ebx
+; AVX512-NEXT:    movl $0, %r14d
+; AVX512-NEXT:    cmoval %ecx, %r14d
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm1[1,1,1,1,4,5,6,7]
+; AVX512-NEXT:    vcvtph2ps %xmm2, %xmm2
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm0[1,1,1,1,4,5,6,7]
+; AVX512-NEXT:    vcvtph2ps %xmm3, %xmm3
+; AVX512-NEXT:    vucomiss %xmm2, %xmm3
+; AVX512-NEXT:    movl $0, %r15d
+; AVX512-NEXT:    cmovpl %ecx, %r15d
+; AVX512-NEXT:    movl $0, %r12d
+; AVX512-NEXT:    cmoval %ecx, %r12d
+; AVX512-NEXT:    vcvtph2ps %xmm1, %xmm2
+; AVX512-NEXT:    vcvtph2ps %xmm0, %xmm3
+; AVX512-NEXT:    vucomiss %xmm2, %xmm3
+; AVX512-NEXT:    movl $0, %r13d
+; AVX512-NEXT:    cmoval %ecx, %r13d
+; AVX512-NEXT:    vmovd %r13d, %xmm2
+; AVX512-NEXT:    vpinsrw $1, %r12d, %xmm2, %xmm2
+; AVX512-NEXT:    vpinsrw $2, %r14d, %xmm2, %xmm2
+; AVX512-NEXT:    vpinsrw $3, %ebp, %xmm2, %xmm2
+; AVX512-NEXT:    vpinsrw $4, %r11d, %xmm2, %xmm2
+; AVX512-NEXT:    vpinsrw $5, %r9d, %xmm2, %xmm2
+; AVX512-NEXT:    vpinsrw $6, %edi, %xmm2, %xmm2
+; AVX512-NEXT:    movl $0, %edi
+; AVX512-NEXT:    cmovpl %ecx, %edi
+; AVX512-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512-NEXT:    vcvtph2ps %xmm3, %xmm3
+; AVX512-NEXT:    vpsrldq {{.*#+}} xmm4 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512-NEXT:    vcvtph2ps %xmm4, %xmm4
+; AVX512-NEXT:    vucomiss %xmm3, %xmm4
+; AVX512-NEXT:    movl $0, %r9d
+; AVX512-NEXT:    cmoval %ecx, %r9d
+; AVX512-NEXT:    vpinsrw $7, %r9d, %xmm2, %xmm2
+; AVX512-NEXT:    vpblendvb %xmm2, %xmm0, %xmm1, %xmm2
+; AVX512-NEXT:    vmovd %edi, %xmm3
+; AVX512-NEXT:    vpinsrw $1, %r15d, %xmm3, %xmm3
+; AVX512-NEXT:    vpinsrw $2, %ebx, %xmm3, %xmm3
+; AVX512-NEXT:    vpinsrw $3, %r10d, %xmm3, %xmm3
+; AVX512-NEXT:    vpinsrw $4, %r8d, %xmm3, %xmm3
+; AVX512-NEXT:    vpinsrw $5, %esi, %xmm3, %xmm3
+; AVX512-NEXT:    vpinsrw $6, %edx, %xmm3, %xmm3
+; AVX512-NEXT:    movl $0, %edx
+; AVX512-NEXT:    cmovpl %ecx, %edx
+; AVX512-NEXT:    vpinsrw $7, %edx, %xmm3, %xmm3
+; AVX512-NEXT:    vpbroadcastw {{.*#+}} xmm4 = [NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN]
+; AVX512-NEXT:    vpblendvb %xmm3, %xmm4, %xmm2, %xmm2
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm2[1,1,1,1,4,5,6,7]
+; AVX512-NEXT:    vcvtph2ps %xmm3, %xmm3
+; AVX512-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX512-NEXT:    vucomiss %xmm4, %xmm3
+; AVX512-NEXT:    movl $65535, %edx # imm = 0xFFFF
+; AVX512-NEXT:    cmovnel %eax, %edx
+; AVX512-NEXT:    cmovpl %eax, %edx
+; AVX512-NEXT:    vcvtph2ps %xmm2, %xmm3
+; AVX512-NEXT:    vucomiss %xmm4, %xmm3
+; AVX512-NEXT:    movl $65535, %esi # imm = 0xFFFF
+; AVX512-NEXT:    cmovnel %eax, %esi
+; AVX512-NEXT:    cmovpl %eax, %esi
+; AVX512-NEXT:    vmovd %esi, %xmm3
+; AVX512-NEXT:    vpinsrw $1, %edx, %xmm3, %xmm3
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm5 = xmm2[1,1,1,1]
+; AVX512-NEXT:    vcvtph2ps %xmm5, %xmm5
+; AVX512-NEXT:    vucomiss %xmm4, %xmm5
+; AVX512-NEXT:    movl $65535, %edx # imm = 0xFFFF
+; AVX512-NEXT:    cmovnel %eax, %edx
+; AVX512-NEXT:    cmovpl %eax, %edx
+; AVX512-NEXT:    vpinsrw $2, %edx, %xmm3, %xmm3
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm2[3,3,3,3,4,5,6,7]
+; AVX512-NEXT:    vcvtph2ps %xmm5, %xmm5
+; AVX512-NEXT:    vucomiss %xmm4, %xmm5
+; AVX512-NEXT:    movl $65535, %edx # imm = 0xFFFF
+; AVX512-NEXT:    cmovnel %eax, %edx
+; AVX512-NEXT:    cmovpl %eax, %edx
+; AVX512-NEXT:    vpinsrw $3, %edx, %xmm3, %xmm3
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm5 = xmm2[2,3,2,3]
+; AVX512-NEXT:    vcvtph2ps %xmm5, %xmm5
+; AVX512-NEXT:    vucomiss %xmm4, %xmm5
+; AVX512-NEXT:    movl $65535, %edx # imm = 0xFFFF
+; AVX512-NEXT:    cmovnel %eax, %edx
+; AVX512-NEXT:    cmovpl %eax, %edx
+; AVX512-NEXT:    vpinsrw $4, %edx, %xmm3, %xmm3
+; AVX512-NEXT:    vpsrldq {{.*#+}} xmm5 = xmm2[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512-NEXT:    vcvtph2ps %xmm5, %xmm5
+; AVX512-NEXT:    vucomiss %xmm4, %xmm5
+; AVX512-NEXT:    movl $65535, %edx # imm = 0xFFFF
+; AVX512-NEXT:    cmovnel %eax, %edx
+; AVX512-NEXT:    cmovpl %eax, %edx
+; AVX512-NEXT:    vpinsrw $5, %edx, %xmm3, %xmm3
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm5 = xmm2[3,3,3,3]
+; AVX512-NEXT:    vcvtph2ps %xmm5, %xmm5
+; AVX512-NEXT:    vucomiss %xmm4, %xmm5
+; AVX512-NEXT:    movl $65535, %edx # imm = 0xFFFF
+; AVX512-NEXT:    cmovnel %eax, %edx
+; AVX512-NEXT:    cmovpl %eax, %edx
+; AVX512-NEXT:    vpinsrw $6, %edx, %xmm3, %xmm3
+; AVX512-NEXT:    vpsrldq {{.*#+}} xmm5 = xmm2[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512-NEXT:    vcvtph2ps %xmm5, %xmm5
+; AVX512-NEXT:    vucomiss %xmm4, %xmm5
+; AVX512-NEXT:    cmovnel %eax, %ecx
+; AVX512-NEXT:    cmovpl %eax, %ecx
+; AVX512-NEXT:    vpinsrw $7, %ecx, %xmm3, %xmm3
+; AVX512-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX512-NEXT:    vpcmpeqw %xmm4, %xmm0, %xmm5
+; AVX512-NEXT:    vpblendvb %xmm5, %xmm0, %xmm2, %xmm0
+; AVX512-NEXT:    vpcmpeqw %xmm4, %xmm1, %xmm4
+; AVX512-NEXT:    vpblendvb %xmm4, %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpblendvb %xmm3, %xmm0, %xmm2, %xmm0
+; AVX512-NEXT:    popq %rbx
+; AVX512-NEXT:    .cfi_def_cfa_offset 48
+; AVX512-NEXT:    popq %r12
+; AVX512-NEXT:    .cfi_def_cfa_offset 40
+; AVX512-NEXT:    popq %r13
+; AVX512-NEXT:    .cfi_def_cfa_offset 32
+; AVX512-NEXT:    popq %r14
+; AVX512-NEXT:    .cfi_def_cfa_offset 24
+; AVX512-NEXT:    popq %r15
+; AVX512-NEXT:    .cfi_def_cfa_offset 16
+; AVX512-NEXT:    popq %rbp
+; AVX512-NEXT:    .cfi_def_cfa_offset 8
+; AVX512-NEXT:    retq
+;
+; X86-LABEL: test_fmaximum_v4f16:
+; X86:       # %bb.0:
+; X86-NEXT:    subl $164, %esp
+; X86-NEXT:    .cfi_def_cfa_offset 168
+; X86-NEXT:    vmovdqa %xmm0, %xmm2
+; X86-NEXT:    vmovdqu %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-NEXT:    vpsrlq $48, %xmm0, %xmm0
+; X86-NEXT:    vmovdqu %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm2[1,1,3,3]
+; X86-NEXT:    vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm1[1,1,3,3]
+; X86-NEXT:    vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-NEXT:    vpsrlq $48, %xmm1, %xmm0
+; X86-NEXT:    vmovdqu %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-NEXT:    vpsrld $16, %xmm2, %xmm0
+; X86-NEXT:    vmovdqu %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-NEXT:    vpsrld $16, %xmm1, %xmm0
+; X86-NEXT:    vmovdqu %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-NEXT:    vpextrw $0, %xmm1, (%esp)
+; X86-NEXT:    calll __extendhfsf2
+; X86-NEXT:    fstpt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Spill
+; X86-NEXT:    vmovdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
+; X86-NEXT:    vpextrw $0, %xmm0, (%esp)
+; X86-NEXT:    calll __extendhfsf2
+; X86-NEXT:    fstpt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Spill
+; X86-NEXT:    vmovdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
+; X86-NEXT:    vpextrw $0, %xmm0, (%esp)
+; X86-NEXT:    calll __extendhfsf2
+; X86-NEXT:    vmovdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
+; X86-NEXT:    vpextrw $0, %xmm0, (%esp)
+; X86-NEXT:    fstps {{[0-9]+}}(%esp)
+; X86-NEXT:    fldt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Reload
+; X86-NEXT:    fstps {{[0-9]+}}(%esp)
+; X86-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; X86-NEXT:    vmovd %xmm2, %eax
+; X86-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    js .LBB33_1
+; X86-NEXT:  # %bb.2:
+; X86-NEXT:    vmovdqa %xmm2, %xmm1
+; X86-NEXT:    jmp .LBB33_3
+; X86-NEXT:  .LBB33_1:
+; X86-NEXT:    vmovdqa %xmm0, %xmm1
+; X86-NEXT:    vmovdqa %xmm2, %xmm0
+; X86-NEXT:  .LBB33_3:
+; X86-NEXT:    vmaxss %xmm1, %xmm0, %xmm1
+; X86-NEXT:    vcmpunordss %xmm0, %xmm0, %xmm2
+; X86-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; X86-NEXT:    vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-NEXT:    calll __extendhfsf2
+; X86-NEXT:    vmovups {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
+; X86-NEXT:    vmovss %xmm0, (%esp)
+; X86-NEXT:    fstps {{[0-9]+}}(%esp)
+; X86-NEXT:    fldt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Reload
+; X86-NEXT:    fstps {{[0-9]+}}(%esp)
+; X86-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X86-NEXT:    vmovd %xmm1, %eax
+; X86-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    js .LBB33_4
+; X86-NEXT:  # %bb.5:
+; X86-NEXT:    vmovdqa %xmm1, %xmm2
+; X86-NEXT:    jmp .LBB33_6
+; X86-NEXT:  .LBB33_4:
+; X86-NEXT:    vmovdqa %xmm0, %xmm2
+; X86-NEXT:    vmovdqa %xmm1, %xmm0
+; X86-NEXT:  .LBB33_6:
+; X86-NEXT:    vmaxss %xmm2, %xmm0, %xmm1
+; X86-NEXT:    vcmpunordss %xmm0, %xmm0, %xmm2
+; X86-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; X86-NEXT:    vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-NEXT:    calll __truncsfhf2
+; X86-NEXT:    vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-NEXT:    vmovups {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
+; X86-NEXT:    vmovss %xmm0, (%esp)
+; X86-NEXT:    calll __truncsfhf2
+; X86-NEXT:    vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-NEXT:    vmovdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
+; X86-NEXT:    vpextrw $0, %xmm0, (%esp)
+; X86-NEXT:    calll __extendhfsf2
+; X86-NEXT:    fstpt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Spill
+; X86-NEXT:    vmovdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
+; X86-NEXT:    vpextrw $0, %xmm0, (%esp)
+; X86-NEXT:    calll __extendhfsf2
+; X86-NEXT:    fstpt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Spill
+; X86-NEXT:    vmovdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
+; X86-NEXT:    vpextrw $0, %xmm0, (%esp)
+; X86-NEXT:    calll __extendhfsf2
+; X86-NEXT:    vmovdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
+; X86-NEXT:    vpextrw $0, %xmm0, (%esp)
+; X86-NEXT:    fstps {{[0-9]+}}(%esp)
+; X86-NEXT:    fldt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Reload
+; X86-NEXT:    fstps {{[0-9]+}}(%esp)
+; X86-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X86-NEXT:    vmovd %xmm1, %eax
+; X86-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    js .LBB33_7
+; X86-NEXT:  # %bb.8:
+; X86-NEXT:    vmovdqa %xmm1, %xmm2
+; X86-NEXT:    jmp .LBB33_9
+; X86-NEXT:  .LBB33_7:
+; X86-NEXT:    vmovdqa %xmm0, %xmm2
+; X86-NEXT:    vmovdqa %xmm1, %xmm0
+; X86-NEXT:  .LBB33_9:
+; X86-NEXT:    vmaxss %xmm2, %xmm0, %xmm1
+; X86-NEXT:    vcmpunordss %xmm0, %xmm0, %xmm2
+; X86-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; X86-NEXT:    vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-NEXT:    calll __extendhfsf2
+; X86-NEXT:    vmovups {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
+; X86-NEXT:    vmovss %xmm0, (%esp)
+; X86-NEXT:    fstps {{[0-9]+}}(%esp)
+; X86-NEXT:    fldt {{[-0-9]+}}(%e{{[sb]}}p) # 10-byte Folded Reload
+; X86-NEXT:    fstps {{[0-9]+}}(%esp)
+; X86-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X86-NEXT:    vmovd %xmm1, %eax
+; X86-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    js .LBB33_10
+; X86-NEXT:  # %bb.11:
+; X86-NEXT:    vmovdqa %xmm1, %xmm2
+; X86-NEXT:    jmp .LBB33_12
+; X86-NEXT:  .LBB33_10:
+; X86-NEXT:    vmovdqa %xmm0, %xmm2
+; X86-NEXT:    vmovdqa %xmm1, %xmm0
+; X86-NEXT:  .LBB33_12:
+; X86-NEXT:    vmaxss %xmm2, %xmm0, %xmm1
+; X86-NEXT:    vcmpunordss %xmm0, %xmm0, %xmm2
+; X86-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; X86-NEXT:    vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-NEXT:    calll __truncsfhf2
+; X86-NEXT:    vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-NEXT:    vmovdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
+; X86-NEXT:    vmovd %xmm0, (%esp)
+; X86-NEXT:    calll __truncsfhf2
+; X86-NEXT:    vmovdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm1 # 16-byte Reload
+; X86-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; X86-NEXT:    vmovdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm1 # 16-byte Reload
+; X86-NEXT:    vpunpcklwd {{[-0-9]+}}(%e{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload
+; X86-NEXT:    # xmm1 = xmm1[0],mem[0],xmm1[1],mem[1],xmm1[2],mem[2],xmm1[3],mem[3]
+; X86-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],zero,zero
+; X86-NEXT:    addl $164, %esp
+; X86-NEXT:    .cfi_def_cfa_offset 4
+; X86-NEXT:    retl
+  %r = call <4 x half> @llvm.maximum.v4f16(<4 x half> %x, <4 x half> %y)
+  ret <4 x half> %r
+}
+
+define <4 x bfloat> @test_fmaximum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) {
+; SSE2-LABEL: test_fmaximum_v4bf16:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pushq %rbp
+; SSE2-NEXT:    .cfi_def_cfa_offset 16
+; SSE2-NEXT:    pushq %r15
+; SSE2-NEXT:    .cfi_def_cfa_offset 24
+; SSE2-NEXT:    pushq %r14
+; SSE2-NEXT:    .cfi_def_cfa_offset 32
+; SSE2-NEXT:    pushq %rbx
+; SSE2-NEXT:    .cfi_def_cfa_offset 40
+; SSE2-NEXT:    subq $56, %rsp
+; SSE2-NEXT:    .cfi_def_cfa_offset 96
+; SSE2-NEXT:    .cfi_offset %rbx, -40
+; SSE2-NEXT:    .cfi_offset %r14, -32
+; SSE2-NEXT:    .cfi_offset %r15, -24
+; SSE2-NEXT:    .cfi_offset %rbp, -16
+; SSE2-NEXT:    movdqa %xmm1, %xmm4
+; SSE2-NEXT:    movdqa %xmm0, %xmm5
+; SSE2-NEXT:    pextrw $0, %xmm1, %r14d
+; SSE2-NEXT:    pextrw $0, %xmm0, %r15d
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    psrld $16, %xmm0
+; SSE2-NEXT:    pextrw $0, %xmm0, %eax
+; SSE2-NEXT:    movdqa %xmm5, %xmm0
+; SSE2-NEXT:    psrld $16, %xmm0
+; SSE2-NEXT:    pextrw $0, %xmm0, %ecx
+; SSE2-NEXT:    shll $16, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm3
+; SSE2-NEXT:    shll $16, %eax
+; SSE2-NEXT:    movd %eax, %xmm2
+; SSE2-NEXT:    testl %ecx, %ecx
+; SSE2-NEXT:    movdqa %xmm3, %xmm1
+; SSE2-NEXT:    js .LBB34_2
+; SSE2-NEXT:  # %bb.1:
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:  .LBB34_2:
+; SSE2-NEXT:    movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE2-NEXT:    shufps {{.*#+}} xmm4 = xmm4[1,1,1,1]
+; SSE2-NEXT:    movdqa %xmm5, (%rsp) # 16-byte Spill
+; SSE2-NEXT:    shufps {{.*#+}} xmm5 = xmm5[1,1,1,1]
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    cmpunordss %xmm1, %xmm0
+; SSE2-NEXT:    movaps %xmm0, %xmm6
+; SSE2-NEXT:    andps %xmm1, %xmm6
+; SSE2-NEXT:    js .LBB34_4
+; SSE2-NEXT:  # %bb.3:
+; SSE2-NEXT:    movdqa %xmm3, %xmm2
+; SSE2-NEXT:  .LBB34_4:
+; SSE2-NEXT:    pextrw $0, %xmm4, %ebp
+; SSE2-NEXT:    pextrw $0, %xmm5, %ebx
+; SSE2-NEXT:    maxss %xmm2, %xmm1
+; SSE2-NEXT:    andnps %xmm1, %xmm0
+; SSE2-NEXT:    orps %xmm6, %xmm0
+; SSE2-NEXT:    callq __truncsfbf2@PLT
+; SSE2-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE2-NEXT:    shll $16, %r15d
+; SSE2-NEXT:    movd %r15d, %xmm3
+; SSE2-NEXT:    shll $16, %r14d
+; SSE2-NEXT:    movd %r14d, %xmm2
+; SSE2-NEXT:    testl %r15d, %r15d
+; SSE2-NEXT:    movdqa %xmm3, %xmm1
+; SSE2-NEXT:    js .LBB34_6
+; SSE2-NEXT:  # %bb.5:
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:  .LBB34_6:
+; SSE2-NEXT:    movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
+; SSE2-NEXT:    psrlq $48, %xmm5
+; SSE2-NEXT:    movdqa (%rsp), %xmm6 # 16-byte Reload
+; SSE2-NEXT:    psrlq $48, %xmm6
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    cmpunordss %xmm1, %xmm0
+; SSE2-NEXT:    movaps %xmm0, %xmm4
+; SSE2-NEXT:    andps %xmm1, %xmm4
+; SSE2-NEXT:    js .LBB34_8
+; SSE2-NEXT:  # %bb.7:
+; SSE2-NEXT:    movdqa %xmm3, %xmm2
+; SSE2-NEXT:  .LBB34_8:
+; SSE2-NEXT:    pextrw $0, %xmm5, %r15d
+; SSE2-NEXT:    pextrw $0, %xmm6, %r14d
+; SSE2-NEXT:    maxss %xmm2, %xmm1
+; SSE2-NEXT:    andnps %xmm1, %xmm0
+; SSE2-NEXT:    orps %xmm4, %xmm0
+; SSE2-NEXT:    callq __truncsfbf2@PLT
+; SSE2-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE2-NEXT:    shll $16, %ebx
+; SSE2-NEXT:    movd %ebx, %xmm1
+; SSE2-NEXT:    shll $16, %ebp
+; SSE2-NEXT:    movd %ebp, %xmm3
+; SSE2-NEXT:    testl %ebx, %ebx
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    js .LBB34_10
+; SSE2-NEXT:  # %bb.9:
+; SSE2-NEXT:    movdqa %xmm3, %xmm2
+; SSE2-NEXT:  .LBB34_10:
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    cmpunordss %xmm2, %xmm0
+; SSE2-NEXT:    movaps %xmm0, %xmm4
+; SSE2-NEXT:    andps %xmm2, %xmm4
+; SSE2-NEXT:    js .LBB34_12
+; SSE2-NEXT:  # %bb.11:
+; SSE2-NEXT:    movdqa %xmm1, %xmm3
+; SSE2-NEXT:  .LBB34_12:
+; SSE2-NEXT:    maxss %xmm3, %xmm2
+; SSE2-NEXT:    andnps %xmm2, %xmm0
+; SSE2-NEXT:    orps %xmm4, %xmm0
+; SSE2-NEXT:    callq __truncsfbf2@PLT
+; SSE2-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
+; SSE2-NEXT:    shll $16, %r14d
+; SSE2-NEXT:    movd %r14d, %xmm1
+; SSE2-NEXT:    shll $16, %r15d
+; SSE2-NEXT:    movd %r15d, %xmm3
+; SSE2-NEXT:    testl %r14d, %r14d
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    js .LBB34_14
+; SSE2-NEXT:  # %bb.13:
+; SSE2-NEXT:    movdqa %xmm3, %xmm2
+; SSE2-NEXT:  .LBB34_14:
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    cmpunordss %xmm2, %xmm0
+; SSE2-NEXT:    movaps %xmm0, %xmm4
+; SSE2-NEXT:    andps %xmm2, %xmm4
+; SSE2-NEXT:    js .LBB34_16
+; SSE2-NEXT:  # %bb.15:
+; SSE2-NEXT:    movdqa %xmm1, %xmm3
+; SSE2-NEXT:  .LBB34_16:
+; SSE2-NEXT:    maxss %xmm3, %xmm2
+; SSE2-NEXT:    andnps %xmm2, %xmm0
+; SSE2-NEXT:    orps %xmm4, %xmm0
+; SSE2-NEXT:    callq __truncsfbf2@PLT
+; SSE2-NEXT:    movdqa (%rsp), %xmm1 # 16-byte Reload
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; SSE2-NEXT:    movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; SSE2-NEXT:    punpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; SSE2-NEXT:    # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT:    addq $56, %rsp
+; SSE2-NEXT:    .cfi_def_cfa_offset 40
+; SSE2-NEXT:    popq %rbx
+; SSE2-NEXT:    .cfi_def_cfa_offset 32
+; SSE2-NEXT:    popq %r14
+; SSE2-NEXT:    .cfi_def_cfa_offset 24
+; SSE2-NEXT:    popq %r15
+; SSE2-NEXT:    .cfi_def_cfa_offset 16
+; SSE2-NEXT:    popq %rbp
+; SSE2-NEXT:    .cfi_def_cfa_offset 8
+; SSE2-NEXT:    retq
+;
+; AVX1-LABEL: test_fmaximum_v4bf16:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    pushq %rbp
+; AVX1-NEXT:    .cfi_def_cfa_offset 16
+; AVX1-NEXT:    pushq %r15
+; AVX1-NEXT:    .cfi_def_cfa_offset 24
+; AVX1-NEXT:    pushq %r14
+; AVX1-NEXT:    .cfi_def_cfa_offset 32
+; AVX1-NEXT:    pushq %r13
+; AVX1-NEXT:    .cfi_def_cfa_offset 40
+; AVX1-NEXT:    pushq %r12
+; AVX1-NEXT:    .cfi_def_cfa_offset 48
+; AVX1-NEXT:    pushq %rbx
+; AVX1-NEXT:    .cfi_def_cfa_offset 56
+; AVX1-NEXT:    subq $56, %rsp
+; AVX1-NEXT:    .cfi_def_cfa_offset 112
+; AVX1-NEXT:    .cfi_offset %rbx, -56
+; AVX1-NEXT:    .cfi_offset %r12, -48
+; AVX1-NEXT:    .cfi_offset %r13, -40
+; AVX1-NEXT:    .cfi_offset %r14, -32
+; AVX1-NEXT:    .cfi_offset %r15, -24
+; AVX1-NEXT:    .cfi_offset %rbp, -16
+; AVX1-NEXT:    vpsrlq $48, %xmm0, %xmm2
+; AVX1-NEXT:    vpsrlq $48, %xmm1, %xmm3
+; AVX1-NEXT:    vmovshdup {{.*#+}} xmm4 = xmm0[1,1,3,3]
+; AVX1-NEXT:    vpextrw $0, %xmm4, %ebx
+; AVX1-NEXT:    vmovshdup {{.*#+}} xmm4 = xmm1[1,1,3,3]
+; AVX1-NEXT:    vpextrw $0, %xmm4, %ebp
+; AVX1-NEXT:    vpextrw $0, %xmm0, %r12d
+; AVX1-NEXT:    vpextrw $0, %xmm1, %r13d
+; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm0
+; AVX1-NEXT:    vpextrw $0, %xmm0, %eax
+; AVX1-NEXT:    vpsrld $16, %xmm1, %xmm0
+; AVX1-NEXT:    vpextrw $0, %xmm0, %ecx
+; AVX1-NEXT:    shll $16, %ecx
+; AVX1-NEXT:    vmovd %ecx, %xmm0
+; AVX1-NEXT:    shll $16, %eax
+; AVX1-NEXT:    vmovd %eax, %xmm4
+; AVX1-NEXT:    js .LBB34_1
+; AVX1-NEXT:  # %bb.2:
+; AVX1-NEXT:    vmovdqa %xmm4, %xmm1
+; AVX1-NEXT:    jmp .LBB34_3
+; AVX1-NEXT:  .LBB34_1:
+; AVX1-NEXT:    vmovdqa %xmm0, %xmm1
+; AVX1-NEXT:    vmovdqa %xmm4, %xmm0
+; AVX1-NEXT:  .LBB34_3:
+; AVX1-NEXT:    vpextrw $0, %xmm2, %r14d
+; AVX1-NEXT:    vpextrw $0, %xmm3, %r15d
+; AVX1-NEXT:    vmaxss %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vcmpunordss %xmm0, %xmm0, %xmm2
+; AVX1-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    callq __truncsfbf2@PLT
+; AVX1-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT:    shll $16, %r13d
+; AVX1-NEXT:    vmovd %r13d, %xmm0
+; AVX1-NEXT:    shll $16, %r12d
+; AVX1-NEXT:    vmovd %r12d, %xmm2
+; AVX1-NEXT:    js .LBB34_4
+; AVX1-NEXT:  # %bb.5:
+; AVX1-NEXT:    vmovdqa %xmm2, %xmm1
+; AVX1-NEXT:    jmp .LBB34_6
+; AVX1-NEXT:  .LBB34_4:
+; AVX1-NEXT:    vmovdqa %xmm0, %xmm1
+; AVX1-NEXT:    vmovdqa %xmm2, %xmm0
+; AVX1-NEXT:  .LBB34_6:
+; AVX1-NEXT:    vmaxss %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vcmpunordss %xmm0, %xmm0, %xmm2
+; AVX1-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    callq __truncsfbf2@PLT
+; AVX1-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT:    shll $16, %ebp
+; AVX1-NEXT:    vmovd %ebp, %xmm0
+; AVX1-NEXT:    shll $16, %ebx
+; AVX1-NEXT:    vmovd %ebx, %xmm2
+; AVX1-NEXT:    js .LBB34_7
+; AVX1-NEXT:  # %bb.8:
+; AVX1-NEXT:    vmovdqa %xmm2, %xmm1
+; AVX1-NEXT:    jmp .LBB34_9
+; AVX1-NEXT:  .LBB34_7:
+; AVX1-NEXT:    vmovdqa %xmm0, %xmm1
+; AVX1-NEXT:    vmovdqa %xmm2, %xmm0
+; AVX1-NEXT:  .LBB34_9:
+; AVX1-NEXT:    vmaxss %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vcmpunordss %xmm0, %xmm0, %xmm2
+; AVX1-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    callq __truncsfbf2@PLT
+; AVX1-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
+; AVX1-NEXT:    shll $16, %r15d
+; AVX1-NEXT:    vmovd %r15d, %xmm0
+; AVX1-NEXT:    shll $16, %r14d
+; AVX1-NEXT:    vmovd %r14d, %xmm2
+; AVX1-NEXT:    js .LBB34_10
+; AVX1-NEXT:  # %bb.11:
+; AVX1-NEXT:    vmovdqa %xmm2, %xmm1
+; AVX1-NEXT:    jmp .LBB34_12
+; AVX1-NEXT:  .LBB34_10:
+; AVX1-NEXT:    vmovdqa %xmm0, %xmm1
+; AVX1-NEXT:    vmovdqa %xmm2, %xmm0
+; AVX1-NEXT:  .LBB34_12:
+; AVX1-NEXT:    vmaxss %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vcmpunordss %xmm0, %xmm0, %xmm2
+; AVX1-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    callq __truncsfbf2@PLT
+; AVX1-NEXT:    vmovdqa (%rsp), %xmm1 # 16-byte Reload
+; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; AVX1-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; AVX1-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload
+; AVX1-NEXT:    # xmm1 = xmm1[0],mem[0],xmm1[1],mem[1],xmm1[2],mem[2],xmm1[3],mem[3]
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],zero,zero
+; AVX1-NEXT:    addq $56, %rsp
+; AVX1-NEXT:    .cfi_def_cfa_offset 56
+; AVX1-NEXT:    popq %rbx
+; AVX1-NEXT:    .cfi_def_cfa_offset 48
+; AVX1-NEXT:    popq %r12
+; AVX1-NEXT:    .cfi_def_cfa_offset 40
+; AVX1-NEXT:    popq %r13
+; AVX1-NEXT:    .cfi_def_cfa_offset 32
+; AVX1-NEXT:    popq %r14
+; AVX1-NEXT:    .cfi_def_cfa_offset 24
+; AVX1-NEXT:    popq %r15
+; AVX1-NEXT:    .cfi_def_cfa_offset 16
+; AVX1-NEXT:    popq %rbp
+; AVX1-NEXT:    .cfi_def_cfa_offset 8
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: test_fmaximum_v4bf16:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    pushq %rbp
+; AVX512-NEXT:    .cfi_def_cfa_offset 16
+; AVX512-NEXT:    pushq %r15
+; AVX512-NEXT:    .cfi_def_cfa_offset 24
+; AVX512-NEXT:    pushq %r14
+; AVX512-NEXT:    .cfi_def_cfa_offset 32
+; AVX512-NEXT:    pushq %r13
+; AVX512-NEXT:    .cfi_def_cfa_offset 40
+; AVX512-NEXT:    pushq %r12
+; AVX512-NEXT:    .cfi_def_cfa_offset 48
+; AVX512-NEXT:    pushq %rbx
+; AVX512-NEXT:    .cfi_def_cfa_offset 56
+; AVX512-NEXT:    pushq %rax
+; AVX512-NEXT:    .cfi_def_cfa_offset 64
+; AVX512-NEXT:    .cfi_offset %rbx, -56
+; AVX512-NEXT:    .cfi_offset %r12, -48
+; AVX512-NEXT:    .cfi_offset %r13, -40
+; AVX512-NEXT:    .cfi_offset %r14, -32
+; AVX512-NEXT:    .cfi_offset %r15, -24
+; AVX512-NEXT:    .cfi_offset %rbp, -16
+; AVX512-NEXT:    vmovq %xmm1, %r13
+; AVX512-NEXT:    movq %r13, %rbx
+; AVX512-NEXT:    shrq $32, %rbx
+; AVX512-NEXT:    vmovq %xmm0, %rbp
+; AVX512-NEXT:    movq %rbp, %r14
+; AVX512-NEXT:    shrq $32, %r14
+; AVX512-NEXT:    movq %r13, %r15
+; AVX512-NEXT:    shrq $48, %r15
+; AVX512-NEXT:    movq %rbp, %r12
+; AVX512-NEXT:    shrq $48, %r12
+; AVX512-NEXT:    movl %ebp, %eax
+; AVX512-NEXT:    andl $-65536, %eax # imm = 0xFFFF0000
+; AVX512-NEXT:    sets %cl
+; AVX512-NEXT:    kmovw %ecx, %k1
+; AVX512-NEXT:    movl %r13d, %ecx
+; AVX512-NEXT:    andl $-65536, %ecx # imm = 0xFFFF0000
+; AVX512-NEXT:    vmovd %ecx, %xmm1
+; AVX512-NEXT:    vmovd %eax, %xmm0
+; AVX512-NEXT:    vmovdqa %xmm0, %xmm2
+; AVX512-NEXT:    vmovss %xmm1, %xmm2, %xmm2 {%k1}
+; AVX512-NEXT:    vmovss %xmm0, %xmm1, %xmm1 {%k1}
+; AVX512-NEXT:    vmaxss %xmm2, %xmm1, %xmm0
+; AVX512-NEXT:    vcmpunordss %xmm1, %xmm1, %k1
+; AVX512-NEXT:    vmovss %xmm1, %xmm0, %xmm0 {%k1}
+; AVX512-NEXT:    callq __truncsfbf2@PLT
+; AVX512-NEXT:    vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
+; AVX512-NEXT:    shll $16, %ebp
+; AVX512-NEXT:    sets %al
+; AVX512-NEXT:    kmovw %eax, %k1
+; AVX512-NEXT:    shll $16, %r13d
+; AVX512-NEXT:    vmovd %r13d, %xmm1
+; AVX512-NEXT:    vmovd %ebp, %xmm0
+; AVX512-NEXT:    vmovdqa %xmm0, %xmm2
+; AVX512-NEXT:    vmovss %xmm1, %xmm2, %xmm2 {%k1}
+; AVX512-NEXT:    vmovss %xmm0, %xmm1, %xmm1 {%k1}
+; AVX512-NEXT:    vmaxss %xmm2, %xmm1, %xmm0
+; AVX512-NEXT:    vcmpunordss %xmm1, %xmm1, %k1
+; AVX512-NEXT:    vmovss %xmm1, %xmm0, %xmm0 {%k1}
+; AVX512-NEXT:    callq __truncsfbf2@PLT
+; AVX512-NEXT:    vpextrw $0, %xmm0, (%rsp)
+; AVX512-NEXT:    shll $16, %r12d
+; AVX512-NEXT:    sets %al
+; AVX512-NEXT:    kmovw %eax, %k1
+; AVX512-NEXT:    shll $16, %r15d
+; AVX512-NEXT:    vmovd %r15d, %xmm1
+; AVX512-NEXT:    vmovd %r12d, %xmm0
+; AVX512-NEXT:    vmovdqa %xmm0, %xmm2
+; AVX512-NEXT:    vmovss %xmm1, %xmm2, %xmm2 {%k1}
+; AVX512-NEXT:    vmovss %xmm0, %xmm1, %xmm1 {%k1}
+; AVX512-NEXT:    vmaxss %xmm2, %xmm1, %xmm0
+; AVX512-NEXT:    vcmpunordss %xmm1, %xmm1, %k1
+; AVX512-NEXT:    vmovss %xmm1, %xmm0, %xmm0 {%k1}
+; AVX512-NEXT:    callq __truncsfbf2@PLT
+; AVX512-NEXT:    vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
+; AVX512-NEXT:    shll $16, %r14d
+; AVX512-NEXT:    sets %al
+; AVX512-NEXT:    kmovw %eax, %k1
+; AVX512-NEXT:    shll $16, %ebx
+; AVX512-NEXT:    vmovd %ebx, %xmm1
+; AVX512-NEXT:    vmovd %r14d, %xmm0
+; AVX512-NEXT:    vmovdqa %xmm0, %xmm2
+; AVX512-NEXT:    vmovss %xmm1, %xmm2, %xmm2 {%k1}
+; AVX512-NEXT:    vmovss %xmm0, %xmm1, %xmm1 {%k1}
+; AVX512-NEXT:    vmaxss %xmm2, %xmm1, %xmm0
+; AVX512-NEXT:    vcmpunordss %xmm1, %xmm1, %k1
+; AVX512-NEXT:    vmovss %xmm1, %xmm0, %xmm0 {%k1}
+; AVX512-NEXT:    callq __truncsfbf2@PLT
+; AVX512-NEXT:    vpextrw $0, %xmm0, {{[0-9]+}}(%rsp)
+; AVX512-NEXT:    vmovaps (%rsp), %xmm0
+; AVX512-NEXT:    addq $8, %rsp
+; AVX512-NEXT:    .cfi_def_cfa_offset 56
+; AVX512-NEXT:    popq %rbx
+; AVX512-NEXT:    .cfi_def_cfa_offset 48
+; AVX512-NEXT:    popq %r12
+; AVX512-NEXT:    .cfi_def_cfa_offset 40
+; AVX512-NEXT:    popq %r13
+; AVX512-NEXT:    .cfi_def_cfa_offset 32
+; AVX512-NEXT:    popq %r14
+; AVX512-NEXT:    .cfi_def_cfa_offset 24
+; AVX512-NEXT:    popq %r15
+; AVX512-NEXT:    .cfi_def_cfa_offset 16
+; AVX512-NEXT:    popq %rbp
+; AVX512-NEXT:    .cfi_def_cfa_offset 8
+; AVX512-NEXT:    retq
+;
+; X86-LABEL: test_fmaximum_v4bf16:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    .cfi_def_cfa_offset 8
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    .cfi_def_cfa_offset 12
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    .cfi_def_cfa_offset 16
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    .cfi_def_cfa_offset 20
+; X86-NEXT:    subl $68, %esp
+; X86-NEXT:    .cfi_def_cfa_offset 88
+; X86-NEXT:    .cfi_offset %esi, -20
+; X86-NEXT:    .cfi_offset %edi, -16
+; X86-NEXT:    .cfi_offset %ebx, -12
+; X86-NEXT:    .cfi_offset %ebp, -8
+; X86-NEXT:    vpsrlq $48, %xmm0, %xmm2
+; X86-NEXT:    vpsrlq $48, %xmm1, %xmm3
+; X86-NEXT:    vmovshdup {{.*#+}} xmm4 = xmm0[1,1,3,3]
+; X86-NEXT:    vpextrw $0, %xmm4, %esi
+; X86-NEXT:    vmovshdup {{.*#+}} xmm4 = xmm1[1,1,3,3]
+; X86-NEXT:    vpextrw $0, %xmm4, %ebx
+; X86-NEXT:    vpextrw $0, %xmm0, %eax
+; X86-NEXT:    vpextrw $0, %xmm1, %ecx
+; X86-NEXT:    vpsrld $16, %xmm0, %xmm0
+; X86-NEXT:    vpextrw $0, %xmm0, %edx
+; X86-NEXT:    vpsrld $16, %xmm1, %xmm0
+; X86-NEXT:    vpextrw $0, %xmm0, %edi
+; X86-NEXT:    shll $16, %edi
+; X86-NEXT:    vmovd %edi, %xmm0
+; X86-NEXT:    shll $16, %edx
+; X86-NEXT:    vmovd %edx, %xmm4
+; X86-NEXT:    js .LBB34_1
+; X86-NEXT:  # %bb.2:
+; X86-NEXT:    vmovdqa %xmm4, %xmm1
+; X86-NEXT:    jmp .LBB34_3
+; X86-NEXT:  .LBB34_1:
+; X86-NEXT:    vmovdqa %xmm0, %xmm1
+; X86-NEXT:    vmovdqa %xmm4, %xmm0
+; X86-NEXT:  .LBB34_3:
+; X86-NEXT:    vpextrw $0, %xmm2, %edi
+; X86-NEXT:    vpextrw $0, %xmm3, %ebp
+; X86-NEXT:    vmaxss %xmm1, %xmm0, %xmm1
+; X86-NEXT:    vcmpunordss %xmm0, %xmm0, %xmm2
+; X86-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; X86-NEXT:    vmovss %xmm0, (%esp)
+; X86-NEXT:    shll $16, %ecx
+; X86-NEXT:    vmovd %ecx, %xmm0
+; X86-NEXT:    shll $16, %eax
+; X86-NEXT:    vmovd %eax, %xmm2
+; X86-NEXT:    js .LBB34_4
+; X86-NEXT:  # %bb.5:
+; X86-NEXT:    vmovdqa %xmm2, %xmm1
+; X86-NEXT:    jmp .LBB34_6
+; X86-NEXT:  .LBB34_4:
+; X86-NEXT:    vmovdqa %xmm0, %xmm1
+; X86-NEXT:    vmovdqa %xmm2, %xmm0
+; X86-NEXT:  .LBB34_6:
+; X86-NEXT:    vmaxss %xmm1, %xmm0, %xmm1
+; X86-NEXT:    vcmpunordss %xmm0, %xmm0, %xmm2
+; X86-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; X86-NEXT:    vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-NEXT:    calll __truncsfbf2
+; X86-NEXT:    vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-NEXT:    vmovups {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
+; X86-NEXT:    vmovss %xmm0, (%esp)
+; X86-NEXT:    shll $16, %ebx
+; X86-NEXT:    vmovd %ebx, %xmm0
+; X86-NEXT:    shll $16, %esi
+; X86-NEXT:    vmovd %esi, %xmm2
+; X86-NEXT:    js .LBB34_7
+; X86-NEXT:  # %bb.8:
+; X86-NEXT:    vmovdqa %xmm2, %xmm1
+; X86-NEXT:    jmp .LBB34_9
+; X86-NEXT:  .LBB34_7:
+; X86-NEXT:    vmovdqa %xmm0, %xmm1
+; X86-NEXT:    vmovdqa %xmm2, %xmm0
+; X86-NEXT:  .LBB34_9:
+; X86-NEXT:    vmaxss %xmm1, %xmm0, %xmm1
+; X86-NEXT:    vcmpunordss %xmm0, %xmm0, %xmm2
+; X86-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; X86-NEXT:    vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-NEXT:    calll __truncsfbf2
+; X86-NEXT:    vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-NEXT:    vmovups {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
+; X86-NEXT:    vmovss %xmm0, (%esp)
+; X86-NEXT:    shll $16, %ebp
+; X86-NEXT:    vmovd %ebp, %xmm0
+; X86-NEXT:    shll $16, %edi
+; X86-NEXT:    vmovd %edi, %xmm2
+; X86-NEXT:    js .LBB34_10
+; X86-NEXT:  # %bb.11:
+; X86-NEXT:    vmovdqa %xmm2, %xmm1
+; X86-NEXT:    jmp .LBB34_12
+; X86-NEXT:  .LBB34_10:
+; X86-NEXT:    vmovdqa %xmm0, %xmm1
+; X86-NEXT:    vmovdqa %xmm2, %xmm0
+; X86-NEXT:  .LBB34_12:
+; X86-NEXT:    vmaxss %xmm1, %xmm0, %xmm1
+; X86-NEXT:    vcmpunordss %xmm0, %xmm0, %xmm2
+; X86-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; X86-NEXT:    vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-NEXT:    calll __truncsfbf2
+; X86-NEXT:    vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-NEXT:    vmovdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
+; X86-NEXT:    vmovd %xmm0, (%esp)
+; X86-NEXT:    calll __truncsfbf2
+; X86-NEXT:    vmovdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm1 # 16-byte Reload
+; X86-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; X86-NEXT:    vmovdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm1 # 16-byte Reload
+; X86-NEXT:    vpunpcklwd {{[-0-9]+}}(%e{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload
+; X86-NEXT:    # xmm1 = xmm1[0],mem[0],xmm1[1],mem[1],xmm1[2],mem[2],xmm1[3],mem[3]
+; X86-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0],xmm0[0],zero,zero
+; X86-NEXT:    addl $68, %esp
+; X86-NEXT:    .cfi_def_cfa_offset 20
+; X86-NEXT:    popl %esi
+; X86-NEXT:    .cfi_def_cfa_offset 16
+; X86-NEXT:    popl %edi
+; X86-NEXT:    .cfi_def_cfa_offset 12
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    .cfi_def_cfa_offset 8
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    .cfi_def_cfa_offset 4
+; X86-NEXT:    retl
+  %r = call <4 x bfloat> @llvm.maximum.v4bf16(<4 x bfloat> %x, <4 x bfloat> %y)
+  ret <4 x bfloat> %r
+}

From a9ce1818174b3e690545b573cc5804d73491ca1a Mon Sep 17 00:00:00 2001
From: David Majnemer <david.majnemer@gmail.com>
Date: Tue, 20 Aug 2024 04:51:15 +0000
Subject: [PATCH 440/441] [NVPTX] Fix bugs involving maximum/minimum and bf16

We would crash on sufficiently old NV hardware (Volta or so) due to
incorrectly marking certain operations legal.
---
 llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp  |   76 +-
 llvm/test/CodeGen/NVPTX/bf16-instructions.ll | 1559 +++++++++++++++---
 2 files changed, 1382 insertions(+), 253 deletions(-)

diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index 577141299b94963..73791102fc04dee 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -429,29 +429,50 @@ NVPTXTargetLowering::NVPTXTargetLowering(const NVPTXTargetMachine &TM,
 
   auto setFP16OperationAction = [&](unsigned Op, MVT VT, LegalizeAction Action,
                                     LegalizeAction NoF16Action) {
-    setOperationAction(Op, VT, STI.allowFP16Math() ? Action : NoF16Action);
+    bool IsOpSupported = STI.allowFP16Math();
+    switch (Op) {
+    // Several FP16 instructions are available on sm_80 only.
+    case ISD::FMINNUM:
+    case ISD::FMAXNUM:
+    case ISD::FMAXNUM_IEEE:
+    case ISD::FMINNUM_IEEE:
+    case ISD::FMAXIMUM:
+    case ISD::FMINIMUM:
+      IsOpSupported &= STI.getSmVersion() >= 80 && STI.getPTXVersion() >= 70;
+      break;
+    }
+    setOperationAction(Op, VT, IsOpSupported ? Action : NoF16Action);
   };
 
   auto setBF16OperationAction = [&](unsigned Op, MVT VT, LegalizeAction Action,
                                     LegalizeAction NoBF16Action) {
     bool IsOpSupported = STI.hasBF16Math();
-    // Few instructions are available on sm_90 only
-    switch(Op) {
-      case ISD::FADD:
-      case ISD::FMUL:
-      case ISD::FSUB:
-      case ISD::SELECT:
-      case ISD::SELECT_CC:
-      case ISD::SETCC:
-      case ISD::FEXP2:
-      case ISD::FCEIL:
-      case ISD::FFLOOR:
-      case ISD::FNEARBYINT:
-      case ISD::FRINT:
-      case ISD::FROUNDEVEN:
-      case ISD::FTRUNC:
-        IsOpSupported = STI.getSmVersion() >= 90 && STI.getPTXVersion() >= 78;
-        break;
+    switch (Op) {
+    // Several BF16 instructions are available on sm_90 only.
+    case ISD::FADD:
+    case ISD::FMUL:
+    case ISD::FSUB:
+    case ISD::SELECT:
+    case ISD::SELECT_CC:
+    case ISD::SETCC:
+    case ISD::FEXP2:
+    case ISD::FCEIL:
+    case ISD::FFLOOR:
+    case ISD::FNEARBYINT:
+    case ISD::FRINT:
+    case ISD::FROUNDEVEN:
+    case ISD::FTRUNC:
+      IsOpSupported = STI.getSmVersion() >= 90 && STI.getPTXVersion() >= 78;
+      break;
+    // Several BF16 instructions are available on sm_80 only.
+    case ISD::FMINNUM:
+    case ISD::FMAXNUM:
+    case ISD::FMAXNUM_IEEE:
+    case ISD::FMINNUM_IEEE:
+    case ISD::FMAXIMUM:
+    case ISD::FMINIMUM:
+      IsOpSupported &= STI.getSmVersion() >= 80 && STI.getPTXVersion() >= 70;
+      break;
     }
     setOperationAction(
         Op, VT, IsOpSupported ? Action : NoBF16Action);
@@ -838,26 +859,23 @@ NVPTXTargetLowering::NVPTXTargetLowering(const NVPTXTargetMachine &TM,
       AddPromotedToType(Op, MVT::bf16, MVT::f32);
   }
 
-  // max.f16, max.f16x2 and max.NaN are supported on sm_80+.
-  auto GetMinMaxAction = [&](LegalizeAction NotSm80Action) {
-    bool IsAtLeastSm80 = STI.getSmVersion() >= 80 && STI.getPTXVersion() >= 70;
-    return IsAtLeastSm80 ? Legal : NotSm80Action;
-  };
   for (const auto &Op : {ISD::FMINNUM, ISD::FMAXNUM}) {
-    setFP16OperationAction(Op, MVT::f16, GetMinMaxAction(Promote), Promote);
     setOperationAction(Op, MVT::f32, Legal);
     setOperationAction(Op, MVT::f64, Legal);
-    setFP16OperationAction(Op, MVT::v2f16, GetMinMaxAction(Expand), Expand);
+    setFP16OperationAction(Op, MVT::f16, Legal, Promote);
+    setFP16OperationAction(Op, MVT::v2f16, Legal, Expand);
     setBF16OperationAction(Op, MVT::v2bf16, Legal, Expand);
     setBF16OperationAction(Op, MVT::bf16, Legal, Promote);
     if (getOperationAction(Op, MVT::bf16) == Promote)
       AddPromotedToType(Op, MVT::bf16, MVT::f32);
   }
+  bool SupportsF32MinMaxNaN =
+      STI.getSmVersion() >= 80 && STI.getPTXVersion() >= 70;
   for (const auto &Op : {ISD::FMINIMUM, ISD::FMAXIMUM}) {
-    setFP16OperationAction(Op, MVT::f16, GetMinMaxAction(Expand), Expand);
-    setFP16OperationAction(Op, MVT::bf16, Legal, Expand);
-    setOperationAction(Op, MVT::f32, GetMinMaxAction(Expand));
-    setFP16OperationAction(Op, MVT::v2f16, GetMinMaxAction(Expand), Expand);
+    setOperationAction(Op, MVT::f32, SupportsF32MinMaxNaN ? Legal : Expand);
+    setFP16OperationAction(Op, MVT::f16, Legal, Expand);
+    setFP16OperationAction(Op, MVT::v2f16, Legal, Expand);
+    setBF16OperationAction(Op, MVT::bf16, Legal, Expand);
     setBF16OperationAction(Op, MVT::v2bf16, Legal, Expand);
   }
 
diff --git a/llvm/test/CodeGen/NVPTX/bf16-instructions.ll b/llvm/test/CodeGen/NVPTX/bf16-instructions.ll
index 4cbab8bf24166c5..f029e5f322968b7 100644
--- a/llvm/test/CodeGen/NVPTX/bf16-instructions.ll
+++ b/llvm/test/CodeGen/NVPTX/bf16-instructions.ll
@@ -1,344 +1,1455 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx71 | FileCheck --check-prefixes=CHECK,SM70 %s
 ; RUN: llc < %s -march=nvptx64 -mcpu=sm_80 -mattr=+ptx71 | FileCheck --check-prefixes=CHECK,SM80 %s
 ; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx78 | FileCheck --check-prefixes=CHECK,SM90 %s
 ; RUN: %if ptxas-11.8 %{ llc < %s -march=nvptx64 -mcpu=sm_80 -mattr=+ptx71 | %ptxas-verify -arch=sm_80 %}
 ; RUN: %if ptxas-11.8 %{ llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx78 | %ptxas-verify -arch=sm_90 %}
 
+target triple = "nvptx64-nvidia-cuda"
+
 ; LDST: .b8 bfloat_array[8] = {1, 2, 3, 4, 5, 6, 7, 8};
 @"bfloat_array" = addrspace(1) constant [4 x bfloat]
                 [bfloat 0xR0201, bfloat 0xR0403, bfloat 0xR0605, bfloat 0xR0807]
 
-; CHECK-LABEL: test_fadd(
-; CHECK-DAG:  ld.param.b16    [[A:%rs[0-9]+]], [test_fadd_param_0];
-; CHECK-DAG:  ld.param.b16    [[B:%rs[0-9]+]], [test_fadd_param_1];
-; SM90:       add.rn.bf16     [[R:%rs[0-9]+]], [[A]], [[B]];
-;
-; SM80-DAG:   cvt.f32.bf16    [[FA:%f[0-9]+]], [[A]];
-; SM80-DAG:   cvt.f32.bf16    [[FB:%f[0-9]+]], [[B]];
-; SM80:       add.rn.f32      [[FR:%f[0-9]+]], [[FA]], [[FB]];
-; SM80:       cvt.rn.bf16.f32 [[R:%rs[0-9]+]], [[FR]];
-; CHECK-NEXT: st.param.b16    [func_retval0+0], [[R]];
-; CHECK-NEXT: ret;
-
 define bfloat @test_fadd(bfloat %0, bfloat %1) {
+; SM70-LABEL: test_fadd(
+; SM70:       {
+; SM70-NEXT:    .reg .pred %p<2>;
+; SM70-NEXT:    .reg .b16 %rs<3>;
+; SM70-NEXT:    .reg .b32 %r<11>;
+; SM70-NEXT:    .reg .f32 %f<4>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.u16 %r1, [test_fadd_param_1];
+; SM70-NEXT:    shl.b32 %r2, %r1, 16;
+; SM70-NEXT:    mov.b32 %f1, %r2;
+; SM70-NEXT:    ld.param.u16 %r3, [test_fadd_param_0];
+; SM70-NEXT:    shl.b32 %r4, %r3, 16;
+; SM70-NEXT:    mov.b32 %f2, %r4;
+; SM70-NEXT:    add.rn.f32 %f3, %f2, %f1;
+; SM70-NEXT:    mov.b32 %r5, %f3;
+; SM70-NEXT:    bfe.u32 %r6, %r5, 16, 1;
+; SM70-NEXT:    add.s32 %r7, %r6, %r5;
+; SM70-NEXT:    add.s32 %r8, %r7, 32767;
+; SM70-NEXT:    setp.nan.f32 %p1, %f3, %f3;
+; SM70-NEXT:    or.b32 %r9, %r5, 4194304;
+; SM70-NEXT:    selp.b32 %r10, %r9, %r8, %p1;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r10; }
+; SM70-NEXT:    st.param.b16 [func_retval0+0], %rs1;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_fadd(
+; SM80:       {
+; SM80-NEXT:    .reg .b16 %rs<4>;
+; SM80-NEXT:    .reg .f32 %f<4>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.b16 %rs1, [test_fadd_param_0];
+; SM80-NEXT:    ld.param.b16 %rs2, [test_fadd_param_1];
+; SM80-NEXT:    cvt.f32.bf16 %f1, %rs2;
+; SM80-NEXT:    cvt.f32.bf16 %f2, %rs1;
+; SM80-NEXT:    add.rn.f32 %f3, %f2, %f1;
+; SM80-NEXT:    cvt.rn.bf16.f32 %rs3, %f3;
+; SM80-NEXT:    st.param.b16 [func_retval0+0], %rs3;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_fadd(
+; SM90:       {
+; SM90-NEXT:    .reg .b16 %rs<4>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.b16 %rs1, [test_fadd_param_0];
+; SM90-NEXT:    ld.param.b16 %rs2, [test_fadd_param_1];
+; SM90-NEXT:    add.rn.bf16 %rs3, %rs1, %rs2;
+; SM90-NEXT:    st.param.b16 [func_retval0+0], %rs3;
+; SM90-NEXT:    ret;
   %3 = fadd bfloat %0, %1
   ret bfloat %3
 }
 
-; CHECK-LABEL: test_fsub(
-; CHECK-DAG:  ld.param.b16    [[A:%rs[0-9]+]], [test_fsub_param_0];
-; CHECK-DAG:  ld.param.b16    [[B:%rs[0-9]+]], [test_fsub_param_1];
-; SM90:       sub.rn.bf16     [[R:%rs[0-9]+]], [[A]], [[B]];
-;
-; SM80-DAG:   cvt.f32.bf16    [[FA:%f[0-9]+]], [[A]];
-; SM80-DAG:   cvt.f32.bf16    [[FB:%f[0-9]+]], [[B]];
-; SM80:       sub.rn.f32      [[FR:%f[0-9]+]], [[FA]], [[FB]];
-; SM80:       cvt.rn.bf16.f32 [[R:%rs[0-9]+]], [[FR]];
-; CHECK-NEXT: st.param.b16    [func_retval0+0], [[R]];
-; CHECK-NEXT: ret;
-
 define bfloat @test_fsub(bfloat %0, bfloat %1) {
+; SM70-LABEL: test_fsub(
+; SM70:       {
+; SM70-NEXT:    .reg .pred %p<2>;
+; SM70-NEXT:    .reg .b16 %rs<3>;
+; SM70-NEXT:    .reg .b32 %r<11>;
+; SM70-NEXT:    .reg .f32 %f<4>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.u16 %r1, [test_fsub_param_1];
+; SM70-NEXT:    shl.b32 %r2, %r1, 16;
+; SM70-NEXT:    mov.b32 %f1, %r2;
+; SM70-NEXT:    ld.param.u16 %r3, [test_fsub_param_0];
+; SM70-NEXT:    shl.b32 %r4, %r3, 16;
+; SM70-NEXT:    mov.b32 %f2, %r4;
+; SM70-NEXT:    sub.rn.f32 %f3, %f2, %f1;
+; SM70-NEXT:    mov.b32 %r5, %f3;
+; SM70-NEXT:    bfe.u32 %r6, %r5, 16, 1;
+; SM70-NEXT:    add.s32 %r7, %r6, %r5;
+; SM70-NEXT:    add.s32 %r8, %r7, 32767;
+; SM70-NEXT:    setp.nan.f32 %p1, %f3, %f3;
+; SM70-NEXT:    or.b32 %r9, %r5, 4194304;
+; SM70-NEXT:    selp.b32 %r10, %r9, %r8, %p1;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r10; }
+; SM70-NEXT:    st.param.b16 [func_retval0+0], %rs1;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_fsub(
+; SM80:       {
+; SM80-NEXT:    .reg .b16 %rs<4>;
+; SM80-NEXT:    .reg .f32 %f<4>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.b16 %rs1, [test_fsub_param_0];
+; SM80-NEXT:    ld.param.b16 %rs2, [test_fsub_param_1];
+; SM80-NEXT:    cvt.f32.bf16 %f1, %rs2;
+; SM80-NEXT:    cvt.f32.bf16 %f2, %rs1;
+; SM80-NEXT:    sub.rn.f32 %f3, %f2, %f1;
+; SM80-NEXT:    cvt.rn.bf16.f32 %rs3, %f3;
+; SM80-NEXT:    st.param.b16 [func_retval0+0], %rs3;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_fsub(
+; SM90:       {
+; SM90-NEXT:    .reg .b16 %rs<4>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.b16 %rs1, [test_fsub_param_0];
+; SM90-NEXT:    ld.param.b16 %rs2, [test_fsub_param_1];
+; SM90-NEXT:    sub.rn.bf16 %rs3, %rs1, %rs2;
+; SM90-NEXT:    st.param.b16 [func_retval0+0], %rs3;
+; SM90-NEXT:    ret;
   %3 = fsub bfloat %0, %1
   ret bfloat %3
 }
 
-; CHECK-LABEL: test_faddx2(
-; CHECK-DAG:  ld.param.b32    [[A:%r[0-9]+]], [test_faddx2_param_0];
-; CHECK-DAG:  ld.param.b32    [[B:%r[0-9]+]], [test_faddx2_param_1];
-; SM90:       add.rn.bf16x2   [[R:%r[0-9]+]], [[A]], [[B]];
-
-; SM80-DAG:   mov.b32         {[[A0:%rs[0-9]+]], [[A1:%rs[0-9]+]]}, [[A]];
-; SM80-DAG:   mov.b32         {[[B0:%rs[0-9]+]], [[B1:%rs[0-9]+]]}, [[B]];
-; SM80-DAG:   cvt.f32.bf16    [[FA1:%f[0-9]+]], [[A1]];
-; SM80-DAG:   cvt.f32.bf16    [[FA0:%f[0-9]+]], [[A0]];
-; SM80-DAG:   cvt.f32.bf16    [[FB0:%f[0-9]+]], [[B0]];
-; SM80-DAG:   cvt.f32.bf16    [[FB1:%f[0-9]+]], [[B1]];
-; SM80-DAG:   add.rn.f32      [[FR0:%f[0-9]+]], [[FA0]], [[FB0]];
-; SM80-DAG:   add.rn.f32      [[FR1:%f[0-9]+]], [[FA1]], [[FB1]];
-; SM80-DAG:   cvt.rn.bf16.f32 [[R0:%rs[0-9]+]], [[FR0]];
-; SM80-DAG:   cvt.rn.bf16.f32 [[R1:%rs[0-9]+]], [[FR1]];
-; SM80:       mov.b32         [[R:%r[0-9]+]], {[[R0]], [[R1]]};
-; CHECK:      st.param.b32    [func_retval0+0], [[R]];
-; CHECK:      ret;
-
 define <2 x bfloat> @test_faddx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
+; SM70-LABEL: test_faddx2(
+; SM70:       {
+; SM70-NEXT:    .reg .pred %p<3>;
+; SM70-NEXT:    .reg .b16 %rs<13>;
+; SM70-NEXT:    .reg .b32 %r<24>;
+; SM70-NEXT:    .reg .f32 %f<7>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.b32 %r1, [test_faddx2_param_0];
+; SM70-NEXT:    ld.param.b32 %r2, [test_faddx2_param_1];
+; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM70-NEXT:    shl.b32 %r4, %r3, 16;
+; SM70-NEXT:    mov.b32 %f1, %r4;
+; SM70-NEXT:    mov.b32 {%rs4, %rs5}, %r1;
+; SM70-NEXT:    cvt.u32.u16 %r5, %rs5;
+; SM70-NEXT:    shl.b32 %r6, %r5, 16;
+; SM70-NEXT:    mov.b32 %f2, %r6;
+; SM70-NEXT:    add.rn.f32 %f3, %f2, %f1;
+; SM70-NEXT:    mov.b32 %r7, %f3;
+; SM70-NEXT:    bfe.u32 %r8, %r7, 16, 1;
+; SM70-NEXT:    add.s32 %r9, %r8, %r7;
+; SM70-NEXT:    add.s32 %r10, %r9, 32767;
+; SM70-NEXT:    setp.nan.f32 %p1, %f3, %f3;
+; SM70-NEXT:    or.b32 %r11, %r7, 4194304;
+; SM70-NEXT:    selp.b32 %r12, %r11, %r10, %p1;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r12; }
+; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
+; SM70-NEXT:    shl.b32 %r14, %r13, 16;
+; SM70-NEXT:    mov.b32 %f4, %r14;
+; SM70-NEXT:    cvt.u32.u16 %r15, %rs4;
+; SM70-NEXT:    shl.b32 %r16, %r15, 16;
+; SM70-NEXT:    mov.b32 %f5, %r16;
+; SM70-NEXT:    add.rn.f32 %f6, %f5, %f4;
+; SM70-NEXT:    mov.b32 %r17, %f6;
+; SM70-NEXT:    bfe.u32 %r18, %r17, 16, 1;
+; SM70-NEXT:    add.s32 %r19, %r18, %r17;
+; SM70-NEXT:    add.s32 %r20, %r19, 32767;
+; SM70-NEXT:    setp.nan.f32 %p2, %f6, %f6;
+; SM70-NEXT:    or.b32 %r21, %r17, 4194304;
+; SM70-NEXT:    selp.b32 %r22, %r21, %r20, %p2;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs11}, %r22; }
+; SM70-NEXT:    mov.b32 %r23, {%rs11, %rs7};
+; SM70-NEXT:    st.param.b32 [func_retval0+0], %r23;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_faddx2(
+; SM80:       {
+; SM80-NEXT:    .reg .b16 %rs<7>;
+; SM80-NEXT:    .reg .b32 %r<4>;
+; SM80-NEXT:    .reg .f32 %f<7>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.b32 %r1, [test_faddx2_param_0];
+; SM80-NEXT:    ld.param.b32 %r2, [test_faddx2_param_1];
+; SM80-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
+; SM80-NEXT:    cvt.f32.bf16 %f1, %rs2;
+; SM80-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
+; SM80-NEXT:    cvt.f32.bf16 %f2, %rs4;
+; SM80-NEXT:    add.rn.f32 %f3, %f2, %f1;
+; SM80-NEXT:    cvt.rn.bf16.f32 %rs5, %f3;
+; SM80-NEXT:    cvt.f32.bf16 %f4, %rs1;
+; SM80-NEXT:    cvt.f32.bf16 %f5, %rs3;
+; SM80-NEXT:    add.rn.f32 %f6, %f5, %f4;
+; SM80-NEXT:    cvt.rn.bf16.f32 %rs6, %f6;
+; SM80-NEXT:    mov.b32 %r3, {%rs6, %rs5};
+; SM80-NEXT:    st.param.b32 [func_retval0+0], %r3;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_faddx2(
+; SM90:       {
+; SM90-NEXT:    .reg .b32 %r<4>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.b32 %r1, [test_faddx2_param_1];
+; SM90-NEXT:    ld.param.b32 %r2, [test_faddx2_param_0];
+; SM90-NEXT:    add.rn.bf16x2 %r3, %r2, %r1;
+; SM90-NEXT:    st.param.b32 [func_retval0+0], %r3;
+; SM90-NEXT:    ret;
   %r = fadd <2 x bfloat> %a, %b
   ret <2 x bfloat> %r
 }
 
-; CHECK-LABEL: test_fsubx2(
-; CHECK-DAG:  ld.param.b32    [[A:%r[0-9]+]], [test_fsubx2_param_0];
-; CHECK-DAG:  ld.param.b32    [[B:%r[0-9]+]], [test_fsubx2_param_1];
-; SM90:       sub.rn.bf16x2   [[R:%r[0-9]+]], [[A]], [[B]];
-
-; SM80-DAG:   mov.b32         {[[A0:%rs[0-9]+]], [[A1:%rs[0-9]+]]}, [[A]];
-; SM80-DAG:   mov.b32         {[[B0:%rs[0-9]+]], [[B1:%rs[0-9]+]]}, [[B]];
-; SM80-DAG:   cvt.f32.bf16    [[FA1:%f[0-9]+]], [[A1]];
-; SM80-DAG:   cvt.f32.bf16    [[FA0:%f[0-9]+]], [[A0]];
-; SM80-DAG:   cvt.f32.bf16    [[FB0:%f[0-9]+]], [[B0]];
-; SM80-DAG:   cvt.f32.bf16    [[FB1:%f[0-9]+]], [[B1]];
-; SM80-DAG:   sub.rn.f32      [[FR0:%f[0-9]+]], [[FA0]], [[FB0]];
-; SM80-DAG:   sub.rn.f32      [[FR1:%f[0-9]+]], [[FA1]], [[FB1]];
-; SM80-DAG:   cvt.rn.bf16.f32 [[R0:%rs[0-9]+]], [[FR0]];
-; SM80-DAG:   cvt.rn.bf16.f32 [[R1:%rs[0-9]+]], [[FR1]];
-; SM80:       mov.b32         [[R:%r[0-9]+]], {[[R0]], [[R1]]};
-
-; CHECK:      st.param.b32    [func_retval0+0], [[R]];
-; CHECK:      ret;
-
 define <2 x bfloat> @test_fsubx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
+; SM70-LABEL: test_fsubx2(
+; SM70:       {
+; SM70-NEXT:    .reg .pred %p<3>;
+; SM70-NEXT:    .reg .b16 %rs<13>;
+; SM70-NEXT:    .reg .b32 %r<24>;
+; SM70-NEXT:    .reg .f32 %f<7>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.b32 %r1, [test_fsubx2_param_0];
+; SM70-NEXT:    ld.param.b32 %r2, [test_fsubx2_param_1];
+; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM70-NEXT:    shl.b32 %r4, %r3, 16;
+; SM70-NEXT:    mov.b32 %f1, %r4;
+; SM70-NEXT:    mov.b32 {%rs4, %rs5}, %r1;
+; SM70-NEXT:    cvt.u32.u16 %r5, %rs5;
+; SM70-NEXT:    shl.b32 %r6, %r5, 16;
+; SM70-NEXT:    mov.b32 %f2, %r6;
+; SM70-NEXT:    sub.rn.f32 %f3, %f2, %f1;
+; SM70-NEXT:    mov.b32 %r7, %f3;
+; SM70-NEXT:    bfe.u32 %r8, %r7, 16, 1;
+; SM70-NEXT:    add.s32 %r9, %r8, %r7;
+; SM70-NEXT:    add.s32 %r10, %r9, 32767;
+; SM70-NEXT:    setp.nan.f32 %p1, %f3, %f3;
+; SM70-NEXT:    or.b32 %r11, %r7, 4194304;
+; SM70-NEXT:    selp.b32 %r12, %r11, %r10, %p1;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r12; }
+; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
+; SM70-NEXT:    shl.b32 %r14, %r13, 16;
+; SM70-NEXT:    mov.b32 %f4, %r14;
+; SM70-NEXT:    cvt.u32.u16 %r15, %rs4;
+; SM70-NEXT:    shl.b32 %r16, %r15, 16;
+; SM70-NEXT:    mov.b32 %f5, %r16;
+; SM70-NEXT:    sub.rn.f32 %f6, %f5, %f4;
+; SM70-NEXT:    mov.b32 %r17, %f6;
+; SM70-NEXT:    bfe.u32 %r18, %r17, 16, 1;
+; SM70-NEXT:    add.s32 %r19, %r18, %r17;
+; SM70-NEXT:    add.s32 %r20, %r19, 32767;
+; SM70-NEXT:    setp.nan.f32 %p2, %f6, %f6;
+; SM70-NEXT:    or.b32 %r21, %r17, 4194304;
+; SM70-NEXT:    selp.b32 %r22, %r21, %r20, %p2;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs11}, %r22; }
+; SM70-NEXT:    mov.b32 %r23, {%rs11, %rs7};
+; SM70-NEXT:    st.param.b32 [func_retval0+0], %r23;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_fsubx2(
+; SM80:       {
+; SM80-NEXT:    .reg .b16 %rs<7>;
+; SM80-NEXT:    .reg .b32 %r<4>;
+; SM80-NEXT:    .reg .f32 %f<7>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.b32 %r1, [test_fsubx2_param_0];
+; SM80-NEXT:    ld.param.b32 %r2, [test_fsubx2_param_1];
+; SM80-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
+; SM80-NEXT:    cvt.f32.bf16 %f1, %rs2;
+; SM80-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
+; SM80-NEXT:    cvt.f32.bf16 %f2, %rs4;
+; SM80-NEXT:    sub.rn.f32 %f3, %f2, %f1;
+; SM80-NEXT:    cvt.rn.bf16.f32 %rs5, %f3;
+; SM80-NEXT:    cvt.f32.bf16 %f4, %rs1;
+; SM80-NEXT:    cvt.f32.bf16 %f5, %rs3;
+; SM80-NEXT:    sub.rn.f32 %f6, %f5, %f4;
+; SM80-NEXT:    cvt.rn.bf16.f32 %rs6, %f6;
+; SM80-NEXT:    mov.b32 %r3, {%rs6, %rs5};
+; SM80-NEXT:    st.param.b32 [func_retval0+0], %r3;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_fsubx2(
+; SM90:       {
+; SM90-NEXT:    .reg .b32 %r<4>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.b32 %r1, [test_fsubx2_param_1];
+; SM90-NEXT:    ld.param.b32 %r2, [test_fsubx2_param_0];
+; SM90-NEXT:    sub.rn.bf16x2 %r3, %r2, %r1;
+; SM90-NEXT:    st.param.b32 [func_retval0+0], %r3;
+; SM90-NEXT:    ret;
   %r = fsub <2 x bfloat> %a, %b
   ret <2 x bfloat> %r
 }
 
-; CHECK-LABEL: test_fmulx2(
-; CHECK-DAG:  ld.param.b32    [[A:%r[0-9]+]], [test_fmulx2_param_0];
-; CHECK-DAG:  ld.param.b32    [[B:%r[0-9]+]], [test_fmulx2_param_1];
-; SM90:       mul.rn.bf16x2   [[R:%r[0-9]+]], [[A]], [[B]];
-
-; SM80-DAG:   mov.b32         {[[A0:%rs[0-9]+]], [[A1:%rs[0-9]+]]}, [[A]];
-; SM80-DAG:   mov.b32         {[[B0:%rs[0-9]+]], [[B1:%rs[0-9]+]]}, [[B]];
-; SM80-DAG:   cvt.f32.bf16    [[FA1:%f[0-9]+]], [[A1]];
-; SM80-DAG:   cvt.f32.bf16    [[FA0:%f[0-9]+]], [[A0]];
-; SM80-DAG:   cvt.f32.bf16    [[FB0:%f[0-9]+]], [[B0]];
-; SM80-DAG:   cvt.f32.bf16    [[FB1:%f[0-9]+]], [[B1]];
-; SM80-DAG:   mul.rn.f32      [[FR0:%f[0-9]+]], [[FA0]], [[FB0]];
-; SM80-DAG:   mul.rn.f32      [[FR1:%f[0-9]+]], [[FA1]], [[FB1]];
-; SM80-DAG:   cvt.rn.bf16.f32 [[R0:%rs[0-9]+]], [[FR0]];
-; SM80-DAG:   cvt.rn.bf16.f32 [[R1:%rs[0-9]+]], [[FR1]];
-; SM80:       mov.b32         [[R:%r[0-9]+]], {[[R0]], [[R1]]};
-
-; CHECK:      st.param.b32    [func_retval0+0], [[R]];
-; CHECK:      ret;
-
 define <2 x bfloat> @test_fmulx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
+; SM70-LABEL: test_fmulx2(
+; SM70:       {
+; SM70-NEXT:    .reg .pred %p<3>;
+; SM70-NEXT:    .reg .b16 %rs<13>;
+; SM70-NEXT:    .reg .b32 %r<24>;
+; SM70-NEXT:    .reg .f32 %f<7>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.b32 %r1, [test_fmulx2_param_0];
+; SM70-NEXT:    ld.param.b32 %r2, [test_fmulx2_param_1];
+; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM70-NEXT:    shl.b32 %r4, %r3, 16;
+; SM70-NEXT:    mov.b32 %f1, %r4;
+; SM70-NEXT:    mov.b32 {%rs4, %rs5}, %r1;
+; SM70-NEXT:    cvt.u32.u16 %r5, %rs5;
+; SM70-NEXT:    shl.b32 %r6, %r5, 16;
+; SM70-NEXT:    mov.b32 %f2, %r6;
+; SM70-NEXT:    mul.rn.f32 %f3, %f2, %f1;
+; SM70-NEXT:    mov.b32 %r7, %f3;
+; SM70-NEXT:    bfe.u32 %r8, %r7, 16, 1;
+; SM70-NEXT:    add.s32 %r9, %r8, %r7;
+; SM70-NEXT:    add.s32 %r10, %r9, 32767;
+; SM70-NEXT:    setp.nan.f32 %p1, %f3, %f3;
+; SM70-NEXT:    or.b32 %r11, %r7, 4194304;
+; SM70-NEXT:    selp.b32 %r12, %r11, %r10, %p1;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r12; }
+; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
+; SM70-NEXT:    shl.b32 %r14, %r13, 16;
+; SM70-NEXT:    mov.b32 %f4, %r14;
+; SM70-NEXT:    cvt.u32.u16 %r15, %rs4;
+; SM70-NEXT:    shl.b32 %r16, %r15, 16;
+; SM70-NEXT:    mov.b32 %f5, %r16;
+; SM70-NEXT:    mul.rn.f32 %f6, %f5, %f4;
+; SM70-NEXT:    mov.b32 %r17, %f6;
+; SM70-NEXT:    bfe.u32 %r18, %r17, 16, 1;
+; SM70-NEXT:    add.s32 %r19, %r18, %r17;
+; SM70-NEXT:    add.s32 %r20, %r19, 32767;
+; SM70-NEXT:    setp.nan.f32 %p2, %f6, %f6;
+; SM70-NEXT:    or.b32 %r21, %r17, 4194304;
+; SM70-NEXT:    selp.b32 %r22, %r21, %r20, %p2;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs11}, %r22; }
+; SM70-NEXT:    mov.b32 %r23, {%rs11, %rs7};
+; SM70-NEXT:    st.param.b32 [func_retval0+0], %r23;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_fmulx2(
+; SM80:       {
+; SM80-NEXT:    .reg .b16 %rs<7>;
+; SM80-NEXT:    .reg .b32 %r<4>;
+; SM80-NEXT:    .reg .f32 %f<7>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.b32 %r1, [test_fmulx2_param_0];
+; SM80-NEXT:    ld.param.b32 %r2, [test_fmulx2_param_1];
+; SM80-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
+; SM80-NEXT:    cvt.f32.bf16 %f1, %rs2;
+; SM80-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
+; SM80-NEXT:    cvt.f32.bf16 %f2, %rs4;
+; SM80-NEXT:    mul.rn.f32 %f3, %f2, %f1;
+; SM80-NEXT:    cvt.rn.bf16.f32 %rs5, %f3;
+; SM80-NEXT:    cvt.f32.bf16 %f4, %rs1;
+; SM80-NEXT:    cvt.f32.bf16 %f5, %rs3;
+; SM80-NEXT:    mul.rn.f32 %f6, %f5, %f4;
+; SM80-NEXT:    cvt.rn.bf16.f32 %rs6, %f6;
+; SM80-NEXT:    mov.b32 %r3, {%rs6, %rs5};
+; SM80-NEXT:    st.param.b32 [func_retval0+0], %r3;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_fmulx2(
+; SM90:       {
+; SM90-NEXT:    .reg .b32 %r<4>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.b32 %r1, [test_fmulx2_param_1];
+; SM90-NEXT:    ld.param.b32 %r2, [test_fmulx2_param_0];
+; SM90-NEXT:    mul.rn.bf16x2 %r3, %r2, %r1;
+; SM90-NEXT:    st.param.b32 [func_retval0+0], %r3;
+; SM90-NEXT:    ret;
   %r = fmul <2 x bfloat> %a, %b
   ret <2 x bfloat> %r
 }
 
-; CHECK-LABEL: test_fdiv(
-; CHECK-DAG:  ld.param.b32    [[A:%r[0-9]+]], [test_fdiv_param_0];
-; CHECK-DAG:  ld.param.b32    [[B:%r[0-9]+]], [test_fdiv_param_1];
-; CHECK-DAG:  mov.b32         {[[A0:%rs[0-9]+]], [[A1:%rs[0-9]+]]}, [[A]]
-; CHECK-DAG:  mov.b32         {[[B0:%rs[0-9]+]], [[B1:%rs[0-9]+]]}, [[B]]
-; CHECK-DAG:  cvt.f32.bf16     [[FA0:%f[0-9]+]], [[A0]];
-; CHECK-DAG:  cvt.f32.bf16     [[FA1:%f[0-9]+]], [[A1]];
-; CHECK-DAG:  cvt.f32.bf16     [[FB0:%f[0-9]+]], [[B0]];
-; CHECK-DAG:  cvt.f32.bf16     [[FB1:%f[0-9]+]], [[B1]];
-; CHECK-DAG:  div.rn.f32      [[FR0:%f[0-9]+]], [[FA0]], [[FB0]];
-; CHECK-DAG:  div.rn.f32      [[FR1:%f[0-9]+]], [[FA1]], [[FB1]];
-; CHECK-DAG:  cvt.rn.bf16.f32  [[R0:%rs[0-9]+]], [[FR0]];
-; CHECK-DAG:  cvt.rn.bf16.f32  [[R1:%rs[0-9]+]], [[FR1]];
-; CHECK-NEXT: mov.b32         [[R:%r[0-9]+]], {[[R0]], [[R1]]}
-; CHECK-NEXT: st.param.b32    [func_retval0+0], [[R]];
-; CHECK-NEXT: ret;
-
 define <2 x bfloat> @test_fdiv(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
+; SM70-LABEL: test_fdiv(
+; SM70:       {
+; SM70-NEXT:    .reg .pred %p<3>;
+; SM70-NEXT:    .reg .b16 %rs<13>;
+; SM70-NEXT:    .reg .b32 %r<24>;
+; SM70-NEXT:    .reg .f32 %f<7>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.b32 %r1, [test_fdiv_param_0];
+; SM70-NEXT:    ld.param.b32 %r2, [test_fdiv_param_1];
+; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM70-NEXT:    shl.b32 %r4, %r3, 16;
+; SM70-NEXT:    mov.b32 %f1, %r4;
+; SM70-NEXT:    mov.b32 {%rs4, %rs5}, %r1;
+; SM70-NEXT:    cvt.u32.u16 %r5, %rs5;
+; SM70-NEXT:    shl.b32 %r6, %r5, 16;
+; SM70-NEXT:    mov.b32 %f2, %r6;
+; SM70-NEXT:    div.rn.f32 %f3, %f2, %f1;
+; SM70-NEXT:    mov.b32 %r7, %f3;
+; SM70-NEXT:    bfe.u32 %r8, %r7, 16, 1;
+; SM70-NEXT:    add.s32 %r9, %r8, %r7;
+; SM70-NEXT:    add.s32 %r10, %r9, 32767;
+; SM70-NEXT:    setp.nan.f32 %p1, %f3, %f3;
+; SM70-NEXT:    or.b32 %r11, %r7, 4194304;
+; SM70-NEXT:    selp.b32 %r12, %r11, %r10, %p1;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r12; }
+; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
+; SM70-NEXT:    shl.b32 %r14, %r13, 16;
+; SM70-NEXT:    mov.b32 %f4, %r14;
+; SM70-NEXT:    cvt.u32.u16 %r15, %rs4;
+; SM70-NEXT:    shl.b32 %r16, %r15, 16;
+; SM70-NEXT:    mov.b32 %f5, %r16;
+; SM70-NEXT:    div.rn.f32 %f6, %f5, %f4;
+; SM70-NEXT:    mov.b32 %r17, %f6;
+; SM70-NEXT:    bfe.u32 %r18, %r17, 16, 1;
+; SM70-NEXT:    add.s32 %r19, %r18, %r17;
+; SM70-NEXT:    add.s32 %r20, %r19, 32767;
+; SM70-NEXT:    setp.nan.f32 %p2, %f6, %f6;
+; SM70-NEXT:    or.b32 %r21, %r17, 4194304;
+; SM70-NEXT:    selp.b32 %r22, %r21, %r20, %p2;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs11}, %r22; }
+; SM70-NEXT:    mov.b32 %r23, {%rs11, %rs7};
+; SM70-NEXT:    st.param.b32 [func_retval0+0], %r23;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_fdiv(
+; SM80:       {
+; SM80-NEXT:    .reg .b16 %rs<7>;
+; SM80-NEXT:    .reg .b32 %r<4>;
+; SM80-NEXT:    .reg .f32 %f<7>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.b32 %r1, [test_fdiv_param_0];
+; SM80-NEXT:    ld.param.b32 %r2, [test_fdiv_param_1];
+; SM80-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
+; SM80-NEXT:    cvt.f32.bf16 %f1, %rs2;
+; SM80-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
+; SM80-NEXT:    cvt.f32.bf16 %f2, %rs4;
+; SM80-NEXT:    div.rn.f32 %f3, %f2, %f1;
+; SM80-NEXT:    cvt.rn.bf16.f32 %rs5, %f3;
+; SM80-NEXT:    cvt.f32.bf16 %f4, %rs1;
+; SM80-NEXT:    cvt.f32.bf16 %f5, %rs3;
+; SM80-NEXT:    div.rn.f32 %f6, %f5, %f4;
+; SM80-NEXT:    cvt.rn.bf16.f32 %rs6, %f6;
+; SM80-NEXT:    mov.b32 %r3, {%rs6, %rs5};
+; SM80-NEXT:    st.param.b32 [func_retval0+0], %r3;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_fdiv(
+; SM90:       {
+; SM90-NEXT:    .reg .b16 %rs<7>;
+; SM90-NEXT:    .reg .b32 %r<4>;
+; SM90-NEXT:    .reg .f32 %f<7>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.b32 %r1, [test_fdiv_param_0];
+; SM90-NEXT:    ld.param.b32 %r2, [test_fdiv_param_1];
+; SM90-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
+; SM90-NEXT:    cvt.f32.bf16 %f1, %rs2;
+; SM90-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
+; SM90-NEXT:    cvt.f32.bf16 %f2, %rs4;
+; SM90-NEXT:    div.rn.f32 %f3, %f2, %f1;
+; SM90-NEXT:    cvt.rn.bf16.f32 %rs5, %f3;
+; SM90-NEXT:    cvt.f32.bf16 %f4, %rs1;
+; SM90-NEXT:    cvt.f32.bf16 %f5, %rs3;
+; SM90-NEXT:    div.rn.f32 %f6, %f5, %f4;
+; SM90-NEXT:    cvt.rn.bf16.f32 %rs6, %f6;
+; SM90-NEXT:    mov.b32 %r3, {%rs6, %rs5};
+; SM90-NEXT:    st.param.b32 [func_retval0+0], %r3;
+; SM90-NEXT:    ret;
   %r = fdiv <2 x bfloat> %a, %b
   ret <2 x bfloat> %r
 }
 
-; CHECK-LABEL: test_extract_0(
-; CHECK:      ld.param.b16    [[A:%rs[0-9]+]], [test_extract_0_param_0];
-; CHECK:      st.param.b16    [func_retval0+0], [[A]];
-; CHECK:      ret;
-
 define bfloat @test_extract_0(<2 x bfloat> %a) #0 {
+; CHECK-LABEL: test_extract_0(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b16 %rs1, [test_extract_0_param_0];
+; CHECK-NEXT:    st.param.b16 [func_retval0+0], %rs1;
+; CHECK-NEXT:    ret;
   %e = extractelement <2 x bfloat> %a, i32 0
   ret bfloat %e
 }
 
-; CHECK-LABEL: test_extract_1(
-; CHECK:      ld.param.b16    [[A:%rs[0-9]+]], [test_extract_1_param_0+2];
-; CHECK:      st.param.b16    [func_retval0+0], [[A]];
-; CHECK:      ret;
-
 define bfloat @test_extract_1(<2 x bfloat> %a) #0 {
+; CHECK-LABEL: test_extract_1(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b16 %rs1, [test_extract_1_param_0+2];
+; CHECK-NEXT:    st.param.b16 [func_retval0+0], %rs1;
+; CHECK-NEXT:    ret;
   %e = extractelement <2 x bfloat> %a, i32 1
   ret bfloat %e
 }
 
-; CHECK-LABEL: test_fpext_float(
-; CHECK:      ld.param.b16    [[A:%rs[0-9]+]], [test_fpext_float_param_0];
-; CHECK:      cvt.f32.bf16     [[R:%f[0-9]+]], [[A]];
-; CHECK:      st.param.f32    [func_retval0+0], [[R]];
-; CHECK:      ret;
 define float @test_fpext_float(bfloat %a) #0 {
+; SM70-LABEL: test_fpext_float(
+; SM70:       {
+; SM70-NEXT:    .reg .b32 %r<3>;
+; SM70-NEXT:    .reg .f32 %f<2>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.u16 %r1, [test_fpext_float_param_0];
+; SM70-NEXT:    shl.b32 %r2, %r1, 16;
+; SM70-NEXT:    mov.b32 %f1, %r2;
+; SM70-NEXT:    st.param.f32 [func_retval0+0], %f1;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_fpext_float(
+; SM80:       {
+; SM80-NEXT:    .reg .b16 %rs<2>;
+; SM80-NEXT:    .reg .f32 %f<2>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.b16 %rs1, [test_fpext_float_param_0];
+; SM80-NEXT:    cvt.f32.bf16 %f1, %rs1;
+; SM80-NEXT:    st.param.f32 [func_retval0+0], %f1;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_fpext_float(
+; SM90:       {
+; SM90-NEXT:    .reg .b16 %rs<2>;
+; SM90-NEXT:    .reg .f32 %f<2>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.b16 %rs1, [test_fpext_float_param_0];
+; SM90-NEXT:    cvt.f32.bf16 %f1, %rs1;
+; SM90-NEXT:    st.param.f32 [func_retval0+0], %f1;
+; SM90-NEXT:    ret;
   %r = fpext bfloat %a to float
   ret float %r
 }
 
-; CHECK-LABEL: test_fptrunc_float(
-; CHECK:      ld.param.f32    [[A:%f[0-9]+]], [test_fptrunc_float_param_0];
-; CHECK:      cvt.rn.bf16.f32  [[R:%rs[0-9]+]], [[A]];
-; CHECK:      st.param.b16    [func_retval0+0], [[R]];
-; CHECK:      ret;
 define bfloat @test_fptrunc_float(float %a) #0 {
+; SM70-LABEL: test_fptrunc_float(
+; SM70:       {
+; SM70-NEXT:    .reg .pred %p<2>;
+; SM70-NEXT:    .reg .b16 %rs<3>;
+; SM70-NEXT:    .reg .b32 %r<7>;
+; SM70-NEXT:    .reg .f32 %f<2>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.f32 %f1, [test_fptrunc_float_param_0];
+; SM70-NEXT:    mov.b32 %r1, %f1;
+; SM70-NEXT:    bfe.u32 %r2, %r1, 16, 1;
+; SM70-NEXT:    add.s32 %r3, %r2, %r1;
+; SM70-NEXT:    add.s32 %r4, %r3, 32767;
+; SM70-NEXT:    setp.nan.f32 %p1, %f1, %f1;
+; SM70-NEXT:    or.b32 %r5, %r1, 4194304;
+; SM70-NEXT:    selp.b32 %r6, %r5, %r4, %p1;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r6; }
+; SM70-NEXT:    st.param.b16 [func_retval0+0], %rs1;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_fptrunc_float(
+; SM80:       {
+; SM80-NEXT:    .reg .b16 %rs<2>;
+; SM80-NEXT:    .reg .f32 %f<2>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.f32 %f1, [test_fptrunc_float_param_0];
+; SM80-NEXT:    cvt.rn.bf16.f32 %rs1, %f1;
+; SM80-NEXT:    st.param.b16 [func_retval0+0], %rs1;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_fptrunc_float(
+; SM90:       {
+; SM90-NEXT:    .reg .b16 %rs<2>;
+; SM90-NEXT:    .reg .f32 %f<2>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.f32 %f1, [test_fptrunc_float_param_0];
+; SM90-NEXT:    cvt.rn.bf16.f32 %rs1, %f1;
+; SM90-NEXT:    st.param.b16 [func_retval0+0], %rs1;
+; SM90-NEXT:    ret;
   %r = fptrunc float %a to bfloat
   ret bfloat %r
 }
 
-; CHECK-LABEL: test_fadd_imm_1(
-; CHECK:      ld.param.b16    [[A:%rs[0-9]+]], [test_fadd_imm_1_param_0];
-; SM90:       mov.b16         [[B:%rs[0-9]+]], 0x3F80;
-; SM90:       add.rn.bf16     [[R:%rs[0-9]+]], [[A]], [[B]];
-
-; SM80-DAG:   cvt.f32.bf16    [[FA:%f[0-9]+]], [[A]];
-; SM80:       add.rn.f32      [[FR:%f[0-9]+]], [[FA]], 0f3F800000;
-; SM80:       cvt.rn.bf16.f32 [[R:%rs[0-9]+]], [[FR]];
-
-; CHECK:      st.param.b16    [func_retval0+0], [[R]];
-; CHECK-NEXT: ret;
 define bfloat @test_fadd_imm_1(bfloat %a) #0 {
+; SM70-LABEL: test_fadd_imm_1(
+; SM70:       {
+; SM70-NEXT:    .reg .pred %p<2>;
+; SM70-NEXT:    .reg .b16 %rs<3>;
+; SM70-NEXT:    .reg .b32 %r<9>;
+; SM70-NEXT:    .reg .f32 %f<3>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.u16 %r1, [test_fadd_imm_1_param_0];
+; SM70-NEXT:    shl.b32 %r2, %r1, 16;
+; SM70-NEXT:    mov.b32 %f1, %r2;
+; SM70-NEXT:    add.rn.f32 %f2, %f1, 0f3F800000;
+; SM70-NEXT:    mov.b32 %r3, %f2;
+; SM70-NEXT:    bfe.u32 %r4, %r3, 16, 1;
+; SM70-NEXT:    add.s32 %r5, %r4, %r3;
+; SM70-NEXT:    add.s32 %r6, %r5, 32767;
+; SM70-NEXT:    setp.nan.f32 %p1, %f2, %f2;
+; SM70-NEXT:    or.b32 %r7, %r3, 4194304;
+; SM70-NEXT:    selp.b32 %r8, %r7, %r6, %p1;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r8; }
+; SM70-NEXT:    st.param.b16 [func_retval0+0], %rs1;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_fadd_imm_1(
+; SM80:       {
+; SM80-NEXT:    .reg .b16 %rs<3>;
+; SM80-NEXT:    .reg .f32 %f<3>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.b16 %rs1, [test_fadd_imm_1_param_0];
+; SM80-NEXT:    cvt.f32.bf16 %f1, %rs1;
+; SM80-NEXT:    add.rn.f32 %f2, %f1, 0f3F800000;
+; SM80-NEXT:    cvt.rn.bf16.f32 %rs2, %f2;
+; SM80-NEXT:    st.param.b16 [func_retval0+0], %rs2;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_fadd_imm_1(
+; SM90:       {
+; SM90-NEXT:    .reg .b16 %rs<4>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.b16 %rs1, [test_fadd_imm_1_param_0];
+; SM90-NEXT:    mov.b16 %rs2, 0x3F80;
+; SM90-NEXT:    add.rn.bf16 %rs3, %rs1, %rs2;
+; SM90-NEXT:    st.param.b16 [func_retval0+0], %rs3;
+; SM90-NEXT:    ret;
   %r = fadd bfloat %a, 1.0
   ret bfloat %r
 }
 
-; CHECK-LABEL: test_select_cc_bf16_f64(
-; CHECK-DAG:      ld.param.f64    [[A:%fd[0-9]+]], [test_select_cc_bf16_f64_param_0];
-; CHECK-DAG:      ld.param.f64    [[B:%fd[0-9]+]], [test_select_cc_bf16_f64_param_1];
-; CHECK:          setp.lt.f64     [[P:%p[0-9]+]], [[A]], [[B]];
-; CHECK-DAG:      ld.param.b16    [[C:%rs[0-9]+]], [test_select_cc_bf16_f64_param_2];
-; CHECK-DAG:      ld.param.b16    [[D:%rs[0-9]+]], [test_select_cc_bf16_f64_param_3];
-; CHECK:          selp.b16        [[R:%rs[0-9]+]], [[C]], [[D]], [[P]];
-; CHECK-NEXT:     st.param.b16    [func_retval0+0], [[R]];
-; CHECK-NEXT:     ret;
 define bfloat @test_select_cc_bf16_f64(double %a, double %b, bfloat %c, bfloat %d) #0 {
+; CHECK-LABEL: test_select_cc_bf16_f64(
+; CHECK:       {
+; CHECK-NEXT:    .reg .pred %p<2>;
+; CHECK-NEXT:    .reg .b16 %rs<4>;
+; CHECK-NEXT:    .reg .f64 %fd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.f64 %fd1, [test_select_cc_bf16_f64_param_0];
+; CHECK-NEXT:    ld.param.f64 %fd2, [test_select_cc_bf16_f64_param_1];
+; CHECK-NEXT:    setp.lt.f64 %p1, %fd1, %fd2;
+; CHECK-NEXT:    ld.param.b16 %rs1, [test_select_cc_bf16_f64_param_2];
+; CHECK-NEXT:    ld.param.b16 %rs2, [test_select_cc_bf16_f64_param_3];
+; CHECK-NEXT:    selp.b16 %rs3, %rs1, %rs2, %p1;
+; CHECK-NEXT:    st.param.b16 [func_retval0+0], %rs3;
+; CHECK-NEXT:    ret;
   %cc = fcmp olt double %a, %b
   %r = select i1 %cc, bfloat %c, bfloat %d
   ret bfloat %r
 }
 
-; CHECK-LABEL: test_extload_bf16x8
-; CHECK: ld.shared.v4.b32 {%r
-; CHECK: mov.b32 {%rs
-; CHECK: mov.b32 {%rs
-; CHECK: mov.b32 {%rs
-; CHECK: mov.b32 {%rs
-; SM80: cvt.f32.bf16 %f{{.*}}, %rs
-; SM80: cvt.f32.bf16 %f{{.*}}, %rs
-; SM80: cvt.f32.bf16 %f{{.*}}, %rs
-; SM80: cvt.f32.bf16 %f{{.*}}, %rs
-; SM80: cvt.f32.bf16 %f{{.*}}, %rs
-; SM80: cvt.f32.bf16 %f{{.*}}, %rs
-; SM80: cvt.f32.bf16 %f{{.*}}, %rs
-; SM80: cvt.f32.bf16 %f{{.*}}, %rs
 define <8 x float> @test_extload_bf16x8(ptr addrspace(3) noundef %arg) #0 {
+; SM70-LABEL: test_extload_bf16x8(
+; SM70:       {
+; SM70-NEXT:    .reg .b16 %rs<17>;
+; SM70-NEXT:    .reg .b32 %r<21>;
+; SM70-NEXT:    .reg .f32 %f<9>;
+; SM70-NEXT:    .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.u64 %rd1, [test_extload_bf16x8_param_0];
+; SM70-NEXT:    ld.shared.v4.b32 {%r1, %r2, %r3, %r4}, [%rd1];
+; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT:    mov.b32 {%rs3, %rs4}, %r2;
+; SM70-NEXT:    mov.b32 {%rs5, %rs6}, %r3;
+; SM70-NEXT:    mov.b32 {%rs7, %rs8}, %r4;
+; SM70-NEXT:    cvt.u32.u16 %r5, %rs8;
+; SM70-NEXT:    shl.b32 %r6, %r5, 16;
+; SM70-NEXT:    mov.b32 %f1, %r6;
+; SM70-NEXT:    cvt.u32.u16 %r7, %rs7;
+; SM70-NEXT:    shl.b32 %r8, %r7, 16;
+; SM70-NEXT:    mov.b32 %f2, %r8;
+; SM70-NEXT:    cvt.u32.u16 %r9, %rs6;
+; SM70-NEXT:    shl.b32 %r10, %r9, 16;
+; SM70-NEXT:    mov.b32 %f3, %r10;
+; SM70-NEXT:    cvt.u32.u16 %r11, %rs5;
+; SM70-NEXT:    shl.b32 %r12, %r11, 16;
+; SM70-NEXT:    mov.b32 %f4, %r12;
+; SM70-NEXT:    cvt.u32.u16 %r13, %rs4;
+; SM70-NEXT:    shl.b32 %r14, %r13, 16;
+; SM70-NEXT:    mov.b32 %f5, %r14;
+; SM70-NEXT:    cvt.u32.u16 %r15, %rs3;
+; SM70-NEXT:    shl.b32 %r16, %r15, 16;
+; SM70-NEXT:    mov.b32 %f6, %r16;
+; SM70-NEXT:    cvt.u32.u16 %r17, %rs2;
+; SM70-NEXT:    shl.b32 %r18, %r17, 16;
+; SM70-NEXT:    mov.b32 %f7, %r18;
+; SM70-NEXT:    cvt.u32.u16 %r19, %rs1;
+; SM70-NEXT:    shl.b32 %r20, %r19, 16;
+; SM70-NEXT:    mov.b32 %f8, %r20;
+; SM70-NEXT:    st.param.v4.f32 [func_retval0+0], {%f8, %f7, %f6, %f5};
+; SM70-NEXT:    st.param.v4.f32 [func_retval0+16], {%f4, %f3, %f2, %f1};
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_extload_bf16x8(
+; SM80:       {
+; SM80-NEXT:    .reg .b16 %rs<9>;
+; SM80-NEXT:    .reg .b32 %r<5>;
+; SM80-NEXT:    .reg .f32 %f<9>;
+; SM80-NEXT:    .reg .b64 %rd<2>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.u64 %rd1, [test_extload_bf16x8_param_0];
+; SM80-NEXT:    ld.shared.v4.b32 {%r1, %r2, %r3, %r4}, [%rd1];
+; SM80-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
+; SM80-NEXT:    mov.b32 {%rs3, %rs4}, %r2;
+; SM80-NEXT:    mov.b32 {%rs5, %rs6}, %r3;
+; SM80-NEXT:    mov.b32 {%rs7, %rs8}, %r4;
+; SM80-NEXT:    cvt.f32.bf16 %f1, %rs8;
+; SM80-NEXT:    cvt.f32.bf16 %f2, %rs7;
+; SM80-NEXT:    cvt.f32.bf16 %f3, %rs6;
+; SM80-NEXT:    cvt.f32.bf16 %f4, %rs5;
+; SM80-NEXT:    cvt.f32.bf16 %f5, %rs4;
+; SM80-NEXT:    cvt.f32.bf16 %f6, %rs3;
+; SM80-NEXT:    cvt.f32.bf16 %f7, %rs2;
+; SM80-NEXT:    cvt.f32.bf16 %f8, %rs1;
+; SM80-NEXT:    st.param.v4.f32 [func_retval0+0], {%f8, %f7, %f6, %f5};
+; SM80-NEXT:    st.param.v4.f32 [func_retval0+16], {%f4, %f3, %f2, %f1};
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_extload_bf16x8(
+; SM90:       {
+; SM90-NEXT:    .reg .b16 %rs<9>;
+; SM90-NEXT:    .reg .b32 %r<5>;
+; SM90-NEXT:    .reg .f32 %f<9>;
+; SM90-NEXT:    .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.u64 %rd1, [test_extload_bf16x8_param_0];
+; SM90-NEXT:    ld.shared.v4.b32 {%r1, %r2, %r3, %r4}, [%rd1];
+; SM90-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
+; SM90-NEXT:    mov.b32 {%rs3, %rs4}, %r2;
+; SM90-NEXT:    mov.b32 {%rs5, %rs6}, %r3;
+; SM90-NEXT:    mov.b32 {%rs7, %rs8}, %r4;
+; SM90-NEXT:    cvt.f32.bf16 %f1, %rs8;
+; SM90-NEXT:    cvt.f32.bf16 %f2, %rs7;
+; SM90-NEXT:    cvt.f32.bf16 %f3, %rs6;
+; SM90-NEXT:    cvt.f32.bf16 %f4, %rs5;
+; SM90-NEXT:    cvt.f32.bf16 %f5, %rs4;
+; SM90-NEXT:    cvt.f32.bf16 %f6, %rs3;
+; SM90-NEXT:    cvt.f32.bf16 %f7, %rs2;
+; SM90-NEXT:    cvt.f32.bf16 %f8, %rs1;
+; SM90-NEXT:    st.param.v4.f32 [func_retval0+0], {%f8, %f7, %f6, %f5};
+; SM90-NEXT:    st.param.v4.f32 [func_retval0+16], {%f4, %f3, %f2, %f1};
+; SM90-NEXT:    ret;
   %load = load <8 x bfloat>, ptr addrspace(3) %arg, align 16
   %res = fpext <8 x bfloat> %load to <8 x float>
   ret <8 x float> %res
 }
 
-; CHECK-LABEL: test_fptosi_i16(
-; CHECK:      ld.param.b16     [[A:%rs[0-9]+]], [test_fptosi_i16_param_0];
-; SM80:       cvt.f32.bf16     [[B:%f[0-9]+]], [[A]];
-; SM80:       cvt.rzi.s16.f32  [[C:%rs[0-9]+]], [[B]];
-; SM80:       cvt.u32.u16      [[R:%r[0-9]+]], [[C]];
-; SM90:       cvt.rzi.s16.bf16 [[B:%rs[0-9]+]], [[A]];
-; SM90:       cvt.u32.u16      [[R:%r[0-9]+]], [[B]];
-; CHECK:      st.param.b32     [func_retval0+0], [[R]];
-; CHECK:      ret;
 define i16 @test_fptosi_i16(bfloat %a) {
+; SM70-LABEL: test_fptosi_i16(
+; SM70:       {
+; SM70-NEXT:    .reg .b16 %rs<2>;
+; SM70-NEXT:    .reg .b32 %r<4>;
+; SM70-NEXT:    .reg .f32 %f<2>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.u16 %r1, [test_fptosi_i16_param_0];
+; SM70-NEXT:    shl.b32 %r2, %r1, 16;
+; SM70-NEXT:    mov.b32 %f1, %r2;
+; SM70-NEXT:    cvt.rzi.s16.f32 %rs1, %f1;
+; SM70-NEXT:    cvt.u32.u16 %r3, %rs1;
+; SM70-NEXT:    st.param.b32 [func_retval0+0], %r3;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_fptosi_i16(
+; SM80:       {
+; SM80-NEXT:    .reg .b16 %rs<3>;
+; SM80-NEXT:    .reg .b32 %r<2>;
+; SM80-NEXT:    .reg .f32 %f<2>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.b16 %rs1, [test_fptosi_i16_param_0];
+; SM80-NEXT:    cvt.f32.bf16 %f1, %rs1;
+; SM80-NEXT:    cvt.rzi.s16.f32 %rs2, %f1;
+; SM80-NEXT:    cvt.u32.u16 %r1, %rs2;
+; SM80-NEXT:    st.param.b32 [func_retval0+0], %r1;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_fptosi_i16(
+; SM90:       {
+; SM90-NEXT:    .reg .b16 %rs<3>;
+; SM90-NEXT:    .reg .b32 %r<2>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.b16 %rs1, [test_fptosi_i16_param_0];
+; SM90-NEXT:    cvt.rzi.s16.bf16 %rs2, %rs1;
+; SM90-NEXT:    cvt.u32.u16 %r1, %rs2;
+; SM90-NEXT:    st.param.b32 [func_retval0+0], %r1;
+; SM90-NEXT:    ret;
   %r = fptosi bfloat %a to i16
   ret i16 %r
 }
 
-; CHECK-LABEL: test_fptoui_i16(
-; CHECK:      ld.param.b16     [[A:%rs[0-9]+]], [test_fptoui_i16_param_0];
-; SM80:       cvt.f32.bf16     [[B:%f[0-9]+]], [[A]];
-; SM80:       cvt.rzi.u16.f32  [[C:%rs[0-9]+]], [[B]];
-; SM80:       cvt.u32.u16      [[R:%r[0-9]+]], [[C]];
-; SM90:       cvt.rzi.u16.bf16 [[B:%rs[0-9]+]], [[A]];
-; SM90:       cvt.u32.u16      [[R:%r[0-9]+]], [[B]];
-; CHECK:      st.param.b32     [func_retval0+0], [[R]];
-; CHECK:      ret;
 define i16 @test_fptoui_i16(bfloat %a) {
+; SM70-LABEL: test_fptoui_i16(
+; SM70:       {
+; SM70-NEXT:    .reg .b16 %rs<2>;
+; SM70-NEXT:    .reg .b32 %r<4>;
+; SM70-NEXT:    .reg .f32 %f<2>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.u16 %r1, [test_fptoui_i16_param_0];
+; SM70-NEXT:    shl.b32 %r2, %r1, 16;
+; SM70-NEXT:    mov.b32 %f1, %r2;
+; SM70-NEXT:    cvt.rzi.u16.f32 %rs1, %f1;
+; SM70-NEXT:    cvt.u32.u16 %r3, %rs1;
+; SM70-NEXT:    st.param.b32 [func_retval0+0], %r3;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_fptoui_i16(
+; SM80:       {
+; SM80-NEXT:    .reg .b16 %rs<3>;
+; SM80-NEXT:    .reg .b32 %r<2>;
+; SM80-NEXT:    .reg .f32 %f<2>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.b16 %rs1, [test_fptoui_i16_param_0];
+; SM80-NEXT:    cvt.f32.bf16 %f1, %rs1;
+; SM80-NEXT:    cvt.rzi.u16.f32 %rs2, %f1;
+; SM80-NEXT:    cvt.u32.u16 %r1, %rs2;
+; SM80-NEXT:    st.param.b32 [func_retval0+0], %r1;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_fptoui_i16(
+; SM90:       {
+; SM90-NEXT:    .reg .b16 %rs<3>;
+; SM90-NEXT:    .reg .b32 %r<2>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.b16 %rs1, [test_fptoui_i16_param_0];
+; SM90-NEXT:    cvt.rzi.u16.bf16 %rs2, %rs1;
+; SM90-NEXT:    cvt.u32.u16 %r1, %rs2;
+; SM90-NEXT:    st.param.b32 [func_retval0+0], %r1;
+; SM90-NEXT:    ret;
   %r = fptoui bfloat %a to i16
   ret i16 %r
 }
 
-; CHECK-LABEL: test_sitofp_i16(
-; CHECK:      ld.param.u16    [[A:%rs[0-9]+]], [test_sitofp_i16_param_0];
-; SM80:       cvt.rn.f32.s16  [[B:%f[0-9]+]], [[A]];
-; SM80:       cvt.rn.bf16.f32 [[R:%rs[0-9]+]], [[B]];
-; SM90:       cvt.rn.bf16.s16 [[R:%rs[0-9]+]], [[A]];
-; CHECK:      st.param.b16    [func_retval0+0], [[R]];
-; CHECK:      ret;
 define bfloat @test_sitofp_i16(i16 %a) {
+; SM70-LABEL: test_sitofp_i16(
+; SM70:       {
+; SM70-NEXT:    .reg .pred %p<2>;
+; SM70-NEXT:    .reg .b16 %rs<4>;
+; SM70-NEXT:    .reg .b32 %r<7>;
+; SM70-NEXT:    .reg .f32 %f<2>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.u16 %rs1, [test_sitofp_i16_param_0];
+; SM70-NEXT:    cvt.rn.f32.s16 %f1, %rs1;
+; SM70-NEXT:    mov.b32 %r1, %f1;
+; SM70-NEXT:    bfe.u32 %r2, %r1, 16, 1;
+; SM70-NEXT:    add.s32 %r3, %r2, %r1;
+; SM70-NEXT:    add.s32 %r4, %r3, 32767;
+; SM70-NEXT:    setp.nan.f32 %p1, %f1, %f1;
+; SM70-NEXT:    or.b32 %r5, %r1, 4194304;
+; SM70-NEXT:    selp.b32 %r6, %r5, %r4, %p1;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs2}, %r6; }
+; SM70-NEXT:    st.param.b16 [func_retval0+0], %rs2;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_sitofp_i16(
+; SM80:       {
+; SM80-NEXT:    .reg .b16 %rs<3>;
+; SM80-NEXT:    .reg .f32 %f<2>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.u16 %rs1, [test_sitofp_i16_param_0];
+; SM80-NEXT:    cvt.rn.f32.s16 %f1, %rs1;
+; SM80-NEXT:    cvt.rn.bf16.f32 %rs2, %f1;
+; SM80-NEXT:    st.param.b16 [func_retval0+0], %rs2;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_sitofp_i16(
+; SM90:       {
+; SM90-NEXT:    .reg .b16 %rs<3>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.u16 %rs1, [test_sitofp_i16_param_0];
+; SM90-NEXT:    cvt.rn.bf16.s16 %rs2, %rs1;
+; SM90-NEXT:    st.param.b16 [func_retval0+0], %rs2;
+; SM90-NEXT:    ret;
   %r = sitofp i16 %a to bfloat
   ret bfloat %r
 }
 
-; CHECK-LABEL: test_uitofp_i8(
-; CHECK:      ld.param.u8 %rs1, [test_uitofp_i8_param_0];
-; SM80:       cvt.rn.f32.u16  [[B:%f[0-9]+]], [[A]];
-; SM80:       cvt.rn.bf16.f32 [[R:%rs[0-9]+]], [[B]];
-; SM90:       cvt.rn.bf16.u16 [[R:%rs[0-9]+]], [[A]];
-; CHECK:      st.param.b16    [func_retval0+0], [[R]];
-; CHECK:      ret;
 define bfloat @test_uitofp_i8(i8 %a) {
+; SM70-LABEL: test_uitofp_i8(
+; SM70:       {
+; SM70-NEXT:    .reg .pred %p<2>;
+; SM70-NEXT:    .reg .b16 %rs<4>;
+; SM70-NEXT:    .reg .b32 %r<7>;
+; SM70-NEXT:    .reg .f32 %f<2>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.u8 %rs1, [test_uitofp_i8_param_0];
+; SM70-NEXT:    cvt.rn.f32.u16 %f1, %rs1;
+; SM70-NEXT:    mov.b32 %r1, %f1;
+; SM70-NEXT:    bfe.u32 %r2, %r1, 16, 1;
+; SM70-NEXT:    add.s32 %r3, %r2, %r1;
+; SM70-NEXT:    add.s32 %r4, %r3, 32767;
+; SM70-NEXT:    setp.nan.f32 %p1, %f1, %f1;
+; SM70-NEXT:    or.b32 %r5, %r1, 4194304;
+; SM70-NEXT:    selp.b32 %r6, %r5, %r4, %p1;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs2}, %r6; }
+; SM70-NEXT:    st.param.b16 [func_retval0+0], %rs2;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_uitofp_i8(
+; SM80:       {
+; SM80-NEXT:    .reg .b16 %rs<3>;
+; SM80-NEXT:    .reg .f32 %f<2>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.u8 %rs1, [test_uitofp_i8_param_0];
+; SM80-NEXT:    cvt.rn.f32.u16 %f1, %rs1;
+; SM80-NEXT:    cvt.rn.bf16.f32 %rs2, %f1;
+; SM80-NEXT:    st.param.b16 [func_retval0+0], %rs2;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_uitofp_i8(
+; SM90:       {
+; SM90-NEXT:    .reg .b16 %rs<3>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.u8 %rs1, [test_uitofp_i8_param_0];
+; SM90-NEXT:    cvt.rn.bf16.u16 %rs2, %rs1;
+; SM90-NEXT:    st.param.b16 [func_retval0+0], %rs2;
+; SM90-NEXT:    ret;
   %r = uitofp i8 %a to bfloat
   ret bfloat %r
 }
 
-; CHECK-LABEL: test_uitofp_i1(
-; CHECK:      ld.param.u8     [[A:%rs[0-9]+]], [test_uitofp_i1_param_0];
-; CHECK:      and.b16         [[B:%rs[0-9]+]], [[A]], 1;
-; CHECK:      setp.eq.b16     [[C:%p[0-9]+]], [[B]], 1;
-; CHECK:      selp.u32        [[D:%r[0-9]+]], 1, 0, [[C]];
-; SM80:       cvt.rn.f32.u32  [[E:%f[0-9]+]], [[D]];
-; SM80:       cvt.rn.bf16.f32 [[R:%rs[0-9]+]], [[E]];
-; SM90:       cvt.rn.bf16.u32 [[R:%rs[0-9]+]], [[D]];
-; CHECK:      st.param.b16    [func_retval0+0], [[R]];
-; CHECK:      ret;
 define bfloat @test_uitofp_i1(i1 %a) {
+; SM70-LABEL: test_uitofp_i1(
+; SM70:       {
+; SM70-NEXT:    .reg .pred %p<3>;
+; SM70-NEXT:    .reg .b16 %rs<5>;
+; SM70-NEXT:    .reg .b32 %r<8>;
+; SM70-NEXT:    .reg .f32 %f<2>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.u8 %rs1, [test_uitofp_i1_param_0];
+; SM70-NEXT:    and.b16 %rs2, %rs1, 1;
+; SM70-NEXT:    setp.eq.b16 %p1, %rs2, 1;
+; SM70-NEXT:    selp.u32 %r1, 1, 0, %p1;
+; SM70-NEXT:    cvt.rn.f32.u32 %f1, %r1;
+; SM70-NEXT:    mov.b32 %r2, %f1;
+; SM70-NEXT:    bfe.u32 %r3, %r2, 16, 1;
+; SM70-NEXT:    add.s32 %r4, %r3, %r2;
+; SM70-NEXT:    add.s32 %r5, %r4, 32767;
+; SM70-NEXT:    setp.nan.f32 %p2, %f1, %f1;
+; SM70-NEXT:    or.b32 %r6, %r2, 4194304;
+; SM70-NEXT:    selp.b32 %r7, %r6, %r5, %p2;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs3}, %r7; }
+; SM70-NEXT:    st.param.b16 [func_retval0+0], %rs3;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_uitofp_i1(
+; SM80:       {
+; SM80-NEXT:    .reg .pred %p<2>;
+; SM80-NEXT:    .reg .b16 %rs<4>;
+; SM80-NEXT:    .reg .b32 %r<2>;
+; SM80-NEXT:    .reg .f32 %f<2>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.u8 %rs1, [test_uitofp_i1_param_0];
+; SM80-NEXT:    and.b16 %rs2, %rs1, 1;
+; SM80-NEXT:    setp.eq.b16 %p1, %rs2, 1;
+; SM80-NEXT:    selp.u32 %r1, 1, 0, %p1;
+; SM80-NEXT:    cvt.rn.f32.u32 %f1, %r1;
+; SM80-NEXT:    cvt.rn.bf16.f32 %rs3, %f1;
+; SM80-NEXT:    st.param.b16 [func_retval0+0], %rs3;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_uitofp_i1(
+; SM90:       {
+; SM90-NEXT:    .reg .pred %p<2>;
+; SM90-NEXT:    .reg .b16 %rs<4>;
+; SM90-NEXT:    .reg .b32 %r<2>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.u8 %rs1, [test_uitofp_i1_param_0];
+; SM90-NEXT:    and.b16 %rs2, %rs1, 1;
+; SM90-NEXT:    setp.eq.b16 %p1, %rs2, 1;
+; SM90-NEXT:    selp.u32 %r1, 1, 0, %p1;
+; SM90-NEXT:    cvt.rn.bf16.u32 %rs3, %r1;
+; SM90-NEXT:    st.param.b16 [func_retval0+0], %rs3;
+; SM90-NEXT:    ret;
   %r = uitofp i1 %a to bfloat
   ret bfloat %r
 }
 
-; CHECK-LABEL: test_uitofp_i16(
-; CHECK:      ld.param.u16    [[A:%rs[0-9]+]], [test_uitofp_i16_param_0];
-; SM80:       cvt.rn.f32.u16  [[B:%f[0-9]+]], [[A]];
-; SM80:       cvt.rn.bf16.f32 [[R:%rs[0-9]+]], [[B]];
-; SM90:       cvt.rn.bf16.u16 [[R:%rs[0-9]+]], [[A]];
-; CHECK:      st.param.b16    [func_retval0+0], [[R]];
-; CHECK:      ret;
 define bfloat @test_uitofp_i16(i16 %a) {
+; SM70-LABEL: test_uitofp_i16(
+; SM70:       {
+; SM70-NEXT:    .reg .pred %p<2>;
+; SM70-NEXT:    .reg .b16 %rs<4>;
+; SM70-NEXT:    .reg .b32 %r<7>;
+; SM70-NEXT:    .reg .f32 %f<2>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.u16 %rs1, [test_uitofp_i16_param_0];
+; SM70-NEXT:    cvt.rn.f32.u16 %f1, %rs1;
+; SM70-NEXT:    mov.b32 %r1, %f1;
+; SM70-NEXT:    bfe.u32 %r2, %r1, 16, 1;
+; SM70-NEXT:    add.s32 %r3, %r2, %r1;
+; SM70-NEXT:    add.s32 %r4, %r3, 32767;
+; SM70-NEXT:    setp.nan.f32 %p1, %f1, %f1;
+; SM70-NEXT:    or.b32 %r5, %r1, 4194304;
+; SM70-NEXT:    selp.b32 %r6, %r5, %r4, %p1;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs2}, %r6; }
+; SM70-NEXT:    st.param.b16 [func_retval0+0], %rs2;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_uitofp_i16(
+; SM80:       {
+; SM80-NEXT:    .reg .b16 %rs<3>;
+; SM80-NEXT:    .reg .f32 %f<2>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.u16 %rs1, [test_uitofp_i16_param_0];
+; SM80-NEXT:    cvt.rn.f32.u16 %f1, %rs1;
+; SM80-NEXT:    cvt.rn.bf16.f32 %rs2, %f1;
+; SM80-NEXT:    st.param.b16 [func_retval0+0], %rs2;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_uitofp_i16(
+; SM90:       {
+; SM90-NEXT:    .reg .b16 %rs<3>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.u16 %rs1, [test_uitofp_i16_param_0];
+; SM90-NEXT:    cvt.rn.bf16.u16 %rs2, %rs1;
+; SM90-NEXT:    st.param.b16 [func_retval0+0], %rs2;
+; SM90-NEXT:    ret;
   %r = uitofp i16 %a to bfloat
   ret bfloat %r
 }
 
-; CHECK-LABEL: test_uitofp_i32(
-; CHECK:      ld.param.u32    [[A:%r[0-9]+]], [test_uitofp_i32_param_0];
-; SM80:       cvt.rn.f32.u32  [[B:%f[0-9]+]], [[A]];
-; SM80:       cvt.rn.bf16.f32 [[R:%rs[0-9]+]], [[B]];
-; SM90:       cvt.rn.bf16.u32 [[R:%rs[0-9]+]], [[A]];
-; CHECK:      st.param.b16    [func_retval0+0], [[R]];
-; CHECK:      ret;
 define bfloat @test_uitofp_i32(i32 %a) {
+; SM70-LABEL: test_uitofp_i32(
+; SM70:       {
+; SM70-NEXT:    .reg .pred %p<2>;
+; SM70-NEXT:    .reg .b16 %rs<3>;
+; SM70-NEXT:    .reg .b32 %r<8>;
+; SM70-NEXT:    .reg .f32 %f<2>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.u32 %r1, [test_uitofp_i32_param_0];
+; SM70-NEXT:    cvt.rn.f32.u32 %f1, %r1;
+; SM70-NEXT:    mov.b32 %r2, %f1;
+; SM70-NEXT:    bfe.u32 %r3, %r2, 16, 1;
+; SM70-NEXT:    add.s32 %r4, %r3, %r2;
+; SM70-NEXT:    add.s32 %r5, %r4, 32767;
+; SM70-NEXT:    setp.nan.f32 %p1, %f1, %f1;
+; SM70-NEXT:    or.b32 %r6, %r2, 4194304;
+; SM70-NEXT:    selp.b32 %r7, %r6, %r5, %p1;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r7; }
+; SM70-NEXT:    st.param.b16 [func_retval0+0], %rs1;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_uitofp_i32(
+; SM80:       {
+; SM80-NEXT:    .reg .b16 %rs<2>;
+; SM80-NEXT:    .reg .b32 %r<2>;
+; SM80-NEXT:    .reg .f32 %f<2>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.u32 %r1, [test_uitofp_i32_param_0];
+; SM80-NEXT:    cvt.rn.f32.u32 %f1, %r1;
+; SM80-NEXT:    cvt.rn.bf16.f32 %rs1, %f1;
+; SM80-NEXT:    st.param.b16 [func_retval0+0], %rs1;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_uitofp_i32(
+; SM90:       {
+; SM90-NEXT:    .reg .b16 %rs<2>;
+; SM90-NEXT:    .reg .b32 %r<2>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.u32 %r1, [test_uitofp_i32_param_0];
+; SM90-NEXT:    cvt.rn.bf16.u32 %rs1, %r1;
+; SM90-NEXT:    st.param.b16 [func_retval0+0], %rs1;
+; SM90-NEXT:    ret;
   %r = uitofp i32 %a to bfloat
   ret bfloat %r
 }
 
-; CHECK-LABEL: test_uitofp_i64(
-; CHECK:      ld.param.u64    [[A:%rd[0-9]+]], [test_uitofp_i64_param_0];
-; SM80:       cvt.rn.f32.u64  [[B:%f[0-9]+]], [[A]];
-; SM80:       cvt.rn.bf16.f32 [[R:%rs[0-9]+]], [[B]];
-; SM90:       cvt.rn.bf16.u64 [[R:%rs[0-9]+]], [[A]];
-; CHECK:      st.param.b16    [func_retval0+0], [[R]];
-; CHECK:      ret;
 define bfloat @test_uitofp_i64(i64 %a) {
+; SM70-LABEL: test_uitofp_i64(
+; SM70:       {
+; SM70-NEXT:    .reg .pred %p<2>;
+; SM70-NEXT:    .reg .b16 %rs<3>;
+; SM70-NEXT:    .reg .b32 %r<7>;
+; SM70-NEXT:    .reg .f32 %f<2>;
+; SM70-NEXT:    .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.u64 %rd1, [test_uitofp_i64_param_0];
+; SM70-NEXT:    cvt.rn.f32.u64 %f1, %rd1;
+; SM70-NEXT:    mov.b32 %r1, %f1;
+; SM70-NEXT:    bfe.u32 %r2, %r1, 16, 1;
+; SM70-NEXT:    add.s32 %r3, %r2, %r1;
+; SM70-NEXT:    add.s32 %r4, %r3, 32767;
+; SM70-NEXT:    setp.nan.f32 %p1, %f1, %f1;
+; SM70-NEXT:    or.b32 %r5, %r1, 4194304;
+; SM70-NEXT:    selp.b32 %r6, %r5, %r4, %p1;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r6; }
+; SM70-NEXT:    st.param.b16 [func_retval0+0], %rs1;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_uitofp_i64(
+; SM80:       {
+; SM80-NEXT:    .reg .b16 %rs<2>;
+; SM80-NEXT:    .reg .f32 %f<2>;
+; SM80-NEXT:    .reg .b64 %rd<2>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.u64 %rd1, [test_uitofp_i64_param_0];
+; SM80-NEXT:    cvt.rn.f32.u64 %f1, %rd1;
+; SM80-NEXT:    cvt.rn.bf16.f32 %rs1, %f1;
+; SM80-NEXT:    st.param.b16 [func_retval0+0], %rs1;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_uitofp_i64(
+; SM90:       {
+; SM90-NEXT:    .reg .b16 %rs<2>;
+; SM90-NEXT:    .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.u64 %rd1, [test_uitofp_i64_param_0];
+; SM90-NEXT:    cvt.rn.bf16.u64 %rs1, %rd1;
+; SM90-NEXT:    st.param.b16 [func_retval0+0], %rs1;
+; SM90-NEXT:    ret;
   %r = uitofp i64 %a to bfloat
   ret bfloat %r
 }
 
-; CHECK-LABEL: test_roundeven(
-; CHECK:      ld.param.b16      [[A:%rs[0-9]+]], [test_roundeven_param_0];
-; SM80:       cvt.rni.f32.f32   [[F:%f[0-9]+]]
-; SM80:       cvt.rn.bf16.f32   [[R:%rs[0-9]+]], [[F]];
-; SM90:       cvt.rni.bf16.bf16 [[R:%rs[0-9]+]], [[A]];
-; CHECK:      st.param.b16      [func_retval0+0], [[R]];
-; CHECK:      ret;
 define bfloat @test_roundeven(bfloat %a) {
+; SM70-LABEL: test_roundeven(
+; SM70:       {
+; SM70-NEXT:    .reg .pred %p<2>;
+; SM70-NEXT:    .reg .b16 %rs<3>;
+; SM70-NEXT:    .reg .b32 %r<9>;
+; SM70-NEXT:    .reg .f32 %f<3>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.u16 %r1, [test_roundeven_param_0];
+; SM70-NEXT:    shl.b32 %r2, %r1, 16;
+; SM70-NEXT:    mov.b32 %f1, %r2;
+; SM70-NEXT:    cvt.rni.f32.f32 %f2, %f1;
+; SM70-NEXT:    mov.b32 %r3, %f2;
+; SM70-NEXT:    bfe.u32 %r4, %r3, 16, 1;
+; SM70-NEXT:    add.s32 %r5, %r4, %r3;
+; SM70-NEXT:    add.s32 %r6, %r5, 32767;
+; SM70-NEXT:    setp.nan.f32 %p1, %f2, %f2;
+; SM70-NEXT:    or.b32 %r7, %r3, 4194304;
+; SM70-NEXT:    selp.b32 %r8, %r7, %r6, %p1;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r8; }
+; SM70-NEXT:    st.param.b16 [func_retval0+0], %rs1;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_roundeven(
+; SM80:       {
+; SM80-NEXT:    .reg .b16 %rs<3>;
+; SM80-NEXT:    .reg .f32 %f<3>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.b16 %rs1, [test_roundeven_param_0];
+; SM80-NEXT:    cvt.f32.bf16 %f1, %rs1;
+; SM80-NEXT:    cvt.rni.f32.f32 %f2, %f1;
+; SM80-NEXT:    cvt.rn.bf16.f32 %rs2, %f2;
+; SM80-NEXT:    st.param.b16 [func_retval0+0], %rs2;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_roundeven(
+; SM90:       {
+; SM90-NEXT:    .reg .b16 %rs<3>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.b16 %rs1, [test_roundeven_param_0];
+; SM90-NEXT:    cvt.rni.bf16.bf16 %rs2, %rs1;
+; SM90-NEXT:    st.param.b16 [func_retval0+0], %rs2;
+; SM90-NEXT:    ret;
   %r = call bfloat @llvm.roundeven.bf16(bfloat %a)
   ret bfloat %r
 }
+
+define bfloat @test_maximum(bfloat %a, bfloat %b) {
+; SM70-LABEL: test_maximum(
+; SM70:       {
+; SM70-NEXT:    .reg .pred %p<6>;
+; SM70-NEXT:    .reg .b16 %rs<11>;
+; SM70-NEXT:    .reg .b32 %r<7>;
+; SM70-NEXT:    .reg .f32 %f<4>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.b16 %rs1, [test_maximum_param_0];
+; SM70-NEXT:    ld.param.b16 %rs3, [test_maximum_param_1];
+; SM70-NEXT:    cvt.u32.u16 %r1, %rs3;
+; SM70-NEXT:    shl.b32 %r2, %r1, 16;
+; SM70-NEXT:    mov.b32 %f1, %r2;
+; SM70-NEXT:    cvt.u32.u16 %r3, %rs1;
+; SM70-NEXT:    shl.b32 %r4, %r3, 16;
+; SM70-NEXT:    mov.b32 %f2, %r4;
+; SM70-NEXT:    setp.gt.f32 %p1, %f2, %f1;
+; SM70-NEXT:    selp.b16 %rs5, %rs1, %rs3, %p1;
+; SM70-NEXT:    setp.nan.f32 %p2, %f2, %f1;
+; SM70-NEXT:    selp.b16 %rs6, 0x7FC0, %rs5, %p2;
+; SM70-NEXT:    setp.eq.s16 %p3, %rs1, 0;
+; SM70-NEXT:    selp.b16 %rs7, %rs1, %rs6, %p3;
+; SM70-NEXT:    setp.eq.s16 %p4, %rs3, 0;
+; SM70-NEXT:    selp.b16 %rs8, %rs3, %rs7, %p4;
+; SM70-NEXT:    cvt.u32.u16 %r5, %rs6;
+; SM70-NEXT:    shl.b32 %r6, %r5, 16;
+; SM70-NEXT:    mov.b32 %f3, %r6;
+; SM70-NEXT:    setp.eq.f32 %p5, %f3, 0f00000000;
+; SM70-NEXT:    selp.b16 %rs10, %rs8, %rs6, %p5;
+; SM70-NEXT:    st.param.b16 [func_retval0+0], %rs10;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_maximum(
+; SM80:       {
+; SM80-NEXT:    .reg .b16 %rs<4>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.b16 %rs1, [test_maximum_param_0];
+; SM80-NEXT:    ld.param.b16 %rs2, [test_maximum_param_1];
+; SM80-NEXT:    max.NaN.bf16 %rs3, %rs1, %rs2;
+; SM80-NEXT:    st.param.b16 [func_retval0+0], %rs3;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_maximum(
+; SM90:       {
+; SM90-NEXT:    .reg .b16 %rs<4>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.b16 %rs1, [test_maximum_param_0];
+; SM90-NEXT:    ld.param.b16 %rs2, [test_maximum_param_1];
+; SM90-NEXT:    max.NaN.bf16 %rs3, %rs1, %rs2;
+; SM90-NEXT:    st.param.b16 [func_retval0+0], %rs3;
+; SM90-NEXT:    ret;
+  %r = call bfloat @llvm.maximum.bf16(bfloat %a, bfloat %b)
+  ret bfloat %r
+}
+
+define bfloat @test_maxnum(bfloat %a, bfloat %b) {
+; SM70-LABEL: test_maxnum(
+; SM70:       {
+; SM70-NEXT:    .reg .pred %p<2>;
+; SM70-NEXT:    .reg .b16 %rs<3>;
+; SM70-NEXT:    .reg .b32 %r<11>;
+; SM70-NEXT:    .reg .f32 %f<4>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.u16 %r1, [test_maxnum_param_1];
+; SM70-NEXT:    shl.b32 %r2, %r1, 16;
+; SM70-NEXT:    mov.b32 %f1, %r2;
+; SM70-NEXT:    ld.param.u16 %r3, [test_maxnum_param_0];
+; SM70-NEXT:    shl.b32 %r4, %r3, 16;
+; SM70-NEXT:    mov.b32 %f2, %r4;
+; SM70-NEXT:    max.f32 %f3, %f2, %f1;
+; SM70-NEXT:    mov.b32 %r5, %f3;
+; SM70-NEXT:    bfe.u32 %r6, %r5, 16, 1;
+; SM70-NEXT:    add.s32 %r7, %r6, %r5;
+; SM70-NEXT:    add.s32 %r8, %r7, 32767;
+; SM70-NEXT:    setp.nan.f32 %p1, %f3, %f3;
+; SM70-NEXT:    or.b32 %r9, %r5, 4194304;
+; SM70-NEXT:    selp.b32 %r10, %r9, %r8, %p1;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r10; }
+; SM70-NEXT:    st.param.b16 [func_retval0+0], %rs1;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_maxnum(
+; SM80:       {
+; SM80-NEXT:    .reg .b16 %rs<4>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.b16 %rs1, [test_maxnum_param_0];
+; SM80-NEXT:    ld.param.b16 %rs2, [test_maxnum_param_1];
+; SM80-NEXT:    max.bf16 %rs3, %rs1, %rs2;
+; SM80-NEXT:    st.param.b16 [func_retval0+0], %rs3;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_maxnum(
+; SM90:       {
+; SM90-NEXT:    .reg .b16 %rs<4>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.b16 %rs1, [test_maxnum_param_0];
+; SM90-NEXT:    ld.param.b16 %rs2, [test_maxnum_param_1];
+; SM90-NEXT:    max.bf16 %rs3, %rs1, %rs2;
+; SM90-NEXT:    st.param.b16 [func_retval0+0], %rs3;
+; SM90-NEXT:    ret;
+  %r = call bfloat @llvm.maxnum.bf16(bfloat %a, bfloat %b)
+  ret bfloat %r
+}
+
+define <2 x bfloat> @test_maximum_v2(<2 x bfloat> %a, <2 x bfloat> %b) {
+; SM70-LABEL: test_maximum_v2(
+; SM70:       {
+; SM70-NEXT:    .reg .pred %p<11>;
+; SM70-NEXT:    .reg .b16 %rs<21>;
+; SM70-NEXT:    .reg .b32 %r<16>;
+; SM70-NEXT:    .reg .f32 %f<7>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.b32 %r1, [test_maximum_v2_param_0];
+; SM70-NEXT:    ld.param.b32 %r2, [test_maximum_v2_param_1];
+; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM70-NEXT:    shl.b32 %r4, %r3, 16;
+; SM70-NEXT:    mov.b32 %f1, %r4;
+; SM70-NEXT:    mov.b32 {%rs4, %rs5}, %r1;
+; SM70-NEXT:    cvt.u32.u16 %r5, %rs5;
+; SM70-NEXT:    shl.b32 %r6, %r5, 16;
+; SM70-NEXT:    mov.b32 %f2, %r6;
+; SM70-NEXT:    setp.gt.f32 %p1, %f2, %f1;
+; SM70-NEXT:    selp.b16 %rs7, %rs5, %rs2, %p1;
+; SM70-NEXT:    setp.nan.f32 %p2, %f2, %f1;
+; SM70-NEXT:    selp.b16 %rs8, 0x7FC0, %rs7, %p2;
+; SM70-NEXT:    setp.eq.s16 %p3, %rs5, 0;
+; SM70-NEXT:    selp.b16 %rs9, %rs5, %rs8, %p3;
+; SM70-NEXT:    setp.eq.s16 %p4, %rs2, 0;
+; SM70-NEXT:    selp.b16 %rs10, %rs2, %rs9, %p4;
+; SM70-NEXT:    cvt.u32.u16 %r7, %rs8;
+; SM70-NEXT:    shl.b32 %r8, %r7, 16;
+; SM70-NEXT:    mov.b32 %f3, %r8;
+; SM70-NEXT:    setp.eq.f32 %p5, %f3, 0f00000000;
+; SM70-NEXT:    selp.b16 %rs12, %rs10, %rs8, %p5;
+; SM70-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM70-NEXT:    shl.b32 %r10, %r9, 16;
+; SM70-NEXT:    mov.b32 %f4, %r10;
+; SM70-NEXT:    cvt.u32.u16 %r11, %rs4;
+; SM70-NEXT:    shl.b32 %r12, %r11, 16;
+; SM70-NEXT:    mov.b32 %f5, %r12;
+; SM70-NEXT:    setp.gt.f32 %p6, %f5, %f4;
+; SM70-NEXT:    selp.b16 %rs15, %rs4, %rs1, %p6;
+; SM70-NEXT:    setp.nan.f32 %p7, %f5, %f4;
+; SM70-NEXT:    selp.b16 %rs16, 0x7FC0, %rs15, %p7;
+; SM70-NEXT:    setp.eq.s16 %p8, %rs4, 0;
+; SM70-NEXT:    selp.b16 %rs17, %rs4, %rs16, %p8;
+; SM70-NEXT:    setp.eq.s16 %p9, %rs1, 0;
+; SM70-NEXT:    selp.b16 %rs18, %rs1, %rs17, %p9;
+; SM70-NEXT:    cvt.u32.u16 %r13, %rs16;
+; SM70-NEXT:    shl.b32 %r14, %r13, 16;
+; SM70-NEXT:    mov.b32 %f6, %r14;
+; SM70-NEXT:    setp.eq.f32 %p10, %f6, 0f00000000;
+; SM70-NEXT:    selp.b16 %rs20, %rs18, %rs16, %p10;
+; SM70-NEXT:    mov.b32 %r15, {%rs20, %rs12};
+; SM70-NEXT:    st.param.b32 [func_retval0+0], %r15;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_maximum_v2(
+; SM80:       {
+; SM80-NEXT:    .reg .b32 %r<4>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.b32 %r1, [test_maximum_v2_param_1];
+; SM80-NEXT:    ld.param.b32 %r2, [test_maximum_v2_param_0];
+; SM80-NEXT:    max.NaN.bf16x2 %r3, %r2, %r1;
+; SM80-NEXT:    st.param.b32 [func_retval0+0], %r3;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_maximum_v2(
+; SM90:       {
+; SM90-NEXT:    .reg .b32 %r<4>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.b32 %r1, [test_maximum_v2_param_1];
+; SM90-NEXT:    ld.param.b32 %r2, [test_maximum_v2_param_0];
+; SM90-NEXT:    max.NaN.bf16x2 %r3, %r2, %r1;
+; SM90-NEXT:    st.param.b32 [func_retval0+0], %r3;
+; SM90-NEXT:    ret;
+  %r = call <2 x bfloat> @llvm.maximum.bf16(<2 x bfloat> %a, <2 x bfloat> %b)
+  ret <2 x bfloat> %r
+}
+
+define <2 x bfloat> @test_maxnum_v2(<2 x bfloat> %a, <2 x bfloat> %b) {
+; SM70-LABEL: test_maxnum_v2(
+; SM70:       {
+; SM70-NEXT:    .reg .pred %p<3>;
+; SM70-NEXT:    .reg .b16 %rs<13>;
+; SM70-NEXT:    .reg .b32 %r<24>;
+; SM70-NEXT:    .reg .f32 %f<7>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.b32 %r1, [test_maxnum_v2_param_0];
+; SM70-NEXT:    ld.param.b32 %r2, [test_maxnum_v2_param_1];
+; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM70-NEXT:    shl.b32 %r4, %r3, 16;
+; SM70-NEXT:    mov.b32 %f1, %r4;
+; SM70-NEXT:    mov.b32 {%rs4, %rs5}, %r1;
+; SM70-NEXT:    cvt.u32.u16 %r5, %rs5;
+; SM70-NEXT:    shl.b32 %r6, %r5, 16;
+; SM70-NEXT:    mov.b32 %f2, %r6;
+; SM70-NEXT:    max.f32 %f3, %f2, %f1;
+; SM70-NEXT:    mov.b32 %r7, %f3;
+; SM70-NEXT:    bfe.u32 %r8, %r7, 16, 1;
+; SM70-NEXT:    add.s32 %r9, %r8, %r7;
+; SM70-NEXT:    add.s32 %r10, %r9, 32767;
+; SM70-NEXT:    setp.nan.f32 %p1, %f3, %f3;
+; SM70-NEXT:    or.b32 %r11, %r7, 4194304;
+; SM70-NEXT:    selp.b32 %r12, %r11, %r10, %p1;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r12; }
+; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
+; SM70-NEXT:    shl.b32 %r14, %r13, 16;
+; SM70-NEXT:    mov.b32 %f4, %r14;
+; SM70-NEXT:    cvt.u32.u16 %r15, %rs4;
+; SM70-NEXT:    shl.b32 %r16, %r15, 16;
+; SM70-NEXT:    mov.b32 %f5, %r16;
+; SM70-NEXT:    max.f32 %f6, %f5, %f4;
+; SM70-NEXT:    mov.b32 %r17, %f6;
+; SM70-NEXT:    bfe.u32 %r18, %r17, 16, 1;
+; SM70-NEXT:    add.s32 %r19, %r18, %r17;
+; SM70-NEXT:    add.s32 %r20, %r19, 32767;
+; SM70-NEXT:    setp.nan.f32 %p2, %f6, %f6;
+; SM70-NEXT:    or.b32 %r21, %r17, 4194304;
+; SM70-NEXT:    selp.b32 %r22, %r21, %r20, %p2;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs11}, %r22; }
+; SM70-NEXT:    mov.b32 %r23, {%rs11, %rs7};
+; SM70-NEXT:    st.param.b32 [func_retval0+0], %r23;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_maxnum_v2(
+; SM80:       {
+; SM80-NEXT:    .reg .b32 %r<4>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.b32 %r1, [test_maxnum_v2_param_1];
+; SM80-NEXT:    ld.param.b32 %r2, [test_maxnum_v2_param_0];
+; SM80-NEXT:    max.bf16x2 %r3, %r2, %r1;
+; SM80-NEXT:    st.param.b32 [func_retval0+0], %r3;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_maxnum_v2(
+; SM90:       {
+; SM90-NEXT:    .reg .b32 %r<4>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.b32 %r1, [test_maxnum_v2_param_1];
+; SM90-NEXT:    ld.param.b32 %r2, [test_maxnum_v2_param_0];
+; SM90-NEXT:    max.bf16x2 %r3, %r2, %r1;
+; SM90-NEXT:    st.param.b32 [func_retval0+0], %r3;
+; SM90-NEXT:    ret;
+  %r = call <2 x bfloat> @llvm.maxnum.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b)
+  ret <2 x bfloat> %r
+}

From f9031f00f2c90bc0af274b45ec3e169b5250a688 Mon Sep 17 00:00:00 2001
From: ChiaHungDuan <chiahungduan@google.com>
Date: Mon, 19 Aug 2024 22:25:52 -0700
Subject: [PATCH 441/441] Revert "[scudo] Add partial chunk heuristic to
 retrieval algorithm." (#104894)

Reverts llvm/llvm-project#104807
---
 compiler-rt/lib/scudo/standalone/secondary.h  | 121 +++++-------------
 .../scudo/standalone/tests/secondary_test.cpp |  36 +-----
 2 files changed, 34 insertions(+), 123 deletions(-)

diff --git a/compiler-rt/lib/scudo/standalone/secondary.h b/compiler-rt/lib/scudo/standalone/secondary.h
index 7a563fa64f7192f..27f8697db7838f5 100644
--- a/compiler-rt/lib/scudo/standalone/secondary.h
+++ b/compiler-rt/lib/scudo/standalone/secondary.h
@@ -72,14 +72,6 @@ namespace {
 struct CachedBlock {
   static constexpr u16 CacheIndexMax = UINT16_MAX;
   static constexpr u16 InvalidEntry = CacheIndexMax;
-  //   * ReleaseMemoryUpperBound default is currently 16 KB
-  //        - We arrived at this value after noticing that mapping
-  //        in larger memory regions performs better than releasing
-  //        memory and forcing a cache hit. According to the data,
-  //        it suggests that beyond 16KB, the release execution time is
-  //        longer than the map execution time. In this way, the default
-  //        is dependent on the platform.
-  static constexpr uptr ReleaseMemoryUpperBound = 1 << 14;
 
   uptr CommitBase = 0;
   uptr CommitSize = 0;
@@ -98,9 +90,8 @@ struct CachedBlock {
 template <typename Config> class MapAllocatorNoCache {
 public:
   void init(UNUSED s32 ReleaseToOsInterval) {}
-  CachedBlock retrieve(UNUSED uptr MaxAllowedFragmentedBytes, UNUSED uptr Size,
-                       UNUSED uptr Alignment, UNUSED uptr HeadersSize,
-                       UNUSED uptr &EntryHeaderPos) {
+  CachedBlock retrieve(UNUSED uptr Size, UNUSED uptr Alignment,
+                       UNUSED uptr HeadersSize, UNUSED uptr &EntryHeaderPos) {
     return {};
   }
   void store(UNUSED Options Options, UNUSED uptr CommitBase,
@@ -343,14 +334,13 @@ class MapAllocatorCache {
     }
   }
 
-  CachedBlock retrieve(uptr MaxAllowedFragmentedBytes, uptr Size,
-                       uptr Alignment, uptr HeadersSize, uptr &EntryHeaderPos)
-      EXCLUDES(Mutex) {
+  CachedBlock retrieve(uptr Size, uptr Alignment, uptr HeadersSize,
+                       uptr &EntryHeaderPos) EXCLUDES(Mutex) {
     const uptr PageSize = getPageSizeCached();
     // 10% of the requested size proved to be the optimal choice for
     // retrieving cached blocks after testing several options.
     constexpr u32 FragmentedBytesDivisor = 10;
-    bool FoundOptimalFit = false;
+    bool Found = false;
     CachedBlock Entry;
     EntryHeaderPos = 0;
     {
@@ -358,88 +348,47 @@ class MapAllocatorCache {
       CallsToRetrieve++;
       if (EntriesCount == 0)
         return {};
-      u16 RetrievedIndex = CachedBlock::InvalidEntry;
+      u32 OptimalFitIndex = 0;
       uptr MinDiff = UINTPTR_MAX;
-
-      //  Since allocation sizes don't always match cached memory chunk sizes
-      //  we allow some memory to be unused (called fragmented bytes). The
-      //  amount of unused bytes is exactly EntryHeaderPos - CommitBase.
-      //
-      //        CommitBase                CommitBase + CommitSize
-      //          V                              V
-      //      +---+------------+-----------------+---+
-      //      |   |            |                 |   |
-      //      +---+------------+-----------------+---+
-      //      ^                ^                     ^
-      //    Guard         EntryHeaderPos          Guard-page-end
-      //    page-begin
-      //
-      //  [EntryHeaderPos, CommitBase + CommitSize) contains the user data as
-      //  well as the header metadata. If EntryHeaderPos - CommitBase exceeds
-      //  MaxAllowedFragmentedBytes, the cached memory chunk is not considered
-      //  valid for retrieval.
-      for (u16 I = LRUHead; I != CachedBlock::InvalidEntry;
+      for (u32 I = LRUHead; I != CachedBlock::InvalidEntry;
            I = Entries[I].Next) {
         const uptr CommitBase = Entries[I].CommitBase;
         const uptr CommitSize = Entries[I].CommitSize;
         const uptr AllocPos =
             roundDown(CommitBase + CommitSize - Size, Alignment);
         const uptr HeaderPos = AllocPos - HeadersSize;
-        if (HeaderPos > CommitBase + CommitSize || HeaderPos < CommitBase)
+        if (HeaderPos > CommitBase + CommitSize)
           continue;
-
-        const uptr Diff = roundDown(HeaderPos, PageSize) - CommitBase;
-
-        if (Diff > MaxAllowedFragmentedBytes || Diff >= MinDiff)
+        if (HeaderPos < CommitBase ||
+            AllocPos > CommitBase + PageSize * MaxUnusedCachePages) {
           continue;
-
-        MinDiff = Diff;
-        RetrievedIndex = I;
-        EntryHeaderPos = HeaderPos;
-
-        const uptr OptimalFitThesholdBytes =
+        }
+        Found = true;
+        const uptr Diff = HeaderPos - CommitBase;
+        // immediately use a cached block if it's size is close enough to the
+        // requested size.
+        const uptr MaxAllowedFragmentedBytes =
             (CommitBase + CommitSize - HeaderPos) / FragmentedBytesDivisor;
-        if (Diff <= OptimalFitThesholdBytes) {
-          FoundOptimalFit = true;
+        if (Diff <= MaxAllowedFragmentedBytes) {
+          OptimalFitIndex = I;
+          EntryHeaderPos = HeaderPos;
           break;
         }
+        // keep track of the smallest cached block
+        // that is greater than (AllocSize + HeaderSize)
+        if (Diff > MinDiff)
+          continue;
+        OptimalFitIndex = I;
+        MinDiff = Diff;
+        EntryHeaderPos = HeaderPos;
       }
-      if (RetrievedIndex != CachedBlock::InvalidEntry) {
-        Entry = Entries[RetrievedIndex];
-        remove(RetrievedIndex);
+      if (Found) {
+        Entry = Entries[OptimalFitIndex];
+        remove(OptimalFitIndex);
         SuccessfulRetrieves++;
       }
     }
 
-    //  The difference between the retrieved memory chunk and the request
-    //  size is at most MaxAllowedFragmentedBytes
-    //
-    //  /     MaxAllowedFragmentedBytes      \
-    // +--------------------------+-----------+
-    // |                          |           |
-    // +--------------------------+-----------+
-    //  \ Bytes to be released   /      ^
-    //                                  |
-    //                         (may or may not be commited)
-    //
-    //   The maximum number of bytes released to the OS is capped by
-    //   ReleaseMemoryUpperBound
-    //
-    //   TODO : Consider making ReleaseMemoryUpperBound configurable since
-    //   the release to OS API can vary across systems.
-    if (!FoundOptimalFit && Entry.Time != 0) {
-      const uptr FragmentedBytes =
-          roundDown(EntryHeaderPos, PageSize) - Entry.CommitBase;
-      const uptr MaxUnusedCacheBytes = MaxUnusedCachePages * PageSize;
-      if (FragmentedBytes > MaxUnusedCacheBytes) {
-        uptr BytesToRelease =
-            roundUp(Min<uptr>(CachedBlock::ReleaseMemoryUpperBound,
-                              FragmentedBytes - MaxUnusedCacheBytes),
-                    PageSize);
-        Entry.MemMap.releaseAndZeroPagesToOS(Entry.CommitBase, BytesToRelease);
-      }
-    }
-
     return Entry;
   }
 
@@ -710,18 +659,8 @@ MapAllocator<Config>::tryAllocateFromCache(const Options &Options, uptr Size,
                                            FillContentsMode FillContents) {
   CachedBlock Entry;
   uptr EntryHeaderPos;
-  uptr MaxAllowedFragmentedBytes;
-  const uptr PageSize = getPageSizeCached();
-
-  if (LIKELY(!useMemoryTagging<Config>(Options))) {
-    MaxAllowedFragmentedBytes =
-        MaxUnusedCachePages * PageSize + CachedBlock::ReleaseMemoryUpperBound;
-  } else {
-    MaxAllowedFragmentedBytes = MaxUnusedCachePages * PageSize;
-  }
 
-  Entry = Cache.retrieve(MaxAllowedFragmentedBytes, Size, Alignment,
-                         getHeadersSize(), EntryHeaderPos);
+  Entry = Cache.retrieve(Size, Alignment, getHeadersSize(), EntryHeaderPos);
   if (!Entry.isValid())
     return nullptr;
 
diff --git a/compiler-rt/lib/scudo/standalone/tests/secondary_test.cpp b/compiler-rt/lib/scudo/standalone/tests/secondary_test.cpp
index e33e93c9ff78d48..e85b6abdb36d228 100644
--- a/compiler-rt/lib/scudo/standalone/tests/secondary_test.cpp
+++ b/compiler-rt/lib/scudo/standalone/tests/secondary_test.cpp
@@ -281,8 +281,8 @@ struct MapAllocatorCacheTest : public Test {
   std::unique_ptr<CacheT> Cache = std::make_unique<CacheT>();
 
   const scudo::uptr PageSize = scudo::getPageSizeCached();
-  // The current test allocation size is set to the maximum
-  // cache entry size
+  // The current test allocation size is set to the minimum size
+  // needed for the scudo allocator to fall back to the secondary allocator
   static constexpr scudo::uptr TestAllocSize =
       CacheConfig::getDefaultMaxEntrySize();
 
@@ -327,8 +327,7 @@ TEST_F(MapAllocatorCacheTest, CacheOrder) {
   for (scudo::uptr I = CacheConfig::getEntriesArraySize(); I > 0; I--) {
     scudo::uptr EntryHeaderPos;
     scudo::CachedBlock Entry =
-        Cache->retrieve(scudo::MaxUnusedCachePages * PageSize, TestAllocSize,
-                        PageSize, 0, EntryHeaderPos);
+        Cache->retrieve(TestAllocSize, PageSize, 0, EntryHeaderPos);
     EXPECT_EQ(Entry.MemMap.getBase(), MemMaps[I - 1].getBase());
   }
 
@@ -337,32 +336,6 @@ TEST_F(MapAllocatorCacheTest, CacheOrder) {
     MemMap.unmap();
 }
 
-TEST_F(MapAllocatorCacheTest, PartialChunkHeuristicRetrievalTest) {
-  const scudo::uptr MaxUnusedCacheBytes = PageSize;
-  const scudo::uptr FragmentedBytes =
-      MaxUnusedCacheBytes + scudo::CachedBlock::ReleaseMemoryUpperBound;
-
-  scudo::uptr EntryHeaderPos;
-  scudo::CachedBlock Entry;
-  scudo::MemMapT MemMap = allocate(PageSize + FragmentedBytes);
-  Cache->store(Options, MemMap.getBase(), MemMap.getCapacity(),
-               MemMap.getBase(), MemMap);
-
-  // FragmentedBytes > MaxAllowedFragmentedBytes so PageSize
-  // cannot be retrieved from the cache
-  Entry = Cache->retrieve(/*MaxAllowedFragmentedBytes=*/0, PageSize, PageSize,
-                          0, EntryHeaderPos);
-  EXPECT_FALSE(Entry.isValid());
-
-  // FragmentedBytes <= MaxAllowedFragmentedBytes so PageSize
-  // can be retrieved from the cache
-  Entry =
-      Cache->retrieve(FragmentedBytes, PageSize, PageSize, 0, EntryHeaderPos);
-  EXPECT_TRUE(Entry.isValid());
-
-  MemMap.unmap();
-}
-
 TEST_F(MapAllocatorCacheTest, MemoryLeakTest) {
   std::vector<scudo::MemMapT> MemMaps;
   // Fill the cache above MaxEntriesCount to force an eviction
@@ -378,8 +351,7 @@ TEST_F(MapAllocatorCacheTest, MemoryLeakTest) {
   for (scudo::uptr I = CacheConfig::getDefaultMaxEntriesCount(); I > 0; I--) {
     scudo::uptr EntryHeaderPos;
     RetrievedEntries.push_back(
-        Cache->retrieve(scudo::MaxUnusedCachePages * PageSize, TestAllocSize,
-                        PageSize, 0, EntryHeaderPos));
+        Cache->retrieve(TestAllocSize, PageSize, 0, EntryHeaderPos));
     EXPECT_EQ(MemMaps[I].getBase(), RetrievedEntries.back().MemMap.getBase());
   }